Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.File;
019import java.io.FileInputStream;
020import java.io.IOException;
021import java.io.InputStream;
022import java.nio.charset.Charset;
023import java.nio.file.Files;
024import java.nio.file.Path;
025import java.nio.file.Paths;
026import java.nio.file.StandardCopyOption;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.UUID;
032import java.util.function.Function;
033import java.util.stream.Collectors;
034
035import org.apache.avalon.framework.parameters.Parameters;
036import org.apache.avalon.framework.service.ServiceException;
037import org.apache.avalon.framework.service.ServiceManager;
038import org.apache.cocoon.acting.ServiceableAction;
039import org.apache.cocoon.environment.ObjectModelHelper;
040import org.apache.cocoon.environment.Redirector;
041import org.apache.cocoon.environment.Request;
042import org.apache.cocoon.environment.SourceResolver;
043import org.apache.cocoon.servlet.multipart.Part;
044import org.apache.cocoon.servlet.multipart.PartOnDisk;
045import org.apache.cocoon.servlet.multipart.RejectedPart;
046import org.apache.commons.io.FileUtils;
047import org.apache.commons.io.FilenameUtils;
048import org.apache.commons.io.IOUtils;
049import org.apache.commons.lang3.StringUtils;
050import org.supercsv.prefs.CsvPreference;
051
052import org.ametys.cms.contenttype.ContentType;
053import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
054import org.ametys.cms.data.type.ModelItemTypeConstants;
055import org.ametys.core.cocoon.JSonReader;
056import org.ametys.plugins.workflow.support.WorkflowHelper;
057import org.ametys.runtime.i18n.I18nizableText;
058import org.ametys.runtime.model.Enumerator;
059import org.ametys.runtime.model.ModelItem;
060import org.ametys.runtime.servlet.RuntimeConfig;
061
062/**
063 * Import contents from an uploaded CSV file.
064 */
065public class ImportCSVFileAction extends ServiceableAction
066{
067
068    private static final String[] _ALLOWED_EXTENSIONS = new String[] {"txt", "csv"};
069
070    private static final String CONTENTIO_STORAGE_DIRECTORY = "contentio/temp";
071
072    private ContentTypeExtensionPoint _contentTypeEP;
073    
074    private WorkflowHelper _workflowHelper;
075    
076    private ImportCSVFileHelper _importCSVFileHelper;
077
078    private Enumerator _synchronizeModeEnumerator;
079    
080    @Override
081    public void service(ServiceManager serviceManager) throws ServiceException
082    {
083        super.service(serviceManager);
084        _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
085        _workflowHelper = (WorkflowHelper) serviceManager.lookup(WorkflowHelper.ROLE);
086        _importCSVFileHelper = (ImportCSVFileHelper) manager.lookup(ImportCSVFileHelper.ROLE);
087        _synchronizeModeEnumerator = (Enumerator) serviceManager.lookup(SynchronizeModeEnumerator.ROLE);
088    }
089    
090    @Override
091    public Map act(Redirector redirector, SourceResolver resolver, Map objectModel, String source, Parameters parameters) throws Exception
092    {
093        Request request = ObjectModelHelper.getRequest(objectModel);
094        
095        Map<String, Object> result = new HashMap<>();
096        String contentTypeId = (String) request.get("contentType");
097        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
098        Part part = (Part) request.get("file");
099        if (part instanceof RejectedPart || part == null)
100        {
101            result.put("success", false);
102            result.put("error", "rejected");
103        }
104        else
105        {
106            PartOnDisk uploadedFilePart = (PartOnDisk) part;
107            File uploadedFile = uploadedFilePart.getFile();
108            
109            String filename = uploadedFilePart.getFileName().toLowerCase();
110            
111            if (!FilenameUtils.isExtension(filename, _ALLOWED_EXTENSIONS))
112            {
113                result.put("error", "invalid-extension");
114                request.setAttribute(JSonReader.OBJECT_TO_READ, result);
115                return EMPTY_MAP;
116            }
117            
118            try (
119                InputStream fileInputStream = new FileInputStream(uploadedFile);
120                InputStream inputStream = IOUtils.buffer(fileInputStream);
121            )
122            {
123                Charset charset = _importCSVFileHelper.detectCharset(inputStream);
124                result.put("charset", charset);
125                String[] headers = _importCSVFileHelper.extractHeaders(inputStream, _getCSVPreference(request), charset);
126              
127                if (headers == null)
128                {
129                    result.put("error", "no-header");
130                    request.setAttribute(JSonReader.OBJECT_TO_READ, result);
131                    return EMPTY_MAP;
132                }
133                
134                List<Map<String, Object>> mapping = _importCSVFileHelper.getMapping(contentType, headers);
135                result.put("mapping", mapping);
136                
137                // Build a map to count how many attribute are there for each group
138                Map<String, Long> attributeCount = mapping.stream()
139                        .map(map -> map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH))
140                        .map(String.class::cast)
141                        .filter(StringUtils::isNotEmpty)
142                        .map(attributePath -> {
143                            String attributePrefix = "";
144                            int endIndex = attributePath.lastIndexOf("/");
145                            if (endIndex != -1)  
146                            {
147                                attributePrefix = attributePath.substring(0, endIndex);
148                            }
149                            return attributePrefix;
150                        })
151                        .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
152                
153                // If an attribute is the only for its group, it is the identifier
154                for (Map<String, Object> map : mapping)
155                {
156                    String attributePath = (String) map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH);
157                    String attributePrefix = StringUtils.EMPTY;
158                    int endIndex = attributePath.lastIndexOf("/");
159                    if (endIndex != -1)  
160                    {
161                        attributePrefix = attributePath.substring(0, endIndex);
162                    }
163    
164                    boolean parentIsContent;
165                    // If there is no prefix, it is an attribute of the content we want to import
166                    if (attributePrefix.equals(StringUtils.EMPTY))
167                    {
168                        parentIsContent = true;
169                    }
170                    // Otherwise, check the modelItem 
171                    else if (contentType.hasModelItem(attributePrefix))
172                    {
173                        ModelItem modelItem = contentType.getModelItem(attributePrefix);
174                        String modelTypeId = modelItem.getType().getId();
175                        parentIsContent = ModelItemTypeConstants.CONTENT_ELEMENT_TYPE_ID.equals(modelTypeId);
176                    }
177                    else
178                    {
179                        parentIsContent = false;
180                    }
181                    
182                    // If an attribute is the only one of its level, and is part of a content, consider it as the identifier
183                    if (attributeCount.getOrDefault(attributePrefix, 0L).equals(1L) && parentIsContent && StringUtils.isNotBlank(attributePath))
184                    {
185                        map.put(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID, true);
186                    }
187                }
188                
189                String newPath = _copyFile(uploadedFile.toPath());
190                result.put("path", newPath);
191                result.put("success", true);
192                result.put("workflows", _getWorkflows());
193                result.put("defaultWorkflow", contentType.getDefaultWorkflowName().orElse(null));
194                result.put("fileName", uploadedFile.getName());
195                
196                List<Map<String, Object>> synchronizeModes = ((Map<String, I18nizableText>) _synchronizeModeEnumerator.getTypedEntries())
197                        .entrySet()
198                        .stream()
199                        .map(entry -> Map.of("value", entry.getKey(), "label", entry.getValue()))
200                        .toList();
201                result.put("synchronizeModes", synchronizeModes);
202            }
203        }
204        
205        request.setAttribute(JSonReader.OBJECT_TO_READ, result);
206        return EMPTY_MAP;
207    }
208    
209    private String _copyFile(Path path) throws IOException
210    {
211        File contentIOStorageDir = FileUtils.getFile(RuntimeConfig.getInstance().getAmetysHome(), CONTENTIO_STORAGE_DIRECTORY);
212        
213        if (!contentIOStorageDir.exists())
214        {
215            if (!contentIOStorageDir.mkdirs())
216            {
217                throw new IOException("Unable to create monitoring directory: " + contentIOStorageDir);
218            }
219        }
220        String id = UUID.randomUUID().toString() + ".csv";
221        Path copy = Files.copy(path, Paths.get(contentIOStorageDir.getPath(), id), StandardCopyOption.REPLACE_EXISTING);
222        
223        return copy.toString();
224        
225    }
226    
227    private CsvPreference _getCSVPreference(Request request)
228    {
229        String escapingChar = (String) request.get("escaping-char");
230        String separatingChar = (String) request.get("separating-char");
231        char separating = separatingChar.charAt(0);
232        char escaping = escapingChar.charAt(0);
233        
234        if (separating == escaping)
235        {
236            throw new IllegalArgumentException("Separating character can not be equals to escaping character");
237        }
238        return new CsvPreference.Builder(escaping, separating, "\r\n").build();
239    }
240    
241    /**
242     * getWorkflows
243     * @return map of workflows
244     */
245    private List<Map<String, Object>> _getWorkflows()
246    {
247        List<Map<String, Object>> workflows = new ArrayList<>();
248        String[] workflowNames = _workflowHelper.getWorkflowNames();
249        for (String workflowName : workflowNames)
250        {
251            Map<String, Object> workflowMap = new HashMap<>();
252            workflowMap.put("value", workflowName);
253            workflowMap.put("label", _workflowHelper.getWorkflowLabel(workflowName));
254            workflows.add(workflowMap);
255        }
256        return workflows;
257    }
258}