Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.File;
019import java.io.FileInputStream;
020import java.io.IOException;
021import java.io.InputStream;
022import java.nio.charset.Charset;
023import java.nio.file.Files;
024import java.nio.file.Path;
025import java.nio.file.Paths;
026import java.nio.file.StandardCopyOption;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.UUID;
032import java.util.function.Function;
033import java.util.stream.Collectors;
034
035import org.apache.avalon.framework.component.Component;
036import org.apache.avalon.framework.context.Context;
037import org.apache.avalon.framework.context.ContextException;
038import org.apache.avalon.framework.context.Contextualizable;
039import org.apache.avalon.framework.service.ServiceException;
040import org.apache.avalon.framework.service.ServiceManager;
041import org.apache.avalon.framework.service.Serviceable;
042import org.apache.cocoon.components.ContextHelper;
043import org.apache.cocoon.environment.Request;
044import org.apache.cocoon.servlet.multipart.Part;
045import org.apache.cocoon.servlet.multipart.PartOnDisk;
046import org.apache.cocoon.servlet.multipart.RejectedPart;
047import org.apache.commons.io.FileUtils;
048import org.apache.commons.io.FilenameUtils;
049import org.apache.commons.io.IOUtils;
050import org.apache.commons.lang3.StringUtils;
051import org.supercsv.prefs.CsvPreference;
052
053import org.ametys.cms.contenttype.ContentType;
054import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
055import org.ametys.cms.data.type.ModelItemTypeConstants;
056import org.ametys.core.cocoon.JSonReader;
057import org.ametys.core.ui.Callable;
058import org.ametys.plugins.workflow.support.WorkflowHelper;
059import org.ametys.runtime.i18n.I18nizableText;
060import org.ametys.runtime.model.Enumerator;
061import org.ametys.runtime.model.ModelItem;
062import org.ametys.runtime.servlet.RuntimeConfig;
063
064/**
065 * Import contents from an uploaded CSV file.
066 */
067public class ImportCSVFile implements Serviceable, Contextualizable, Component
068{
069    /** Avalon Role */
070    public static final String ROLE = ImportCSVFile.class.getName();
071    
072    /** The ametys home folder where csv files are stored before import */
073    static final String CONTENTIO_STORAGE_DIRECTORY = "contentio/temp"; // FIXME should be relate to AmetysTmp not AmetysHome CONTENTIO-334
074    
075    private static final String[] _ALLOWED_EXTENSIONS = new String[] {"txt", "csv"};
076    
077    private ContentTypeExtensionPoint _contentTypeEP;
078    
079    private WorkflowHelper _workflowHelper;
080    
081    private ImportCSVFileHelper _importCSVFileHelper;
082
083    private Enumerator _synchronizeModeEnumerator;
084    
085    private Context _context;
086    
087    @Override
088    public void service(ServiceManager serviceManager) throws ServiceException
089    {
090        _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
091        _workflowHelper = (WorkflowHelper) serviceManager.lookup(WorkflowHelper.ROLE);
092        _importCSVFileHelper = (ImportCSVFileHelper) serviceManager.lookup(ImportCSVFileHelper.ROLE);
093        _synchronizeModeEnumerator = (Enumerator) serviceManager.lookup(SynchronizeModeEnumerator.ROLE);
094    }
095    
096    public void contextualize(Context context) throws ContextException
097    {
098        _context = context;
099    }
100    
101    /**
102     * Import CSV file
103     * @param escapingChar The escaping character
104     * @param separatingChar The separating character
105     * @param contentTypeId The content type id
106     * @param part The CSV file's Part
107     * @return The result of the import
108     * @throws Exception If an error occurs
109     */
110    @Callable (rights = "Plugins_ContentIO_ImportFile", context = "/cms")
111    public Map importCSVFile(String escapingChar, String separatingChar, String contentTypeId, Part part) throws Exception
112    {
113        Request request = ContextHelper.getRequest(_context);
114        
115        Map<String, Object> result = new HashMap<>();
116        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
117
118        if (part instanceof RejectedPart || part == null)
119        {
120            result.put("success", false);
121            result.put("error", "rejected");
122        }
123        else
124        {
125            PartOnDisk uploadedFilePart = (PartOnDisk) part;
126            File uploadedFile = uploadedFilePart.getFile();
127            
128            String filename = uploadedFilePart.getFileName().toLowerCase();
129            
130            if (!FilenameUtils.isExtension(filename, _ALLOWED_EXTENSIONS))
131            {
132                result.put("error", "invalid-extension");
133                request.setAttribute(JSonReader.OBJECT_TO_READ, result);
134                return result;
135            }
136            
137            try (
138                InputStream fileInputStream = new FileInputStream(uploadedFile);
139                InputStream inputStream = IOUtils.buffer(fileInputStream);
140            )
141            {
142                Charset charset = _importCSVFileHelper.detectCharset(inputStream);
143                result.put("charset", charset);
144                String[] headers = _importCSVFileHelper.extractHeaders(inputStream, _getCSVPreference(escapingChar, separatingChar), charset);
145              
146                if (headers == null)
147                {
148                    result.put("error", "no-header");
149                    request.setAttribute(JSonReader.OBJECT_TO_READ, result);
150                    return result;
151                }
152                
153                List<Map<String, Object>> mapping = _importCSVFileHelper.getMapping(contentType, headers);
154                result.put("mapping", mapping);
155                
156                // Build a map to count how many attribute are there for each group
157                Map<String, Long> attributeCount = mapping.stream()
158                        .map(map -> map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH))
159                        .map(String.class::cast)
160                        .filter(StringUtils::isNotEmpty)
161                        .map(attributePath -> {
162                            String attributePrefix = "";
163                            int endIndex = attributePath.lastIndexOf("/");
164                            if (endIndex != -1)
165                            {
166                                attributePrefix = attributePath.substring(0, endIndex);
167                            }
168                            return attributePrefix;
169                        })
170                        .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
171                
172                // If an attribute is the only for its group, it is the identifier
173                for (Map<String, Object> map : mapping)
174                {
175                    String attributePath = (String) map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH);
176                    String attributePrefix = StringUtils.EMPTY;
177                    int endIndex = attributePath.lastIndexOf("/");
178                    if (endIndex != -1)
179                    {
180                        attributePrefix = attributePath.substring(0, endIndex);
181                    }
182    
183                    boolean parentIsContent;
184                    // If there is no prefix, it is an attribute of the content we want to import
185                    if (attributePrefix.equals(StringUtils.EMPTY))
186                    {
187                        parentIsContent = true;
188                    }
189                    // Otherwise, check the modelItem
190                    else if (contentType.hasModelItem(attributePrefix))
191                    {
192                        ModelItem modelItem = contentType.getModelItem(attributePrefix);
193                        String modelTypeId = modelItem.getType().getId();
194                        parentIsContent = ModelItemTypeConstants.CONTENT_ELEMENT_TYPE_ID.equals(modelTypeId);
195                    }
196                    else
197                    {
198                        parentIsContent = false;
199                    }
200                    
201                    // If an attribute is the only one of its level, and is part of a content, consider it as the identifier
202                    if (attributeCount.getOrDefault(attributePrefix, 0L).equals(1L) && parentIsContent && StringUtils.isNotBlank(attributePath))
203                    {
204                        map.put(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID, true);
205                    }
206                }
207                
208                String importId = UUID.randomUUID().toString();
209                _copyFile(uploadedFile.toPath(), importId);
210                result.put("importId", importId);
211                result.put("success", true);
212                result.put("workflows", _getWorkflows());
213                result.put("defaultWorkflow", contentType.getDefaultWorkflowName().orElse(null));
214                result.put("fileName", uploadedFile.getName());
215                
216                List<Map<String, Object>> synchronizeModes = ((Map<String, I18nizableText>) _synchronizeModeEnumerator.getEntries())
217                        .entrySet()
218                        .stream()
219                        .map(entry -> Map.of("value", entry.getKey(), "label", entry.getValue()))
220                        .toList();
221                result.put("synchronizeModes", synchronizeModes);
222            }
223        }
224        
225        request.setAttribute(JSonReader.OBJECT_TO_READ, result);
226        return result;
227    }
228    
229    private void _copyFile(Path path, String importId) throws IOException
230    {
231        File contentIOStorageDir = FileUtils.getFile(RuntimeConfig.getInstance().getAmetysHome(), CONTENTIO_STORAGE_DIRECTORY);
232        
233        if (!contentIOStorageDir.exists())
234        {
235            if (!contentIOStorageDir.mkdirs())
236            {
237                throw new IOException("Unable to create monitoring directory: " + contentIOStorageDir);
238            }
239        }
240        Files.copy(path, Paths.get(contentIOStorageDir.getPath(), importId + ".csv"), StandardCopyOption.REPLACE_EXISTING);
241    }
242    
243    private CsvPreference _getCSVPreference(String escapingChar, String separatingChar)
244    {
245        char separating = separatingChar.charAt(0);
246        char escaping = escapingChar.charAt(0);
247        
248        if (separating == escaping)
249        {
250            throw new IllegalArgumentException("Separating character can not be equals to escaping character");
251        }
252        return new CsvPreference.Builder(escaping, separating, "\r\n").build();
253    }
254    
255    /**
256     * getWorkflows
257     * @return map of workflows
258     */
259    private List<Map<String, Object>> _getWorkflows()
260    {
261        List<Map<String, Object>> workflows = new ArrayList<>();
262        String[] workflowNames = _workflowHelper.getWorkflowNames();
263        for (String workflowName : workflowNames)
264        {
265            Map<String, Object> workflowMap = new HashMap<>();
266            workflowMap.put("value", workflowName);
267            workflowMap.put("label", _workflowHelper.getWorkflowLabel(workflowName));
268            workflows.add(workflowMap);
269        }
270        return workflows;
271    }
272}