Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.File;
019import java.io.FileInputStream;
020import java.io.IOException;
021import java.io.InputStream;
022import java.nio.charset.Charset;
023import java.nio.file.Files;
024import java.nio.file.Path;
025import java.nio.file.Paths;
026import java.nio.file.StandardCopyOption;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.UUID;
032import java.util.function.Function;
033import java.util.stream.Collectors;
034
035import org.apache.avalon.framework.component.Component;
036import org.apache.avalon.framework.context.Context;
037import org.apache.avalon.framework.context.ContextException;
038import org.apache.avalon.framework.context.Contextualizable;
039import org.apache.avalon.framework.service.ServiceException;
040import org.apache.avalon.framework.service.ServiceManager;
041import org.apache.avalon.framework.service.Serviceable;
042import org.apache.cocoon.components.ContextHelper;
043import org.apache.cocoon.environment.Request;
044import org.apache.cocoon.servlet.multipart.Part;
045import org.apache.cocoon.servlet.multipart.PartOnDisk;
046import org.apache.cocoon.servlet.multipart.RejectedPart;
047import org.apache.commons.io.FileUtils;
048import org.apache.commons.io.FilenameUtils;
049import org.apache.commons.io.IOUtils;
050import org.apache.commons.lang3.StringUtils;
051import org.supercsv.prefs.CsvPreference;
052
053import org.ametys.cms.contenttype.ContentType;
054import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
055import org.ametys.cms.data.type.ModelItemTypeConstants;
056import org.ametys.core.cocoon.JSonReader;
057import org.ametys.core.ui.Callable;
058import org.ametys.plugins.workflow.support.WorkflowHelper;
059import org.ametys.runtime.i18n.I18nizableText;
060import org.ametys.runtime.model.Enumerator;
061import org.ametys.runtime.model.ModelItem;
062import org.ametys.runtime.servlet.RuntimeConfig;
063
064/**
065 * Import contents from an uploaded CSV file.
066 */
067public class ImportCSVFile implements Serviceable, Contextualizable, Component
068{
069    /** Avalon Role */
070    public static final String ROLE = ImportCSVFile.class.getName();
071    
072    private static final String[] _ALLOWED_EXTENSIONS = new String[] {"txt", "csv"};
073
074    private static final String CONTENTIO_STORAGE_DIRECTORY = "contentio/temp";
075
076    private ContentTypeExtensionPoint _contentTypeEP;
077    
078    private WorkflowHelper _workflowHelper;
079    
080    private ImportCSVFileHelper _importCSVFileHelper;
081
082    private Enumerator _synchronizeModeEnumerator;
083    
084    private Context _context;
085    
086    @Override
087    public void service(ServiceManager serviceManager) throws ServiceException
088    {
089        _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
090        _workflowHelper = (WorkflowHelper) serviceManager.lookup(WorkflowHelper.ROLE);
091        _importCSVFileHelper = (ImportCSVFileHelper) serviceManager.lookup(ImportCSVFileHelper.ROLE);
092        _synchronizeModeEnumerator = (Enumerator) serviceManager.lookup(SynchronizeModeEnumerator.ROLE);
093    }
094    
095    public void contextualize(Context context) throws ContextException
096    {
097        _context = context;
098    }
099    
100    /**
101     * Import CSV file
102     * @param escapingChar The escaping character
103     * @param separatingChar The separating character
104     * @param contentTypeId The content type id
105     * @param part The CSV file's Part
106     * @return The result of the import
107     * @throws Exception If an error occurs
108     */
109    @Callable (rights = "Plugins_ContentIO_ImportFile", context = "/cms")
110    public Map importCSVFile(String escapingChar, String separatingChar, String contentTypeId, Part part) throws Exception
111    {
112        Request request = ContextHelper.getRequest(_context);
113        
114        Map<String, Object> result = new HashMap<>();
115        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
116
117        if (part instanceof RejectedPart || part == null)
118        {
119            result.put("success", false);
120            result.put("error", "rejected");
121        }
122        else
123        {
124            PartOnDisk uploadedFilePart = (PartOnDisk) part;
125            File uploadedFile = uploadedFilePart.getFile();
126            
127            String filename = uploadedFilePart.getFileName().toLowerCase();
128            
129            if (!FilenameUtils.isExtension(filename, _ALLOWED_EXTENSIONS))
130            {
131                result.put("error", "invalid-extension");
132                request.setAttribute(JSonReader.OBJECT_TO_READ, result);
133                return result;
134            }
135            
136            try (
137                InputStream fileInputStream = new FileInputStream(uploadedFile);
138                InputStream inputStream = IOUtils.buffer(fileInputStream);
139            )
140            {
141                Charset charset = _importCSVFileHelper.detectCharset(inputStream);
142                result.put("charset", charset);
143                String[] headers = _importCSVFileHelper.extractHeaders(inputStream, _getCSVPreference(escapingChar, separatingChar), charset);
144              
145                if (headers == null)
146                {
147                    result.put("error", "no-header");
148                    request.setAttribute(JSonReader.OBJECT_TO_READ, result);
149                    return result;
150                }
151                
152                List<Map<String, Object>> mapping = _importCSVFileHelper.getMapping(contentType, headers);
153                result.put("mapping", mapping);
154                
155                // Build a map to count how many attribute are there for each group
156                Map<String, Long> attributeCount = mapping.stream()
157                        .map(map -> map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH))
158                        .map(String.class::cast)
159                        .filter(StringUtils::isNotEmpty)
160                        .map(attributePath -> {
161                            String attributePrefix = "";
162                            int endIndex = attributePath.lastIndexOf("/");
163                            if (endIndex != -1)  
164                            {
165                                attributePrefix = attributePath.substring(0, endIndex);
166                            }
167                            return attributePrefix;
168                        })
169                        .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
170                
171                // If an attribute is the only for its group, it is the identifier
172                for (Map<String, Object> map : mapping)
173                {
174                    String attributePath = (String) map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH);
175                    String attributePrefix = StringUtils.EMPTY;
176                    int endIndex = attributePath.lastIndexOf("/");
177                    if (endIndex != -1)  
178                    {
179                        attributePrefix = attributePath.substring(0, endIndex);
180                    }
181    
182                    boolean parentIsContent;
183                    // If there is no prefix, it is an attribute of the content we want to import
184                    if (attributePrefix.equals(StringUtils.EMPTY))
185                    {
186                        parentIsContent = true;
187                    }
188                    // Otherwise, check the modelItem 
189                    else if (contentType.hasModelItem(attributePrefix))
190                    {
191                        ModelItem modelItem = contentType.getModelItem(attributePrefix);
192                        String modelTypeId = modelItem.getType().getId();
193                        parentIsContent = ModelItemTypeConstants.CONTENT_ELEMENT_TYPE_ID.equals(modelTypeId);
194                    }
195                    else
196                    {
197                        parentIsContent = false;
198                    }
199                    
200                    // If an attribute is the only one of its level, and is part of a content, consider it as the identifier
201                    if (attributeCount.getOrDefault(attributePrefix, 0L).equals(1L) && parentIsContent && StringUtils.isNotBlank(attributePath))
202                    {
203                        map.put(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID, true);
204                    }
205                }
206                
207                String newPath = _copyFile(uploadedFile.toPath());
208                result.put("path", newPath);
209                result.put("success", true);
210                result.put("workflows", _getWorkflows());
211                result.put("defaultWorkflow", contentType.getDefaultWorkflowName().orElse(null));
212                result.put("fileName", uploadedFile.getName());
213                
214                List<Map<String, Object>> synchronizeModes = ((Map<String, I18nizableText>) _synchronizeModeEnumerator.getTypedEntries())
215                        .entrySet()
216                        .stream()
217                        .map(entry -> Map.of("value", entry.getKey(), "label", entry.getValue()))
218                        .toList();
219                result.put("synchronizeModes", synchronizeModes);
220            }
221        }
222        
223        request.setAttribute(JSonReader.OBJECT_TO_READ, result);
224        return result;
225    }
226    
227    private String _copyFile(Path path) throws IOException
228    {
229        File contentIOStorageDir = FileUtils.getFile(RuntimeConfig.getInstance().getAmetysHome(), CONTENTIO_STORAGE_DIRECTORY);
230        
231        if (!contentIOStorageDir.exists())
232        {
233            if (!contentIOStorageDir.mkdirs())
234            {
235                throw new IOException("Unable to create monitoring directory: " + contentIOStorageDir);
236            }
237        }
238        String id = UUID.randomUUID().toString() + ".csv";
239        Path copy = Files.copy(path, Paths.get(contentIOStorageDir.getPath(), id), StandardCopyOption.REPLACE_EXISTING);
240        
241        return copy.toString();
242        
243    }
244    
245    private CsvPreference _getCSVPreference(String escapingChar, String separatingChar)
246    {
247        char separating = separatingChar.charAt(0);
248        char escaping = escapingChar.charAt(0);
249        
250        if (separating == escaping)
251        {
252            throw new IllegalArgumentException("Separating character can not be equals to escaping character");
253        }
254        return new CsvPreference.Builder(escaping, separating, "\r\n").build();
255    }
256    
257    /**
258     * getWorkflows
259     * @return map of workflows
260     */
261    private List<Map<String, Object>> _getWorkflows()
262    {
263        List<Map<String, Object>> workflows = new ArrayList<>();
264        String[] workflowNames = _workflowHelper.getWorkflowNames();
265        for (String workflowName : workflowNames)
266        {
267            Map<String, Object> workflowMap = new HashMap<>();
268            workflowMap.put("value", workflowName);
269            workflowMap.put("label", _workflowHelper.getWorkflowLabel(workflowName));
270            workflows.add(workflowMap);
271        }
272        return workflows;
273    }
274}