Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.File;
019import java.io.FileInputStream;
020import java.io.IOException;
021import java.io.InputStream;
022import java.nio.charset.Charset;
023import java.nio.file.Files;
024import java.nio.file.Path;
025import java.nio.file.Paths;
026import java.nio.file.StandardCopyOption;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.UUID;
032import java.util.function.Function;
033import java.util.stream.Collectors;
034
035import org.apache.avalon.framework.parameters.Parameters;
036import org.apache.avalon.framework.service.ServiceException;
037import org.apache.avalon.framework.service.ServiceManager;
038import org.apache.cocoon.acting.ServiceableAction;
039import org.apache.cocoon.environment.ObjectModelHelper;
040import org.apache.cocoon.environment.Redirector;
041import org.apache.cocoon.environment.Request;
042import org.apache.cocoon.environment.SourceResolver;
043import org.apache.cocoon.servlet.multipart.Part;
044import org.apache.cocoon.servlet.multipart.PartOnDisk;
045import org.apache.cocoon.servlet.multipart.RejectedPart;
046import org.apache.commons.io.FileUtils;
047import org.apache.commons.io.FilenameUtils;
048import org.apache.commons.io.IOUtils;
049import org.apache.commons.lang3.StringUtils;
050import org.supercsv.prefs.CsvPreference;
051
052import org.ametys.cms.contenttype.ContentType;
053import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
054import org.ametys.cms.data.type.ModelItemTypeConstants;
055import org.ametys.core.cocoon.JSonReader;
056import org.ametys.plugins.workflow.support.WorkflowProvider;
057import org.ametys.runtime.i18n.I18nizableText;
058import org.ametys.runtime.model.ModelItem;
059import org.ametys.runtime.servlet.RuntimeConfig;
060
061/**
062 * Import contents from an uploaded CSV file.
063 */
064public class ImportCSVFileAction extends ServiceableAction
065{
066
067    private static final String[] _ALLOWED_EXTENSIONS = new String[] {"txt", "csv"};
068
069    private static final String CONTENTIO_STORAGE_DIRECTORY = "contentio/temp";
070
071    private ContentTypeExtensionPoint _contentTypeEP;
072    
073    private WorkflowProvider _workflowProvider;
074    
075    private ImportCSVFileHelper _importCSVFileHelper;
076    
077    @Override
078    public void service(ServiceManager serviceManager) throws ServiceException
079    {
080        super.service(serviceManager);
081        _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
082        _workflowProvider = (WorkflowProvider) serviceManager.lookup(WorkflowProvider.ROLE);
083        _importCSVFileHelper = (ImportCSVFileHelper) manager.lookup(ImportCSVFileHelper.ROLE);
084    }
085    
086    @Override
087    public Map act(Redirector redirector, SourceResolver resolver, Map objectModel, String source, Parameters parameters) throws Exception
088    {
089        Request request = ObjectModelHelper.getRequest(objectModel);
090        
091        Map<String, Object> result = new HashMap<>();
092        String contentTypeId = (String) request.get("contentType");
093        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
094        Part part = (Part) request.get("file");
095        if (part instanceof RejectedPart || part == null)
096        {
097            result.put("success", false);
098            result.put("error", "rejected");
099        }
100        else
101        {
102            PartOnDisk uploadedFilePart = (PartOnDisk) part;
103            File uploadedFile = uploadedFilePart.getFile();
104            
105            String filename = uploadedFilePart.getFileName().toLowerCase();
106            
107            if (!FilenameUtils.isExtension(filename, _ALLOWED_EXTENSIONS))
108            {
109                result.put("error", "invalid-extension");
110                request.setAttribute(JSonReader.OBJECT_TO_READ, result);
111                return EMPTY_MAP;
112            }
113            
114            try (
115                InputStream fileInputStream = new FileInputStream(uploadedFile);
116                InputStream inputStream = IOUtils.buffer(fileInputStream);
117            )
118            {
119                Charset charset = _importCSVFileHelper.detectCharset(inputStream);
120                result.put("charset", charset);
121                String[] headers = _importCSVFileHelper.extractHeaders(inputStream, _getCSVPreference(request), charset);
122              
123                if (headers == null)
124                {
125                    result.put("error", "no-header");
126                    request.setAttribute(JSonReader.OBJECT_TO_READ, result);
127                    return EMPTY_MAP;
128                }
129                
130                List<Map<String, Object>> mapping = _importCSVFileHelper.getMapping(contentType, headers);
131                result.put("mapping", mapping);
132                
133                // Build a map to count how many attribute are there for each group
134                Map<String, Long> attributeCount = mapping.stream()
135                        .map(map -> map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH))
136                        .map(String.class::cast)
137                        .filter(StringUtils::isNotEmpty)
138                        .map(attributePath -> {
139                            String attributePrefix = "";
140                            int endIndex = attributePath.lastIndexOf("/");
141                            if (endIndex != -1)  
142                            {
143                                attributePrefix = attributePath.substring(0, endIndex);
144                            }
145                            return attributePrefix;
146                        })
147                        .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
148                
149                // If an attribute is the only for its group, it is the identifier
150                for (Map<String, Object> map : mapping)
151                {
152                    String attributePath = (String) map.get(ImportCSVFileHelper.MAPPING_COLUMN_ATTRIBUTE_PATH);
153                    String attributePrefix = StringUtils.EMPTY;
154                    int endIndex = attributePath.lastIndexOf("/");
155                    if (endIndex != -1)  
156                    {
157                        attributePrefix = attributePath.substring(0, endIndex);
158                    }
159    
160                    boolean parentIsContent;
161                    // If there is no prefix, it is an attribute of the content we want to import
162                    if (attributePrefix.equals(StringUtils.EMPTY))
163                    {
164                        parentIsContent = true;
165                    }
166                    // Otherwise, check the modelItem 
167                    else if (contentType.hasModelItem(attributePrefix))
168                    {
169                        ModelItem modelItem = contentType.getModelItem(attributePrefix);
170                        String modelTypeId = modelItem.getType().getId();
171                        parentIsContent = ModelItemTypeConstants.CONTENT_ELEMENT_TYPE_ID.equals(modelTypeId);
172                    }
173                    else
174                    {
175                        parentIsContent = false;
176                    }
177                    
178                    // If an attribute is the only one of its level, and is part of a content, consider it as the identifier
179                    if (attributeCount.getOrDefault(attributePrefix, 0L).equals(1L) && parentIsContent && StringUtils.isNotBlank(attributePath))
180                    {
181                        map.put(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID, true);
182                    }
183                }
184                
185                String newPath = _copyFile(uploadedFile.toPath());
186                result.put("path", newPath);
187                result.put("success", true);
188                result.put("workflows", _getWorkflows());
189                result.put("defaultWorkflow", contentType.getDefaultWorkflowName().orElse(null));
190                result.put("fileName", uploadedFile.getName());
191            }
192        }
193        
194        request.setAttribute(JSonReader.OBJECT_TO_READ, result);
195        return EMPTY_MAP;
196    }
197    
198    private String _copyFile(Path path) throws IOException
199    {
200        File contentIOStorageDir = FileUtils.getFile(RuntimeConfig.getInstance().getAmetysHome(), CONTENTIO_STORAGE_DIRECTORY);
201        
202        if (!contentIOStorageDir.exists())
203        {
204            if (!contentIOStorageDir.mkdirs())
205            {
206                throw new IOException("Unable to create monitoring directory: " + contentIOStorageDir);
207            }
208        }
209        String id = UUID.randomUUID().toString() + ".csv";
210        Path copy = Files.copy(path, Paths.get(contentIOStorageDir.getPath(), id), StandardCopyOption.REPLACE_EXISTING);
211        
212        return copy.toString();
213        
214    }
215    
216    private CsvPreference _getCSVPreference(Request request)
217    {
218        String escapingChar = (String) request.get("escaping-char");
219        String separatingChar = (String) request.get("separating-char");
220        char separating = separatingChar.charAt(0);
221        char escaping = escapingChar.charAt(0);
222        
223        if (separating == escaping)
224        {
225            throw new IllegalArgumentException("Separating character can not be equals to escaping character");
226        }
227        return new CsvPreference.Builder(escaping, separating, "\r\n").build();
228    }
229    
230    /**
231     * getWorkflows
232     * @return map of workflows
233     */
234    private List<Map<String, Object>> _getWorkflows()
235    {
236        List<Map<String, Object>> workflows = new ArrayList<>();
237        String[] workflowNames = _workflowProvider.getAmetysObjectWorkflow().getWorkflowNames();
238        for (String workflowName : workflowNames)
239        {
240            Map<String, Object> workflowMap = new HashMap<>();
241            workflowMap.put("value", workflowName);
242            workflowMap.put("label", new I18nizableText("application", "WORKFLOW_" + workflowName));
243            workflows.add(workflowMap);
244        }
245        return workflows;
246    }
247}