001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.BufferedReader;
019import java.io.File;
020import java.io.FileInputStream;
021import java.io.IOException;
022import java.io.InputStreamReader;
023import java.nio.charset.Charset;
024import java.nio.file.Files;
025import java.nio.file.Path;
026import java.nio.file.Paths;
027import java.nio.file.StandardCopyOption;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.HashMap;
031import java.util.List;
032import java.util.Map;
033import java.util.UUID;
034import java.util.stream.Collectors;
035
036import org.apache.avalon.framework.parameters.Parameters;
037import org.apache.avalon.framework.service.ServiceException;
038import org.apache.avalon.framework.service.ServiceManager;
039import org.apache.cocoon.acting.ServiceableAction;
040import org.apache.cocoon.environment.ObjectModelHelper;
041import org.apache.cocoon.environment.Redirector;
042import org.apache.cocoon.environment.Request;
043import org.apache.cocoon.environment.SourceResolver;
044import org.apache.cocoon.servlet.multipart.Part;
045import org.apache.cocoon.servlet.multipart.PartOnDisk;
046import org.apache.cocoon.servlet.multipart.RejectedPart;
047import org.apache.commons.io.FileUtils;
048import org.apache.commons.io.FilenameUtils;
049import org.apache.tika.detect.DefaultEncodingDetector;
050import org.apache.tika.io.TikaInputStream;
051import org.apache.tika.metadata.Metadata;
052import org.supercsv.io.CsvMapReader;
053import org.supercsv.io.ICsvMapReader;
054import org.supercsv.prefs.CsvPreference;
055
056import org.ametys.cms.contenttype.ContentType;
057import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
058import org.ametys.core.cocoon.JSonReader;
059import org.ametys.plugins.workflow.support.WorkflowProvider;
060import org.ametys.runtime.i18n.I18nizableText;
061import org.ametys.runtime.servlet.RuntimeConfig;
062
063/**
064 * Import contents from an uploaded CSV file.
065 */
066public class ImportCSVFileAction extends ServiceableAction
067{
068
069    private static final String[] _ALLOWED_EXTENSIONS = new String[] {"txt", "csv"};
070
071    private static final String CONTENTIO_STORAGE_DIRECTORY = "contentio/temp";
072
073    private ContentTypeExtensionPoint _contentTypeEP;
074    
075    private WorkflowProvider _workflowProvider;
076    
077    @Override
078    public void service(ServiceManager serviceManager) throws ServiceException
079    {
080        super.service(serviceManager);
081        _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
082        _workflowProvider = (WorkflowProvider) serviceManager.lookup(WorkflowProvider.ROLE);
083    }
084    
085    @Override
086    public Map act(Redirector redirector, SourceResolver resolver, Map objectModel, String source, Parameters parameters) throws Exception
087    {
088        Request request = ObjectModelHelper.getRequest(objectModel);
089        
090        Map<String, Object> result = new HashMap<>();
091        String contentTypeId = (String) request.get("contentType");
092        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
093        Part part = (Part) request.get("file");
094        if (part instanceof RejectedPart || part == null)
095        {
096            result.put("success", false);
097            result.put("error", "rejected");
098        }
099        else
100        {
101            PartOnDisk uploadedFilePart = (PartOnDisk) part;
102            File uploadedFile = uploadedFilePart.getFile();
103            
104            String filename = uploadedFilePart.getFileName().toLowerCase();
105            
106            if (!FilenameUtils.isExtension(filename, _ALLOWED_EXTENSIONS))
107            {
108                result.put("error", "invalid-extension");
109                request.setAttribute(JSonReader.OBJECT_TO_READ, result);
110                return EMPTY_MAP;
111            }
112            TikaInputStream stream = TikaInputStream.get(uploadedFile.toPath());
113            DefaultEncodingDetector defaultEncodingDetector = new DefaultEncodingDetector();
114            Charset charset = defaultEncodingDetector.detect(stream, new Metadata());
115            result.put("charset", charset);
116            String[] headers = _extractHeaders(uploadedFile, request, charset);
117          
118            if (headers != null)
119            {
120                List<Map<String, String>> mapping = Arrays.asList(headers)
121                        .stream()
122                        .map(header -> Map.of("header", header, "attributePath", contentType.hasModelItem(header) ? header.replace("/", ".") : ""))
123                        .collect(Collectors.toList());
124                result.put("mapping", mapping);
125            }
126            else
127            {
128                result.put("error", "no-header");
129                request.setAttribute(JSonReader.OBJECT_TO_READ, result);
130                return EMPTY_MAP;
131            }
132            String newPath = _copyFile(uploadedFile.toPath());
133            result.put("path", newPath);
134            result.put("success", true);
135            result.put("workflows", _getWorkflows());
136        }
137        
138        request.setAttribute(JSonReader.OBJECT_TO_READ, result);
139        return EMPTY_MAP;
140    }
141        
142    private String _copyFile(Path path) throws IOException
143    {
144        File contentIOStorageDir = FileUtils.getFile(RuntimeConfig.getInstance().getAmetysHome(), CONTENTIO_STORAGE_DIRECTORY);
145        
146        if (!contentIOStorageDir.exists())
147        {
148            if (!contentIOStorageDir.mkdirs())
149            {
150                throw new IOException("Unable to create monitoring directory: " + contentIOStorageDir);
151            }
152        }
153        String id = UUID.randomUUID().toString() + ".csv";
154        Path copy = Files.copy(path, Paths.get(contentIOStorageDir.getPath(), id), StandardCopyOption.REPLACE_EXISTING);
155        
156        return copy.toString();
157        
158    }
159    
160    private CsvPreference _getCSVPreference(Request request)
161    {
162        String escapingChar = (String) request.get("escaping-char");
163        String separatingChar = (String) request.get("separating-char");
164        char separating = separatingChar.charAt(0);
165        char escaping = escapingChar.charAt(0);
166        
167        if (separating == escaping)
168        {
169            throw new IllegalArgumentException("Separating character can not be equals to escaping character");
170        }
171        return new CsvPreference.Builder(escaping, separating, "\r\n").build();
172    }
173    
174    private String[] _extractHeaders(File uploadedFile, Request request, Charset charset) throws IOException
175    {
176        
177        try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(uploadedFile), charset)))
178        {
179            reader.mark(8192);
180            
181            CsvPreference preference = _getCSVPreference(request);
182            String headerLine = reader.readLine();
183            if (headerLine != null)
184            {
185                reader.reset();
186                try (ICsvMapReader mapReader = new CsvMapReader(reader, preference))
187                {
188                    String[] headers = mapReader.getHeader(true);
189                    return headers;
190                }
191            }
192            else
193            {
194                return null;
195            } 
196        }
197    }
198
199    /**
200     * getWorkflows
201     * @return map of workflows
202     */
203    private List<Map<String, Object>> _getWorkflows()
204    {
205        List<Map<String, Object>> workflows = new ArrayList<>();
206        String[] workflowNames = _workflowProvider.getAmetysObjectWorkflow().getWorkflowNames();
207        for (String workflowName : workflowNames)
208        {
209            Map<String, Object> workflowMap = new HashMap<>();
210            workflowMap.put("value", workflowName);
211            workflowMap.put("label", new I18nizableText("application", "WORKFLOW_" + workflowName));
212            workflows.add(workflowMap);
213        }
214        return workflows;
215    }
216}