Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.BufferedReader;
019import java.io.ByteArrayInputStream;
020import java.io.FileInputStream;
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.InputStreamReader;
024import java.nio.charset.Charset;
025import java.nio.file.Files;
026import java.nio.file.Paths;
027import java.util.ArrayList;
028import java.util.Arrays;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Objects;
033import java.util.Optional;
034import java.util.function.Predicate;
035import java.util.stream.Collectors;
036
037import org.apache.avalon.framework.activity.Initializable;
038import org.apache.avalon.framework.component.Component;
039import org.apache.avalon.framework.service.ServiceException;
040import org.apache.avalon.framework.service.ServiceManager;
041import org.apache.avalon.framework.service.Serviceable;
042import org.apache.commons.io.IOUtils;
043import org.apache.commons.lang3.StringUtils;
044import org.apache.tika.detect.DefaultEncodingDetector;
045import org.apache.tika.metadata.Metadata;
046import org.supercsv.io.CsvListReader;
047import org.supercsv.io.CsvMapReader;
048import org.supercsv.io.ICsvListReader;
049import org.supercsv.io.ICsvMapReader;
050import org.supercsv.prefs.CsvPreference;
051
052import org.ametys.cms.contenttype.ContentAttributeDefinition;
053import org.ametys.cms.contenttype.ContentType;
054import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
055import org.ametys.cms.contenttype.ContentTypesHelper;
056import org.ametys.core.ui.Callable;
057import org.ametys.core.user.CurrentUserProvider;
058import org.ametys.core.user.User;
059import org.ametys.core.user.UserIdentity;
060import org.ametys.core.user.UserManager;
061import org.ametys.core.util.I18nUtils;
062import org.ametys.core.util.mail.SendMailHelper;
063import org.ametys.core.util.mail.SendMailHelper.MailBuilder;
064import org.ametys.runtime.config.Config;
065import org.ametys.runtime.i18n.I18nizableText;
066import org.ametys.runtime.i18n.I18nizableTextParameter;
067import org.ametys.runtime.model.ElementDefinition;
068import org.ametys.runtime.model.ModelHelper;
069import org.ametys.runtime.model.ModelViewItemGroup;
070import org.ametys.runtime.model.View;
071import org.ametys.runtime.model.ViewElement;
072import org.ametys.runtime.model.ViewElementAccessor;
073import org.ametys.runtime.model.ViewItem;
074import org.ametys.runtime.model.exception.BadItemTypeException;
075import org.ametys.runtime.plugin.component.AbstractLogEnabled;
076
077import jakarta.mail.MessagingException;
078
079/**
080 * Import contents from an uploaded CSV file.
081 */
082public class ImportCSVFileHelper extends AbstractLogEnabled implements Component, Serviceable, Initializable
083{
084    /** Avalon Role */
085    public static final String ROLE = ImportCSVFileHelper.class.getName();
086
087    /** Result key containing updated content's ids */
088    public static final String RESULT_IMPORTED_COUNT = "importedCount";
089    /** Result key containing number of errors */
090    public static final String RESULT_NB_ERRORS = "nbErrors";
091    /** Result key containing number of warnings */
092    public static final String RESULT_NB_WARNINGS = "nbWarnings";
093    /** Result key containing a short an explanation for a general error */
094    public static final String RESULT_ERROR = "error";
095    /** Result key containing the filename */
096    public static final String RESULT_FILENAME = "fileName";
097    /** Column name of attribute path in CSV mapping */
098    public static final String MAPPING_COLUMN_ATTRIBUTE_PATH = "attributePath";
099    /** Column name of header in CSV mapping */
100    public static final String MAPPING_COLUMN_HEADER = "header";
101    /** Column name to identify if column is an ID in CSV mapping */
102    public static final String MAPPING_COLUMN_IS_ID = "isId";
103    /** Key in nested mapping to define the ID columns */
104    public static final String NESTED_MAPPING_ID = "id";
105    /** Key in nested mapping to define the values columns */
106    public static final String NESTED_MAPPING_VALUES = "values";
107    
108    private CurrentUserProvider _currentUserProvider;
109    private UserManager _userManager;
110    
111    private ContentTypeExtensionPoint _contentTypeEP;
112    private ContentTypesHelper _contentTypesHelper;
113    
114    private I18nUtils _i18nUtils;
115    private String _sysadminMail;
116    
117    private CSVImporter _csvImporter;
118
119    @Override
120    public void service(ServiceManager serviceManager) throws ServiceException
121    {
122        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
123        _userManager = (UserManager) serviceManager.lookup(UserManager.ROLE);
124        
125        _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
126        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
127        
128        _i18nUtils = (I18nUtils) serviceManager.lookup(I18nUtils.ROLE);
129        
130        _csvImporter = (CSVImporter) serviceManager.lookup(CSVImporter.ROLE);
131    }
132    
133    public void initialize() throws Exception
134    {
135        _sysadminMail = Config.getInstance().getValue("smtp.mail.sysadminto");
136    }
137
138    /**
139     * Retrieves the values for import CSV parameters
140     * @param contentTypeId the configured content type identifier
141     * @return the values for import CSV parameters
142     */
143    @Callable
144    public Map<String, Object> getImportCSVParametersValues(String contentTypeId)
145    {
146        Map<String, Object> values = new HashMap<>();
147        
148        // Content types
149        List<String> contentTypeIds = StringUtils.isEmpty(contentTypeId) ? List.of() : List.of(contentTypeId);
150        Map<String, Object> filteredContentTypes = _contentTypesHelper.getContentTypesList(contentTypeIds, true, true, true, false, false);
151        values.put("availableContentTypes", filteredContentTypes.get("contentTypes"));
152        
153        // Recipient - get current user email
154        String currentUserEmail = null;
155        UserIdentity currentUser = _currentUserProvider.getUser();
156        
157        String login = currentUser.getLogin();
158        if (StringUtils.isNotBlank(login))
159        {
160            String userPopulationId = currentUser.getPopulationId();
161            User user = _userManager.getUser(userPopulationId, login);
162            currentUserEmail = user.getEmail();
163        }
164        values.put("defaultRecipient", currentUserEmail);
165        
166        return values;
167    }
168    
169    /**
170     * Gets the configuration for creating/editing a collection of synchronizable contents.
171     * @param contentTypeId Content type id
172     * @param formValues map of form values
173     * @param mappingValues list of header and content attribute mapping
174     * @return A map containing information about what is needed to create/edit a collection of synchronizable contents
175     * @throws IOException IOException while reading CSV
176     */
177    @SuppressWarnings("unchecked")
178    @Callable
179    public Map<String, Object> validateConfiguration(String contentTypeId, Map formValues, List<Map<String, Object>> mappingValues) throws IOException
180    {
181        Map<String, Object> result = new HashMap<>();
182
183        List<Map<String, Object>> filteredMappingValues = _filterMapping(mappingValues);
184        
185        List<String> contentAttributes = filteredMappingValues.stream()
186                .map(column -> column.get(MAPPING_COLUMN_ATTRIBUTE_PATH))
187                .map(String.class::cast)
188                .collect(Collectors.toList());
189
190        Map<String, Object> mapping = new HashMap<>();
191        filteredMappingValues.forEach(column -> generateNestedMapping(mapping, column));
192        
193        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
194        result.put("contentTypeName", contentType.getLabel());
195        View view = null;
196        for (String itemPath : contentAttributes)
197        {
198            if (!ModelHelper.hasModelItem(itemPath, List.of(contentType)))
199            {
200                result.put("error", "bad-mapping");
201                result.put("details", List.of(itemPath, contentTypeId));
202                return result;
203            }
204        }
205        
206        try 
207        {
208            view = View.of(contentType, contentAttributes.toArray(new String[contentAttributes.size()]));
209        }
210        catch (IllegalArgumentException | BadItemTypeException e)
211        {
212            getLogger().error("Error while creating view", e);
213            result.put("error", "cant-create-view");
214            result.put("details", contentTypeId);
215            return result;
216        }
217        
218        if (!mapping.containsKey(NESTED_MAPPING_ID))
219        {  
220            result.put("error", "missing-id");
221            return result;
222        } 
223        
224        for (ViewItem viewItem : view.getViewItems())
225        {
226            if (!_checkViewItemContainer(viewItem, (Map<String, Object>) mapping.get(NESTED_MAPPING_VALUES), result))
227            {
228                return result;
229            }
230        }
231        result.put("success", true);
232        return result;
233    }
234    /**
235     * Gets the configuration for creating/editing a collection of synchronizable contents.
236     * @param config get all CSV related parameters: path, separating/escaping char, charset and contentType
237     * @param formValues map of form values
238     * @param mappingValues list of header and content attribute mapping
239     * @return A map containing information about what is needed to create/edit a collection of synchronizable contents
240     * @throws IOException IOException while reading CSV
241     */
242    @Callable
243    public Map<String, Object> importContents(Map<String, Object> config, Map<String, Object> formValues, List<Map<String, Object>> mappingValues) throws IOException
244    {
245        Map<String, Object> result = new HashMap<>();
246        result.put(RESULT_FILENAME, config.get("fileName"));
247        
248        String path = (String) config.get("path");
249        String contentTypeId = (String) config.get("contentType");
250        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
251        
252        // CSV configuration
253        CsvPreference csvPreference = new CsvPreference.Builder(
254                config.getOrDefault("escaping-char", '"').toString().charAt(0),
255                config.getOrDefault("separating-char", ';').toString().charAt(0),
256                "\r\n"
257            )
258            .build();
259
260        String encoding = (String) config.get("charset");
261        Charset charset = Charset.forName(encoding);
262        
263        String language = (String) formValues.get("language");
264        int createAction = Integer.valueOf((String) formValues.getOrDefault("createAction", "1"));
265        int editAction = Integer.valueOf((String) formValues.getOrDefault("editAction", "2"));
266        String workflowName = (String) formValues.get("workflow");
267        
268        Optional<String> recipient = Optional.ofNullable(config.get("recipient"))
269                .filter(String.class::isInstance)
270                .map(String.class::cast)
271                .filter(StringUtils::isNotBlank);
272        
273        List<Map<String, Object>> filteredMappingValues = _filterMapping(mappingValues);
274        
275        try (
276            InputStream fileInputStream = new FileInputStream(path);
277            InputStream inputStream = IOUtils.buffer(fileInputStream);
278        )
279        {
280            result.putAll(_importContents(inputStream, csvPreference, charset, contentType, workflowName, language, createAction, editAction, filteredMappingValues));
281            
282            // If an exception occured and the recipient is empty, transfer the error mail to the configured sysadmin
283            if ("error".equals(result.get(RESULT_ERROR)) && recipient.isEmpty())
284            {
285                recipient = Optional.ofNullable(_sysadminMail)
286                        .filter(StringUtils::isNotBlank);
287            }
288            
289            if (recipient.isPresent())
290            {
291                _sendMail(result, recipient.get());
292            }
293        }
294        finally
295        {
296            deleteFile(path);
297        }
298
299        return result;
300    }
301    
302    /**
303     * Import contents from path, content type and language.
304     * CSV Excel north Europe preferences are used, charset is detected on the file, workflow is detected on content type, default creation (1) and
305     * edition (2) actions are used, values are automatically mapped from the header. Headers should contains a star (*) to detect identifier columns.
306     * @param inputStream The (buffered) input stream to the CSV resource
307     * @param contentType The content type
308     * @param language The language
309     * @return the result of the CSV import
310     * @throws IOException if an error occurs
311     */
312    public Map<String, Object> importContents(InputStream inputStream, ContentType contentType, String language) throws IOException
313    {
314        CsvPreference csvPreference = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE;
315        
316        // Detect the charset of the current file
317        Charset charset = detectCharset(inputStream);
318        
319        // Copy the first 8192 bytes of the initial input stream to another input stream (to get headers)
320        inputStream.mark(8192);
321        try (InputStream headerStream = new ByteArrayInputStream(inputStream.readNBytes(8192)))
322        {
323            inputStream.reset();
324            
325            // Get headers
326            String[] headers = extractHeaders(headerStream, csvPreference, charset);
327            
328            // Get the mapping of the values
329            List<Map<String, Object>> mappingValues = getMapping(contentType, headers);
330            
331            // Get the default workflow associated to the content type
332            String workflowName = contentType.getDefaultWorkflowName().orElseThrow(() -> new IllegalArgumentException("The workflow can't be defined."));
333            
334            // Import contents (last use of the input stream)
335            return _importContents(inputStream, csvPreference, charset, contentType, workflowName, language, 1, 2, mappingValues);
336        }
337    }
338    
339    private Map<String, Object> _importContents(InputStream inputStream, CsvPreference csvPreference, Charset charset, ContentType contentType, String workflowName, String language, int createAction, int editAction, List<Map<String, Object>> mappingValues)
340    {
341        Map<String, Object> result = new HashMap<>();
342        
343        List<String> contentAttributes = mappingValues.stream()
344            .map(column -> column.get(MAPPING_COLUMN_ATTRIBUTE_PATH))
345            .map(String.class::cast)
346            .collect(Collectors.toList());
347
348        Map<String, Object> mapping = new HashMap<>();
349        mappingValues.forEach(column -> generateNestedMapping(mapping, column));
350        
351        View view = View.of(contentType, contentAttributes.toArray(new String[contentAttributes.size()]));
352
353        try (
354            InputStreamReader inputStreamReader = new InputStreamReader(inputStream, charset);
355            BufferedReader reader = new BufferedReader(inputStreamReader);
356            ICsvListReader csvMapReadernew = new CsvListReader(reader, csvPreference)
357        )
358        {
359            Map<String, Object> importResult = _csvImporter.importContentsFromCSV(mapping, view, contentType, csvMapReadernew, createAction, editAction, workflowName, language);
360
361            @SuppressWarnings("unchecked")
362            List<String> contentIds = (List<String>) importResult.get(CSVImporter.RESULT_CONTENT_IDS);
363            if (contentIds.size() > 0)
364            {
365                result.put(RESULT_IMPORTED_COUNT, contentIds.size());
366                result.put(RESULT_NB_ERRORS, importResult.get(CSVImporter.RESULT_NB_ERRORS));
367                result.put(RESULT_NB_WARNINGS, importResult.get(CSVImporter.RESULT_NB_WARNINGS));
368            }
369            else
370            {
371                result.put(RESULT_ERROR, "no-import");
372            }
373        }
374        catch (Exception e)
375        {  
376            getLogger().error("Error while importing contents", e);
377            result.put(RESULT_ERROR, "error");
378        }
379        
380        return result;
381    }
382    
383    private boolean _checkViewItemContainer(ViewItem viewItem, Map<String, Object> mapping, Map<String, Object> result)
384    {
385        if (viewItem instanceof ViewElement)
386        {
387            ViewElement viewElement = (ViewElement) viewItem;
388            ElementDefinition elementDefinition = viewElement.getDefinition();
389            if (elementDefinition instanceof ContentAttributeDefinition)
390            {
391                ViewElementAccessor viewElementAccessor = (ViewElementAccessor) viewElement;
392                ContentAttributeDefinition contentAttributeDefinition = (ContentAttributeDefinition) elementDefinition;
393                return _checkViewItemContentContainer(viewElementAccessor, mapping, result, contentAttributeDefinition);
394            }
395            else
396            {
397                return true;
398            }
399        }
400        else if (viewItem instanceof ModelViewItemGroup)
401        {
402            return _checkViewItemGroupContainer(viewItem, mapping, result);
403        }
404        else
405        {
406            result.put("error", "unknown-type");
407            result.put("details", viewItem.getName());
408            return false;
409        }
410    }
411
412    @SuppressWarnings("unchecked")
413    private boolean _checkViewItemContentContainer(ViewElementAccessor viewElementAccessor, Map<String, Object> mapping, Map<String, Object> result, ContentAttributeDefinition contentAttributeDefinition)
414    {
415
416        boolean success = true;
417        String contentTypeId = contentAttributeDefinition.getContentTypeId();
418        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
419        if (mapping.get(viewElementAccessor.getName()) instanceof String)
420        {
421            result.put("error", "string-as-container");
422            result.put("details", mapping.get(viewElementAccessor.getName()));
423            return false;
424        }
425        Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewElementAccessor.getName());
426        if (!nestedMap.containsKey(NESTED_MAPPING_ID) || ((List<String>) nestedMap.get(NESTED_MAPPING_ID)).isEmpty())
427        {
428            result.put("error", "missing-id-content");
429            result.put("details", List.of(contentType.getLabel(), contentAttributeDefinition.getName()));
430            return false;
431        }
432        Map<String, Object> nestedMapValues = (Map<String, Object>) (nestedMap.get(NESTED_MAPPING_VALUES));
433        for (ViewItem nestedViewItem : viewElementAccessor.getViewItems())
434        {
435            success = success && _checkViewItemContainer(nestedViewItem, nestedMapValues, result);
436        }
437        return success;
438    }
439    
440    @SuppressWarnings("unchecked")
441    private boolean _checkViewItemGroupContainer(ViewItem viewItem, Map<String, Object> mapping, Map<String, Object> result)
442    {
443        boolean success = true;
444        ModelViewItemGroup modelViewItemGroup = (ModelViewItemGroup) viewItem;
445        List<ViewItem> elementDefinition = modelViewItemGroup.getViewItems();
446        Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewItem.getName());
447        
448        Map<String, Object> nestedMapValues = (Map<String, Object>) (nestedMap.get(NESTED_MAPPING_VALUES));
449                    
450        for (ViewItem nestedViewItem : elementDefinition)
451        {
452            success = success && _checkViewItemContainer(nestedViewItem, nestedMapValues, result);
453        }
454        return success;
455    }
456
457    private static Map<String, Object> generateNestedMapping(Map<String, Object> mapping, Map<String, Object> column) 
458    {
459        return generateNestedMapping(mapping, (String) column.get(MAPPING_COLUMN_ATTRIBUTE_PATH), (String) column.get(ImportCSVFileHelper.MAPPING_COLUMN_HEADER), (boolean) column.get(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID)); 
460    }
461
462    @SuppressWarnings("unchecked")
463    private static Map<String, Object> generateNestedMapping(Map<String, Object> mapping, String attributePath, String column, boolean isId) 
464    {
465        Map<String, Object> values = (Map<String, Object>) mapping.computeIfAbsent(NESTED_MAPPING_VALUES, __ -> new HashMap<>());
466
467        int separatorIndex = attributePath.indexOf('/');
468        if (separatorIndex == -1)
469        {
470            if (isId)
471            {
472                List<String> ids = (List<String>) mapping.computeIfAbsent(NESTED_MAPPING_ID, __ -> new ArrayList<>());
473                ids.add(attributePath);
474            }
475            values.put(attributePath, column);
476        }
477        else
478        {
479            Map<String, Object> subValuesMapping = (Map<String, Object>) values.computeIfAbsent(attributePath.substring(0, separatorIndex), __ -> new HashMap<>());
480            generateNestedMapping(subValuesMapping, attributePath.substring(separatorIndex + 1), column, isId);
481        }
482        
483        mapping.put(NESTED_MAPPING_VALUES, values);
484        return mapping;
485    }
486
487    private void _sendMail(Map<String, Object> importResult, String recipient)
488    {
489        I18nizableText subject = _getMailSubject(importResult);
490        I18nizableText body = _getMailBody(importResult);
491        
492        try
493        {
494            MailBuilder mailBuilder = SendMailHelper.newMail()
495                                                    .withSubject(_i18nUtils.translate(subject))
496                                                    .withRecipient(recipient)
497                                                    .withTextBody(_i18nUtils.translate(body));
498            
499            mailBuilder.sendMail();
500        }
501        catch (MessagingException | IOException e)
502        {
503            if (getLogger().isWarnEnabled())
504            {
505                getLogger().warn("Unable to send the e-mail '" + subject  + "' to '" + recipient + "'", e);
506            }
507        }
508        catch (Exception e)
509        {
510            getLogger().error("An unknown error has occured while sending the mail.", e);
511        }
512    }
513    
514    private I18nizableText _getMailSubject(Map<String, Object> importResult)
515    {
516        Map<String, I18nizableTextParameter> i18nParams = Map.of("fileName", new I18nizableText((String) importResult.get(RESULT_FILENAME)));
517        if (importResult.containsKey(ImportCSVFileHelper.RESULT_ERROR))
518        {
519            return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_ERROR_SUBJECT", i18nParams);
520        }
521        else
522        {
523            return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_SUCCESS_SUBJECT", i18nParams);
524        }
525    }
526    
527    private I18nizableText _getMailBody(Map<String, Object> importResult)
528    {
529        Map<String, I18nizableTextParameter> i18nParams = new HashMap<>();
530        i18nParams.put("fileName", new I18nizableText((String) importResult.get(RESULT_FILENAME)));
531        
532        if (importResult.containsKey(ImportCSVFileHelper.RESULT_ERROR))
533        {
534            if ("no-import".equals(importResult.get(ImportCSVFileHelper.RESULT_ERROR)))
535            {
536                return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_ERROR_BODY_NO_IMPORT", i18nParams);
537            }
538            else
539            {
540                return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_ERROR_BODY_TEXT", i18nParams);
541            }
542        }
543        else
544        {
545            int nbErrors = (int) importResult.get(RESULT_NB_ERRORS);
546            int nbWarnings = (int) importResult.get(RESULT_NB_WARNINGS);
547            i18nParams.put("importedCount", new I18nizableText(String.valueOf(importResult.get(RESULT_IMPORTED_COUNT))));
548            i18nParams.put("nbErrors", new I18nizableText(String.valueOf(nbErrors)));
549            i18nParams.put("nbWarnings", new I18nizableText(String.valueOf(nbWarnings)));
550            
551            if (nbErrors == 0 && nbWarnings == 0)
552            {
553                return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_SUCCESS_BODY", i18nParams);
554            }
555            else if (nbWarnings == 0)
556            {
557                return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_PARTIAL_SUCCESS_BODY_ERRORS", i18nParams);
558            }
559            else if (nbErrors == 0)
560            {
561                return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_PARTIAL_SUCCESS_BODY_WARNINGS", i18nParams);
562            }
563            else
564            {
565                return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_PARTIAL_SUCCESS_BODY_ERRORS_AND_WARNINGS", i18nParams);
566            }
567        }
568    }
569
570    /**
571     * Delete the file related to the given path
572     * @param path path of the file
573     * @throws IOException if an error occurs while deleting the file
574     */
575    @Callable
576    public void deleteFile(String path) throws IOException 
577    {
578        Files.delete(Paths.get(path));
579    }
580    
581    /**
582     * Detect the charset of a given resource.
583     * @param inputStream The input stream to the resource, need to support {@link InputStream#mark(int)} and {@link InputStream#reset()}.
584     * @return the charset
585     * @throws IOException if an error occurs
586     */
587    public Charset detectCharset(InputStream inputStream) throws IOException
588    {
589        assert inputStream.markSupported();
590        DefaultEncodingDetector defaultEncodingDetector = new DefaultEncodingDetector();
591        return defaultEncodingDetector.detect(inputStream, new Metadata());
592    }
593
594    /**
595     * Extract headers from the CSV resource.
596     * @param inputStream The input stream to the resource
597     * @param csvPreference The CSV preference (separators)
598     * @param charset The charset of the resource
599     * @return the list of headers in file
600     * @throws IOException if an error occurs
601     */
602    public String[] extractHeaders(InputStream inputStream, CsvPreference csvPreference, Charset charset) throws IOException
603    {
604        try (
605            InputStreamReader inputStreamReader = new InputStreamReader(inputStream, charset);
606            BufferedReader reader = new BufferedReader(inputStreamReader);
607            ICsvMapReader mapReader = new CsvMapReader(reader, csvPreference);
608        )
609        {
610            String[] headers = mapReader.getHeader(true);
611            return headers;
612        }
613    }
614
615    /**
616     * Get the mapping from the headers and corresponding to the content type.
617     * @param contentType The content type
618     * @param headers The CSV headers
619     * @return the list of columns descriptions. An item of the list is one column, each column is defined by a header "header" (text defined into the CSV header), an
620     * attribute path "attributePath" (real path extracted from the header, empty if the model item does not exist) and an identifier flag "isId" (<code>true</code>
621     * if the header ends with a star "*").
622     */
623    public List<Map<String, Object>> getMapping(ContentType contentType, String[] headers)
624    {
625        return Arrays.asList(headers)
626            .stream()
627            .filter(StringUtils::isNotEmpty)
628            .map(header -> _transformHeaderToMap(contentType, header))
629            .collect(Collectors.toList());
630    }
631    
632    private Map<String, Object> _transformHeaderToMap(ContentType contentType, String header)
633    {
634        Map<String, Object> headerMap = new HashMap<>();
635        headerMap.put(ImportCSVFileHelper.MAPPING_COLUMN_HEADER, header);
636        headerMap.put(
637            MAPPING_COLUMN_ATTRIBUTE_PATH, 
638            Optional.of(header)
639                .map(s -> s.replace("*", ""))
640                .map(s -> s.replace(".", "/"))
641                .map(String::trim)
642                .filter(contentType::hasModelItem)
643                .orElse(StringUtils.EMPTY)
644        );
645        headerMap.put(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID, header.endsWith("*"));
646        return headerMap;
647    }
648    
649    private List<Map<String, Object>> _filterMapping(List<Map<String, Object>> mappingValues)
650    {
651        // Suppress empty attributePath
652        // Transform attributePath with . to attributePath with / (Autocompletion in IHM has . as separator, but the API need /)
653        return mappingValues.stream()
654            .map(column -> 
655                {
656                    Optional<String> attributePath = Optional.of(MAPPING_COLUMN_ATTRIBUTE_PATH)
657                        .map(column::get)
658                        .map(String.class::cast)
659                        .filter(StringUtils::isNotEmpty)
660                        .map(s -> s.replace(".", "/"));
661                    
662                    if (attributePath.isEmpty())
663                    {
664                        return null;
665                    }
666                    
667                    // Replace original column
668                    Map<String, Object> columnCopy = new HashMap<>(column);
669                    columnCopy.put(MAPPING_COLUMN_ATTRIBUTE_PATH, attributePath.get());
670                    return columnCopy;
671                }
672            )
673            .filter(Predicate.not(Objects::isNull))
674            .collect(Collectors.toList());
675    }
676}