Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.IOException;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.Collection;
022import java.util.HashMap;
023import java.util.List;
024import java.util.Map;
025import java.util.Map.Entry;
026import java.util.Objects;
027import java.util.Optional;
028import java.util.Set;
029import java.util.function.Function;
030import java.util.stream.Collectors;
031import java.util.stream.Stream;
032
033import org.apache.avalon.framework.component.Component;
034import org.apache.avalon.framework.service.ServiceException;
035import org.apache.avalon.framework.service.ServiceManager;
036import org.apache.avalon.framework.service.Serviceable;
037import org.apache.commons.lang3.LocaleUtils;
038import org.apache.commons.lang3.StringUtils;
039import org.supercsv.io.ICsvListReader;
040import org.supercsv.util.Util;
041
042import org.ametys.cms.ObservationConstants;
043import org.ametys.cms.contenttype.ContentAttributeDefinition;
044import org.ametys.cms.contenttype.ContentType;
045import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
046import org.ametys.cms.data.ContentValue;
047import org.ametys.cms.indexing.solr.SolrIndexHelper;
048import org.ametys.cms.repository.Content;
049import org.ametys.cms.repository.ContentQueryHelper;
050import org.ametys.cms.repository.ModifiableDefaultContent;
051import org.ametys.cms.repository.ModifiableWorkflowAwareContent;
052import org.ametys.cms.workflow.AbstractContentWorkflowComponent;
053import org.ametys.cms.workflow.ContentWorkflowHelper;
054import org.ametys.cms.workflow.CreateContentFunction;
055import org.ametys.core.util.I18nUtils;
056import org.ametys.plugins.contentio.csv.SynchronizeModeEnumerator.ImportMode;
057import org.ametys.plugins.contentio.in.ContentImportException;
058import org.ametys.plugins.repository.AmetysObjectIterable;
059import org.ametys.plugins.repository.AmetysObjectResolver;
060import org.ametys.plugins.repository.data.holder.group.Repeater;
061import org.ametys.plugins.repository.data.holder.group.RepeaterEntry;
062import org.ametys.plugins.repository.data.holder.values.SynchronizableRepeater;
063import org.ametys.plugins.repository.data.holder.values.SynchronizableValue;
064import org.ametys.plugins.repository.data.holder.values.SynchronizableValue.Mode;
065import org.ametys.plugins.repository.data.type.ModelItemTypeConstants;
066import org.ametys.plugins.repository.metadata.MultilingualString;
067import org.ametys.plugins.repository.metadata.MultilingualStringHelper;
068import org.ametys.plugins.repository.query.expression.AndExpression;
069import org.ametys.plugins.repository.query.expression.Expression;
070import org.ametys.plugins.repository.query.expression.Expression.Operator;
071import org.ametys.plugins.repository.query.expression.ExpressionContext;
072import org.ametys.plugins.repository.query.expression.MultilingualStringExpression;
073import org.ametys.plugins.repository.query.expression.StringExpression;
074import org.ametys.runtime.model.ElementDefinition;
075import org.ametys.runtime.model.ModelItem;
076import org.ametys.runtime.model.ModelViewItemGroup;
077import org.ametys.runtime.model.View;
078import org.ametys.runtime.model.ViewElement;
079import org.ametys.runtime.model.ViewElementAccessor;
080import org.ametys.runtime.model.ViewItem;
081import org.ametys.runtime.model.type.ElementType;
082import org.ametys.runtime.plugin.component.AbstractLogEnabled;
083
084import com.opensymphony.workflow.WorkflowException;
085
086/**
087 * Import contents from an uploaded CSV file.
088 */
089public class CSVImporter extends AbstractLogEnabled implements Component, Serviceable
090{
091    
092    /** Avalon Role */
093    public static final String ROLE = CSVImporter.class.getName();
094
095    /** Result key containing updated content's ids */
096    public static final String RESULT_CONTENT_IDS = "contentIds";
097    /** Result key containing number of errors */
098    public static final String RESULT_NB_ERRORS = "nbErrors";
099    /** Result key containing number of warnings */
100    public static final String RESULT_NB_WARNINGS = "nbWarnings";
101
102    private ContentWorkflowHelper _contentWorkflowHelper;
103
104    private ContentTypeExtensionPoint _contentTypeEP;
105    
106    private AmetysObjectResolver _resolver;
107
108    private I18nUtils _i18nUtils;
109
110    private SolrIndexHelper _solrIndexHelper;
111    
112    public void service(ServiceManager smanager) throws ServiceException
113    {
114        _resolver = (AmetysObjectResolver) smanager.lookup(AmetysObjectResolver.ROLE);
115        _contentTypeEP = (ContentTypeExtensionPoint) smanager.lookup(ContentTypeExtensionPoint.ROLE);
116        _contentWorkflowHelper = (ContentWorkflowHelper) smanager.lookup(ContentWorkflowHelper.ROLE);
117        _i18nUtils = (I18nUtils) smanager.lookup(I18nUtils.ROLE);
118        _solrIndexHelper = (SolrIndexHelper) smanager.lookup(SolrIndexHelper.ROLE);
119    }
120
121    /**
122     * Extract contents from CSV file
123     * @param mapping mapping of content attributes and CSV file header
124     * @param view View of importing content
125     * @param contentType content type to import
126     * @param listReader mapReader to parse CSV file
127     * @param createAction creation action id
128     * @param editAction edition action id
129     * @param workflowName workflow name
130     * @param language language of created content.
131     * @param importMode The import mode
132     * @param parentContent Optional parent content
133     * @param additionalTransientVars The optional additional transient vars to add to the transient vars
134     * @return list of created contents
135     * @throws IOException IOException while reading CSV
136     */
137    public Map<String, Object> importContentsFromCSV(Map<String, Object> mapping, View view, ContentType contentType, ICsvListReader listReader, int createAction, int editAction, String workflowName, String language, ImportMode importMode, Optional< ? extends Content> parentContent, Map<String, Object> additionalTransientVars) throws IOException
138    {
139        try
140        {
141            _solrIndexHelper.pauseSolrCommitForEvents(_getIndexationEvents());
142            
143            List<String> contentIds = new ArrayList<>();
144            String[] columns = listReader.getHeader(true);
145            int nbErrors = 0;
146            int nbWarnings = 0;
147            List<String> row = null;
148            
149            while ((row = listReader.read()) != null)
150            {
151                try
152                {
153                    if (listReader.length() != columns.length)
154                    {
155                        getLogger().error("[{}] Import from CSV file: content skipped because of invalid row: {}", contentType.getId(), row);
156                        nbErrors++;
157                        continue;
158                    }
159    
160                    Map<String, String> rowMap = new HashMap<>();
161                    Util.filterListToMap(rowMap, columns, row);
162                    List<ViewItem> errors = new ArrayList<>();
163                    SynchronizeResult synchronizeResult = processContent(view, rowMap, contentType, mapping, createAction, editAction, workflowName, language, errors, importMode, parentContent, additionalTransientVars);
164                    Optional<ModifiableWorkflowAwareContent> content = synchronizeResult.content();
165                    
166                    // If we are in CREATE_ONLY mode but content was not created, do not add it to the contents list
167                    if (content.isPresent() && (importMode != ImportMode.CREATE_ONLY || synchronizeResult.isCreated()))
168                    {
169                        contentIds.add(content.get().getId());
170                    }
171                    
172                    if (!errors.isEmpty())
173                    {
174                        nbWarnings++;
175                    }
176                }
177                catch (Exception e)
178                {
179                    nbErrors++;
180                    getLogger().error("[{}] Import from CSV file: error importing the content on line {}", contentType.getId(), listReader.getLineNumber(), e);
181                }
182            }
183            
184            Map<String, Object> results = new HashMap<>();
185            results.put(RESULT_CONTENT_IDS, contentIds);
186            results.put(RESULT_NB_ERRORS, nbErrors);
187            results.put(RESULT_NB_WARNINGS, nbWarnings);
188            return results;
189        }
190        finally
191        {
192            _solrIndexHelper.restartSolrCommitForEvents(_getIndexationEvents());
193        }
194    }
195    
196    private String[] _getIndexationEvents()
197    {
198        return new String[] {
199            ObservationConstants.EVENT_CONTENT_ADDED,
200            ObservationConstants.EVENT_CONTENT_MODIFIED,
201            ObservationConstants.EVENT_CONTENT_WORKFLOW_CHANGED,
202            ObservationConstants.EVENT_CONTENT_VALIDATED
203        };
204    }
205    
206    /**
207     * Process content for CSV import or synchronization
208     * @param view The view
209     * @param row The row containing the values
210     * @param contentType The content type
211     * @param mapping The mapping
212     * @param createAction The create action
213     * @param editAction The edit action
214     * @param workflowName The wokflow name
215     * @param language The language
216     * @param errors The errors
217     * @param importMode The import mode
218     * @param parentContent The parent content
219     * @param additionalTransientVars The additional transient vars
220     * @return The synchronize result
221     * @throws Exception If an error occurs
222     */
223    protected SynchronizeResult processContent(View view, Map<String, String> row, ContentType contentType, Map<String, Object> mapping, int createAction, int editAction, String workflowName, String language, List<ViewItem> errors, ImportMode importMode, Optional< ? extends Content> parentContent, Map<String, Object> additionalTransientVars) throws Exception
224    {
225        @SuppressWarnings("unchecked")
226        List<String> attributeIdNames = (List<String>) mapping.get(ImportCSVFileHelper.NESTED_MAPPING_ID);
227        @SuppressWarnings("unchecked")
228        Map<String, Object> mappingValues = (Map<String, Object>) mapping.get(ImportCSVFileHelper.NESTED_MAPPING_VALUES);
229
230        SynchronizeResult synchronizeResult = _synchronizeContent(row, contentType, view, attributeIdNames, mappingValues, createAction, editAction, workflowName, language, errors, importMode, parentContent, additionalTransientVars);
231        return synchronizeResult;
232    }
233
234    private void _editContent(int editAction, View view, Map<String, Object> values, ModifiableWorkflowAwareContent content) throws WorkflowException
235    {
236        Collection<ModelItem> viewItemsDiff = content.getDifferences(view, values);
237        if (!viewItemsDiff.isEmpty())
238        {
239            _contentWorkflowHelper.editContent(content, values, editAction, View.of(viewItemsDiff.toArray(ModelItem[]::new)));
240        }
241    }
242    
243    private Object _getValue(Optional<? extends Content> parentContent, ViewItem viewItem, Map<String, Object> mapping, Map<String, String> row, int createAction, int editAction, String language, List<ViewItem> errors, String prefix, ImportMode importMode, Map<String, Object> additionalTransientVars) throws Exception
244    {
245        if (viewItem instanceof ViewElement viewElement)
246        {
247            ElementDefinition elementDefinition = viewElement.getDefinition();
248            if (elementDefinition instanceof ContentAttributeDefinition contentAttributeDefinition && viewElement instanceof ViewElementAccessor viewElementAccessor)
249            {
250                return _getContentAttributeDefinitionValues(parentContent, mapping, row, createAction, editAction, language, viewElementAccessor, contentAttributeDefinition, errors, importMode, additionalTransientVars);
251            }
252            else
253            {
254                return _getAttributeDefinitionValues(parentContent, mapping, row, elementDefinition, language, prefix);
255            }
256        }
257        else if (viewItem instanceof ModelViewItemGroup modelViewItemGroup)
258        {
259            List<ViewItem> children = modelViewItemGroup.getViewItems();
260            @SuppressWarnings("unchecked")
261            Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewItem.getName());
262            @SuppressWarnings("unchecked")
263            Map<String, Object> nestedMapValues = (Map<String, Object>) (nestedMap.get(ImportCSVFileHelper.NESTED_MAPPING_VALUES));
264            if (ModelItemTypeConstants.REPEATER_TYPE_ID.equals(modelViewItemGroup.getDefinition().getType().getId()))
265            {
266                return _getRepeaterValues(parentContent, modelViewItemGroup, row, createAction, editAction, language, children, nestedMap, errors, prefix, importMode, additionalTransientVars);
267            }
268            else
269            {
270                return _getCompositeValues(parentContent, viewItem, row, createAction, editAction, language, children, nestedMapValues, errors, prefix, importMode, additionalTransientVars);
271            }
272        }
273        else
274        {
275            errors.add(viewItem);
276            throw new RuntimeException("Import from CSV file: unsupported type of ViewItem for view: " + viewItem.getName());
277        }
278    }
279
280    private Map<String, Object> _getCompositeValues(Optional<? extends Content> parentContent, ViewItem viewItem, Map<String, String> row, int createAction, int editAction,
281            String language, List<ViewItem> children, Map<String, Object> nestedMapValues, List<ViewItem> errors, String prefix, ImportMode importMode, Map<String, Object> additionalTransientVars)
282    {
283        Map<String, Object> compositeValues = new HashMap<>();
284        for (ViewItem child : children)
285        {
286            try
287            {
288                compositeValues.put(child.getName(), _getValue(parentContent, child, nestedMapValues, row, createAction, editAction, language, errors, prefix + viewItem.getName() + ModelItem.ITEM_PATH_SEPARATOR, importMode, additionalTransientVars));
289            }
290            catch (Exception e)
291            {
292                errors.add(viewItem);
293                getLogger().error("Import from CSV file: error while trying to get values for view: {}", viewItem.getName(), e);
294            }
295        }
296        return compositeValues;
297    }
298
299    private SynchronizableRepeater _getRepeaterValues(Optional<? extends Content> parentContent, ModelViewItemGroup viewItem, Map<String, String> row, int createAction, int editAction, String language,
300            List<ViewItem> children, Map<String, Object> nestedMap, List<ViewItem> errors, String prefix, ImportMode importMode, Map<String, Object> additionalTransientVars)
301    {
302        @SuppressWarnings("unchecked")
303        Map<String, Object> mappingValues = (Map<String, Object>) nestedMap.get(ImportCSVFileHelper.NESTED_MAPPING_VALUES);
304        @SuppressWarnings("unchecked")
305        List<String> attributeIdNames = (List<String>) nestedMap.getOrDefault(ImportCSVFileHelper.NESTED_MAPPING_ID, List.of());
306        Map<String, Object> repeaterValues = new HashMap<>();
307        List<Map<String, Object>> repeaterValuesList = new ArrayList<>();
308        List<Integer> indexList = new ArrayList<>();
309
310        Optional<Repeater> repeater = parentContent.map(p -> p.getRepeater(prefix + viewItem.getName()));
311        
312        if (repeater.isPresent() && !attributeIdNames.isEmpty() && _allAttributesFilled(mappingValues, attributeIdNames))
313        {
314            indexList = repeater.get()
315                .getEntries()
316                .stream()
317                .filter(entry ->
318                {
319                    return attributeIdNames.stream()
320                        .allMatch(attributeName ->
321                        {
322                            // Get the entry value
323                            Object entryValue = entry.getValue(attributeName);
324                            
325                            // Get the row value for the attribute (from CSV file)
326                            Object rowValue = Optional.of(attributeName)
327                                    .map(mappingValues::get)
328                                    .map(String.class::cast)
329                                    .map(row::get)
330                                    .orElse(null);
331                            
332                            // Transform the row value to the right type then compare entry value and row value
333                            return Optional.of(attributeName)
334                                .map(viewItem::getModelViewItem)
335                                .map(ViewElement.class::cast)
336                                .map(ViewElement::getDefinition)
337                                .map(ElementDefinition::getType)
338                                .map(def -> def.castValue(rowValue))
339                                .map(value -> value.equals(entryValue))
340                                .orElse(false);
341                        });
342                })
343                .map(RepeaterEntry::getPosition)
344                .collect(Collectors.toList());
345        }
346        
347        // If entries match with attribute ids, we replace the first entries
348        // Else if the repeater exists, we assume we are on the next model item after the last one
349        // Else (the repeater doesn't exist, we set the rowIndex to first index
350        Integer rowIndex = indexList.isEmpty() ? repeater.map(Repeater::getSize).orElse(1) : indexList.get(0);
351        
352        for (ViewItem child : children)
353        {
354            try
355            {
356                Object entryValues = _getValue(parentContent, child, mappingValues, row, createAction, editAction, language, errors, prefix + viewItem.getName() + "[" + rowIndex + "]" + ModelItem.ITEM_PATH_SEPARATOR, importMode, additionalTransientVars);
357                if (entryValues != null)
358                {
359                    repeaterValues.put(child.getName(), entryValues);
360                }
361            }
362            catch (Exception e)
363            {
364                errors.add(viewItem);
365                getLogger().error("Import from CSV file: error while trying to get values for view: {}", viewItem.getName(), e);
366            }
367        }
368        repeaterValuesList.add(repeaterValues);
369        
370        if (indexList.isEmpty())
371        {
372            return SynchronizableRepeater.appendOrRemove(repeaterValuesList, Set.of());
373        }
374        else
375        {
376            // If several rows match the id, only replace the first but add a warning
377            if (indexList.size() > 1)
378            {
379                errors.add(viewItem);
380            }
381            return SynchronizableRepeater.replace(repeaterValuesList, List.of(rowIndex));
382        }
383    }
384    
385    private Object _getContentAttributeDefinitionValues(Optional<? extends Content> parentContent, Map<String, Object> mapping, Map<String, String> row,
386            int createAction, int editAction, String language, ViewElementAccessor viewElementAccessor, ContentAttributeDefinition contentAttributeDefinition, List<ViewItem> errors, ImportMode importMode, Map<String, Object> additionalTransientVars) throws Exception
387    {
388        String contentTypeId = contentAttributeDefinition.getContentTypeId();
389        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
390        @SuppressWarnings("unchecked")
391        Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewElementAccessor.getName());
392        @SuppressWarnings("unchecked")
393        Map<String, Object> mappingValues = (Map<String, Object>) nestedMap.get(ImportCSVFileHelper.NESTED_MAPPING_VALUES);
394        @SuppressWarnings("unchecked")
395        List<String> attributeIdNames = (List<String>) nestedMap.get(ImportCSVFileHelper.NESTED_MAPPING_ID);
396        if (_allAttributesFilled(mappingValues, attributeIdNames))
397        {
398            View view = new View();
399            view.addViewItems(viewElementAccessor.getViewItems());
400            SynchronizeResult synchronizeResult = _synchronizeContent(row, contentType, view, attributeIdNames, mappingValues, createAction, editAction, null, language, errors, importMode, parentContent, additionalTransientVars);
401            
402            Optional<ModifiableWorkflowAwareContent> attachedContent = synchronizeResult.content();
403            
404            if (synchronizeResult.hasKey && attachedContent.isEmpty() && importMode == ImportMode.UPDATE_ONLY)
405            {
406                errors.add(viewElementAccessor);
407            }
408            
409            // If content is multiple, we keep the old value list and we check if content was already inside, and add it otherwise
410            if (attachedContent.isPresent() && contentAttributeDefinition.isMultiple())
411            {
412                Optional<ContentValue[]> multipleContents = parentContent.map(c -> c.getValue(contentAttributeDefinition.getPath()));
413                if (!_containsContent(attachedContent.get(), multipleContents))
414                {
415                    // If there is no list or if it is empty, add it. Otherwise, we have to check if it is inside.
416                    SynchronizableValue syncValue = new SynchronizableValue(List.of(attachedContent.get()));
417                    syncValue.setMode(Mode.APPEND);
418                    return syncValue;
419                }
420                else if (multipleContents.isPresent())
421                {
422                    // return existing values as otherwise the values would be erased
423                    SynchronizableValue syncValue = new SynchronizableValue(Arrays.asList(multipleContents.get()));
424                    syncValue.setMode(Mode.REPLACE);
425                    return syncValue;
426                }
427            }
428            else
429            {
430                return attachedContent.orElse(null);
431            }
432        }
433        return null;
434    }
435    
436    private boolean _containsContent(ModifiableWorkflowAwareContent attachedContent, Optional<ContentValue[]> multipleContents)
437    {
438        return multipleContents
439                .map(Arrays::stream)
440                .orElseGet(Stream::empty)
441                .map(ContentValue::getContentId)
442                .anyMatch(valueFromContent -> valueFromContent.equals(attachedContent.getId()));
443    }
444    
445    private Object _getAttributeDefinitionValues(Optional<? extends Content> parentContent, Map<String, Object> mapping, Map<String, String> row, ElementDefinition elementDefinition, String language, String prefix)
446    {
447        ElementType elementType = elementDefinition.getType();
448        String elementName = elementDefinition.getName();
449        String elementColumn = (String) mapping.get(elementName);
450        String valueAsString = row.get(elementColumn);
451        
452        Object value;
453        if (ModelItemTypeConstants.MULTILINGUAL_STRING_ELEMENT_TYPE_ID.equals(elementType.getId()) && !MultilingualStringHelper.matchesMultilingualStringPattern(valueAsString))
454        {
455            MultilingualString multilingualString = new MultilingualString();
456            multilingualString.add(LocaleUtils.toLocale(language), valueAsString);
457            value = multilingualString;
458        }
459        else
460        {
461            value = elementType.castValue(valueAsString);
462        }
463        
464        if (elementDefinition.isMultiple())
465        {
466            // Build path with index for repeaters.
467            String pathWithIndex = prefix + elementDefinition.getName();
468            
469            Optional<Object[]> values = parentContent.map(c -> c.getValue(pathWithIndex));
470            if (!_containsValue(value, parentContent.map(c -> c.getValue(pathWithIndex))))
471            {
472                // If there is no list or if it is empty, add it. Otherwise, we have to check if it is inside.
473                // If there is no parentContent, still append as we want to fill the values map anyway.
474                SynchronizableValue syncValue = new SynchronizableValue(value != null ? List.of(value) : List.of());
475                syncValue.setMode(Mode.APPEND);
476                return syncValue;
477            }
478            else if (values.isPresent())
479            {
480                // return existing values as otherwise the values would be erased
481                SynchronizableValue syncValue = new SynchronizableValue(Arrays.asList(values.get()));
482                syncValue.setMode(Mode.REPLACE);
483                return syncValue;
484            }
485        }
486        else
487        {
488            return value;
489        }
490        
491        return null;
492    }
493    
494    private boolean _containsValue(Object value, Optional<Object[]> multipleValues)
495    {
496        return multipleValues
497                .map(Arrays::stream)
498                .orElseGet(Stream::empty)
499                .anyMatch(valueFromContent -> valueFromContent.equals(value));
500    }
501
502    private SynchronizeResult _synchronizeContent(Map<String, String> row, ContentType contentType, View view, List<String> attributeIdNames, Map<String, Object> mappingValues, int createAction, int editAction, String workflowName, String language, List<ViewItem> errors, ImportMode importMode, Optional< ? extends Content> parentContent, Map<String, Object> additionalTransientVars) throws Exception
503    {
504        SynchronizeResult synchronizeResult = _getOrCreateContent(mappingValues, row, contentType, Optional.ofNullable(workflowName), createAction, language, attributeIdNames, parentContent, importMode, additionalTransientVars);
505        Optional<ModifiableWorkflowAwareContent> content = synchronizeResult.content();
506        
507        // If we are on CREATE_ONLY mode and content already exists, or if we are on UPDATE_ONLY mode and content does not exists, stop recursivity
508        if (importMode == ImportMode.CREATE_ONLY && !synchronizeResult.isCreated() || importMode == ImportMode.UPDATE_ONLY && content.isEmpty())
509        {
510            return synchronizeResult;
511        }
512        
513        Map<String, Object> values = _getValues(content, row, view, mappingValues, createAction, editAction, language, errors, importMode, additionalTransientVars);
514        if (!values.isEmpty())
515        {
516            if (content.isEmpty())
517            {
518                // Throw this exception only when values are filled, as an empty content should not trigger any warning
519                throw new ContentImportException("Can't create and fill content of content type '" + contentType.getId() + "' and following values '" + values + "' : at least one of those identifiers is null : " + attributeIdNames);
520            }
521            else
522            {
523                try
524                {
525                    _editContent(editAction, view, values, content.get());
526                }
527                catch (WorkflowException e)
528                {
529                    errors.addAll(view.getViewItems());
530                    getLogger().error("[{}] Import from CSV file: error editing the content [{}] after import, some values have not been set", contentType.getId(), content.get().getId(), e);
531                }
532            }
533        }
534        
535        return synchronizeResult;
536    }
537    
538    /**
539     * Returns a list of additional expression to retrieve the existing content
540     * @param mapping The mapping
541     * @param row The row
542     * @param contentType The content type
543     * @param parentContent The parent content
544     * @param additionalTransientVars Additional transients vars
545     * @param valuesUsedAsIds The values used as ids, if some values are added in expression, they can be added in this list to be printed in error logs
546     * @return a list of additional expression to retrieve the existing content
547     */
548    protected List<Expression> additionalExpressionToRetrieveContent(Map<String, Object> mapping, Map<String, String> row, ContentType contentType, Optional<? extends Content> parentContent, Map<String, Object> additionalTransientVars, List<String> valuesUsedAsIds)
549    {
550        // No additional expression by default
551        return List.of();
552    }
553    
554    private SynchronizeResult _getOrCreateContent(Map<String, Object> mapping, Map<String, String> row, ContentType contentType, Optional<String> workflowName, int createAction, String language, List<String> attributeIdNames, Optional<? extends Content> parentContent, ImportMode importMode, Map<String, Object> additionalTransientVars) throws ContentImportException, WorkflowException
555    {
556        AndExpression expression = new AndExpression();
557        List<String> values = new ArrayList<>();
558        
559        for (String attributeName : attributeIdNames)
560        {
561            ModelItem modelItem = contentType.getModelItem(attributeName);
562            String attributePath = (String) mapping.get(attributeName);
563            String value = row.get(attributePath);
564            values.add(value);
565            
566            if (value == null)
567            {
568                return new SynchronizeResult(false, Optional.empty(), false);
569            }
570            
571            // Get content
572            if (ModelItemTypeConstants.MULTILINGUAL_STRING_ELEMENT_TYPE_ID.equals(modelItem.getType().getId()))
573            {
574                expression.add(new MultilingualStringExpression(attributeName, Operator.EQ, value, language));
575            }
576            else
577            {
578                expression.add(new StringExpression(attributeName, Operator.EQ, value));
579            }
580        }
581        
582        expression.addAll(additionalExpressionToRetrieveContent(mapping, row, contentType, parentContent, additionalTransientVars, values));
583
584        expression.add(_contentTypeEP.createHierarchicalCTExpression(contentType.getId()));
585        
586        if (!contentType.isMultilingual())
587        {
588            expression.add(new StringExpression("language", Operator.EQ, language, ExpressionContext.newInstance().withInternal(true)));
589        }
590        
591        String xPathQuery = ContentQueryHelper.getContentXPathQuery(expression);
592        AmetysObjectIterable<ModifiableDefaultContent> matchingContents = _resolver.query(xPathQuery);
593        if (matchingContents.getSize() > 1)
594        {
595            throw new ContentImportException("More than one content found for type " + contentType.getId() + " with "
596                    + attributeIdNames + " as identifier and " + values + " as value");
597        }
598        else if (matchingContents.getSize() == 1)
599        {
600            return new SynchronizeResult(false, Optional.of(matchingContents.iterator().next()), true);
601        }
602        else if (importMode == ImportMode.UPDATE_ONLY)
603        {
604            return new SynchronizeResult(false, Optional.empty(), true);
605        }
606        
607        // Create content
608
609        if (contentType.isAbstract())
610        {
611            throw new ContentImportException("Can not create content for type " + contentType.getId() + " with "
612                    + attributeIdNames + " as identifier and " + values + " as value, the content type is abstract");
613        }
614        
615        Map<String, Object> result;
616        String title;
617        if (mapping.containsKey("title"))
618        {
619            title = row.get(mapping.get("title"));
620        }
621        else
622        {
623            title = _i18nUtils.translate(contentType.getDefaultTitle(), language);
624        }
625        
626        
627        String finalWorkflowName = workflowName.or(contentType::getDefaultWorkflowName)
628                                               .orElseThrow(() -> new ContentImportException("No workflow specified for content type " + contentType.getId() + " with "
629                                                       + attributeIdNames + " as identifier and " + values + " as value"));
630
631        Map<String, Object> inputs = new HashMap<>();
632        inputs.put(CreateContentFunction.INITIAL_VALUE_SUPPLIER, new Function<List<String>, Object>()
633        {
634            public Object apply(List<String> keys)
635            {
636                // Browse the mapping to find the column related to the attribute
637                Object nestedValue = mapping;
638                for (String key : keys)
639                {
640                    nestedValue = ((Map) nestedValue).get(key);
641                    // If nestedValue is null, the attribute is absent from the map, no value can be found
642                    if (nestedValue == null)
643                    {
644                        return null;
645                    }
646                    // If nestedValue is a map, the key is a complex element such a content or a composite,
647                    // we need to keep browsing the map to find the column
648                    if (nestedValue instanceof Map)
649                    {
650                        nestedValue = ((Map) nestedValue).get(ImportCSVFileHelper.NESTED_MAPPING_VALUES);
651                    }
652                }
653                
654                // Get the value of the attribute for the current row
655                return row.get(nestedValue.toString());
656            }
657        });
658        
659        parentContent.ifPresent(content -> inputs.put(CreateContentFunction.PARENT_CONTEXT_VALUE, content.getId()));
660        
661        // Add all additional transient vars
662        inputs.putAll(additionalTransientVars);
663        
664        // CONTENTIO-253 To avoid issue with title starting with a non letter character, we prefix the name with the contentTypeId
665        String prefix = StringUtils.substringAfterLast(contentType.getId(), ".").toLowerCase();
666        String contentName = prefix + "-" + title;
667        
668        if (contentType.isMultilingual())
669        {
670            inputs.put(CreateContentFunction.CONTENT_LANGUAGE_KEY, language);
671            result = _contentWorkflowHelper.createContent(finalWorkflowName, createAction, contentName, Map.of(language, title), new String[] {contentType.getId()}, null, inputs);
672        }
673        else
674        {
675            result = _contentWorkflowHelper.createContent(finalWorkflowName, createAction, contentName, title, new String[] {contentType.getId()}, null, language, inputs);
676        }
677        
678        ModifiableWorkflowAwareContent content = (ModifiableWorkflowAwareContent) result.get(AbstractContentWorkflowComponent.CONTENT_KEY);
679        return new SynchronizeResult(true, Optional.of(content), true);
680    }
681    
682    private Map<String, Object> _getValues(Optional<ModifiableWorkflowAwareContent> content, Map<String, String> row, View view, Map<String, Object> mappingValues, int createAction, int editAction, String language, List<ViewItem> errors, ImportMode importMode, Map<String, Object> additionalTransientVars)
683    {
684        Map<String, Object> values = new HashMap<>();
685        
686        for (ViewItem viewItem : view.getViewItems())
687        {
688            try
689            {
690                Object value = _getValue(content, viewItem, mappingValues, row, createAction, editAction, language, errors, StringUtils.EMPTY, importMode, additionalTransientVars);
691                if (value != null)
692                {
693                    values.put(viewItem.getName(), value);
694                }
695            }
696            catch (Exception e)
697            {
698                errors.add(viewItem);
699                getLogger().error("Import from CSV file: error while trying to get values for item '{}'", viewItem.getName(), e);
700            }
701        }
702        
703        return values;
704    }
705    
706    private boolean _allAttributesFilled(Map<String, Object> mappingValues, List<String> attributeNames)
707    {
708        return mappingValues.entrySet()
709            .stream()
710            .filter(entry -> attributeNames.contains(entry.getKey()))
711            .map(Entry::getValue)
712            .allMatch(Objects::nonNull);
713    }
714    
715    /**
716     * Synchronize result
717     * @param isCreated If the content was created
718     * @param content The content
719     * @param hasKey has key
720     */
721    protected record SynchronizeResult(boolean isCreated, Optional<ModifiableWorkflowAwareContent> content, boolean hasKey) { /* empty */ }
722    
723}