Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.csv;
017
018import java.io.IOException;
019import java.util.ArrayList;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Locale;
023import java.util.Map;
024import java.util.Set;
025
026import org.apache.avalon.framework.component.Component;
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang3.ObjectUtils;
031import org.supercsv.io.ICsvListReader;
032import org.supercsv.util.Util;
033
034import org.ametys.cms.FilterNameHelper;
035import org.ametys.cms.contenttype.ContentAttributeDefinition;
036import org.ametys.cms.contenttype.ContentType;
037import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
038import org.ametys.cms.data.type.AbstractMultilingualStringElementType;
039import org.ametys.cms.data.type.impl.MultilingualStringRepositoryElementType;
040import org.ametys.cms.repository.Content;
041import org.ametys.cms.repository.ContentQueryHelper;
042import org.ametys.cms.repository.ContentTypeExpression;
043import org.ametys.cms.repository.ModifiableDefaultContent;
044import org.ametys.cms.repository.ModifiableWorkflowAwareContent;
045import org.ametys.cms.workflow.AbstractContentWorkflowComponent;
046import org.ametys.cms.workflow.ContentWorkflowHelper;
047import org.ametys.cms.workflow.CreateContentFunction;
048import org.ametys.core.util.I18nUtils;
049import org.ametys.plugins.contentio.in.ContentImportException;
050import org.ametys.plugins.repository.AmetysObjectIterable;
051import org.ametys.plugins.repository.AmetysObjectResolver;
052import org.ametys.plugins.repository.data.holder.ModifiableModelAwareDataHolder;
053import org.ametys.plugins.repository.data.holder.group.impl.ModifiableModelAwareRepeater;
054import org.ametys.plugins.repository.data.holder.values.SynchronizableRepeater;
055import org.ametys.plugins.repository.data.holder.values.SynchronizableValue;
056import org.ametys.plugins.repository.data.holder.values.SynchronizableValue.Mode;
057import org.ametys.plugins.repository.data.type.ModelItemTypeConstants;
058import org.ametys.plugins.repository.metadata.MultilingualString;
059import org.ametys.plugins.repository.query.expression.AndExpression;
060import org.ametys.plugins.repository.query.expression.Expression;
061import org.ametys.plugins.repository.query.expression.Expression.Operator;
062import org.ametys.plugins.repository.query.expression.MultilingualStringExpression;
063import org.ametys.plugins.repository.query.expression.StringExpression;
064import org.ametys.runtime.model.ElementDefinition;
065import org.ametys.runtime.model.ModelItem;
066import org.ametys.runtime.model.ModelViewItemGroup;
067import org.ametys.runtime.model.View;
068import org.ametys.runtime.model.ViewElement;
069import org.ametys.runtime.model.ViewElementAccessor;
070import org.ametys.runtime.model.ViewItem;
071import org.ametys.runtime.model.type.ElementType;
072import org.ametys.runtime.plugin.component.AbstractLogEnabled;
073
074import com.opensymphony.workflow.WorkflowException;
075
076/**
077 * Import contents from an uploaded CSV file.
078 */
079public class CSVImporter extends AbstractLogEnabled implements Component, Serviceable
080{
081    /** Avalon Role */
082    public static final String ROLE = CSVImporter.class.getName();
083
084    private ContentWorkflowHelper _contentWorkflowHelper;
085
086    private ContentTypeExtensionPoint _contentTypeEP;
087    
088    private AmetysObjectResolver _resolver;
089
090    private I18nUtils _i18nUtils;
091    
092    @Override
093    public void service(ServiceManager smanager) throws ServiceException
094    {
095        _resolver = (AmetysObjectResolver) smanager.lookup(AmetysObjectResolver.ROLE);
096        _contentTypeEP = (ContentTypeExtensionPoint) smanager.lookup(ContentTypeExtensionPoint.ROLE);
097        _contentWorkflowHelper = (ContentWorkflowHelper) smanager.lookup(ContentWorkflowHelper.ROLE);
098        _i18nUtils = (I18nUtils) smanager.lookup(I18nUtils.ROLE);
099    }
100
101    /**
102     * Extract contents from CSV file
103     * @param mapping mapping of content attributes and CSV file header
104     * @param view View of importing content
105     * @param contentType content type to import
106     * @param listReader mapReader to parse CSV file
107     * @param createAction creation action id
108     * @param editAction edition action id
109     * @param workflowName workflow name
110     * @param language language of created content.
111     * @return list of created contents
112     * @throws IOException IOException while reading CSV
113     */
114    public Map<String, Object> importContentsFromCSV(Map<String, Object> mapping, View view, ContentType contentType, ICsvListReader listReader, int createAction, int editAction, String workflowName, String language) throws IOException
115    {
116        List<String> contentIds = new ArrayList<>();
117        String[] columns = listReader.getHeader(true);
118        int nbErrors = 0;
119        int nbWarnings = 0;
120        List<String> row = null;
121        
122        while ((row = listReader.read()) != null)
123        {
124            try
125            {
126
127                if (listReader.length() != columns.length) 
128                {
129                    getLogger().error("Import from CSV file: content skipped because of invalid row: {}", row);
130                    nbErrors++;
131                    continue;
132                }
133
134                Map<String, String> rowMap = new HashMap<>();
135                Util.filterListToMap(rowMap, columns, row);
136                List<ViewItem> errors = new ArrayList<>();
137                Content content = _processContent(view, rowMap, contentType, mapping, createAction, editAction, workflowName, language, errors);
138                contentIds.add(content.getId());
139                if (!errors.isEmpty())
140                {
141                    nbWarnings++;
142                }
143            }
144            catch (Exception e)
145            {
146                nbErrors++;
147                getLogger().error("Import from CSV file: error importing the content on line {}", listReader.getLineNumber(), e);
148            }
149        }
150        
151        Map<String, Object> results = new HashMap<>();
152        results.put("contentIds", contentIds);
153        results.put("nbErrors", nbErrors);
154        results.put("nbWarnings", nbWarnings);
155        return results;
156    }
157    
158    private Content _processContent(View view, Map<String, String> row, ContentType contentType, Map<String, Object> mapping, int createAction, int editAction, String workflowName, String language, List<ViewItem> errors) throws Exception
159    {
160        String attributeName = (String) mapping.get("id");
161        @SuppressWarnings("unchecked")
162        Map<String, Object> mappingValues = (Map<String, Object>) mapping.get("values");
163        Map<String, Object> values = new HashMap<>();
164        ModifiableWorkflowAwareContent content = _getOrCreateContent(mappingValues, row, contentType, workflowName, createAction, language, view.getModelViewItem(attributeName));
165        
166        for (ViewItem viewItem : view.getViewItems())
167        {
168            try
169            {
170                if (!_isId(viewItem, attributeName))
171                {
172                    Object value = _getValue(content, viewItem, mappingValues, row, createAction, editAction, workflowName, language, errors, "");
173                    if (value != null)
174                    {
175                        values.put(viewItem.getName(), value);
176                    }
177                }
178            }
179            catch (Exception e)
180            {
181                errors.add(viewItem);
182                getLogger().error("Import from CSV file: error while trying to get values for view: {}", viewItem.getName(), e);
183            }
184        }
185        
186        _editContent(editAction, values, content);
187        
188        return content;
189    }
190
191    private void _editContent(int editAction, Map<String, Object> values, ModifiableWorkflowAwareContent content) throws WorkflowException
192    {
193        if (!values.isEmpty())
194        {
195            _contentWorkflowHelper.editContent(content, values, editAction);
196        }
197    }
198    
199    private boolean _isId(ViewItem viewItem, String attributeName)
200    {
201        if (viewItem instanceof ViewElement)
202        {
203            ViewElement viewElement = (ViewElement) viewItem;
204            ElementDefinition elementDefinition = viewElement.getDefinition();
205            if (!(elementDefinition instanceof ContentAttributeDefinition))
206            {
207                String elementName = elementDefinition.getName();
208                return elementName.equals(attributeName);
209            }
210        }
211        return false;
212    }
213    
214    private Object _getValue(ModifiableModelAwareDataHolder dataHolder, ViewItem viewItem, Map<String, Object> mapping, Map<String, String> row, int createAction, int editAction, String workflowName, String language, List<ViewItem> errors, String prefix) throws Exception
215    {
216        if (viewItem instanceof ViewElement)
217        {
218            ViewElement viewElement = (ViewElement) viewItem;
219            ElementDefinition elementDefinition = viewElement.getDefinition();
220            if (elementDefinition instanceof ContentAttributeDefinition)
221            {
222                return _getContentAttributeDefinitionValues(viewItem, mapping, row, createAction, editAction, workflowName, language, viewElement, elementDefinition, errors);
223            }
224            else
225            {
226                return _getAttributeDefinitionValues(mapping, row, elementDefinition, language);
227            }
228        }
229        else if (viewItem instanceof ModelViewItemGroup)
230        {
231            ModelViewItemGroup modelViewItemGroup = (ModelViewItemGroup) viewItem;
232            List<ViewItem> children = modelViewItemGroup.getViewItems();
233            @SuppressWarnings("unchecked")
234            Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewItem.getName());
235            @SuppressWarnings("unchecked")
236            Map<String, Object> nestedMapValues = (Map<String, Object>) (nestedMap.get("values"));
237            if (ModelItemTypeConstants.REPEATER_TYPE_ID.equals(modelViewItemGroup.getDefinition().getType().getId()))
238            {
239                return _getRepeaterValues(dataHolder, modelViewItemGroup, row, createAction, editAction, workflowName, language, children, nestedMap, errors, prefix);
240            }
241            else
242            {
243                return _getCompositeValues(dataHolder, viewItem, row, createAction, editAction, workflowName, language, children, nestedMapValues, errors, prefix);
244            }
245        }
246        else
247        {
248            errors.add(viewItem);
249            throw new RuntimeException("Import from CSV file: unsupported type of ViewItem for view: " + viewItem.getName());
250        }
251    }
252
253    private Map<String, Object> _getCompositeValues(ModifiableModelAwareDataHolder dataHolder, ViewItem viewItem, Map<String, String> row, int createAction, int editAction,
254            String workflowName, String language, List<ViewItem> children, Map<String, Object> nestedMapValues, List<ViewItem> errors, String prefix)
255    {
256        Map<String, Object> compositeValues = new HashMap<>();
257        for (ViewItem child : children)
258        {
259            try
260            {
261                compositeValues.put(child.getName(), _getValue(dataHolder, child, nestedMapValues, row, createAction, editAction, workflowName, language, errors, prefix + viewItem.getName() + ModelItem.ITEM_PATH_SEPARATOR));
262            }
263            catch (Exception e)
264            {
265                errors.add(viewItem);
266                getLogger().error("Import from CSV file: error while trying to get values for view: {}", viewItem.getName(), e);
267            }
268        }
269        return compositeValues;
270    }
271
272    private SynchronizableRepeater _getRepeaterValues(ModifiableModelAwareDataHolder dataHolder, ModelViewItemGroup viewItem, Map<String, String> row, int createAction, int editAction, String workflowName, String language,
273            List<ViewItem> children, Map<String, Object> nestedMap, List<ViewItem> errors, String prefix)
274    {
275        @SuppressWarnings("unchecked")
276        Map<String, Object> mappingValues = (Map<String, Object>) (nestedMap.get("values"));
277        String attributeName = (String) nestedMap.get("id");
278        String idValue = (String) mappingValues.get(attributeName);
279        Map<String, Object> repeaterValues = new HashMap<>();
280        List<Map<String, Object>> repeaterValuesList = new ArrayList<>();
281        List<Integer> indexList = new ArrayList<>();
282
283        if (ObjectUtils.allNotNull(idValue, row.get(idValue), dataHolder) && dataHolder.hasValue(prefix + viewItem.getName()))
284        {
285            ViewElement viewElement = (ViewElement) viewItem.getModelViewItem(attributeName);
286            ElementDefinition elementDefinition = viewElement.getDefinition();
287            ElementType elementType = elementDefinition.getType();
288            ModifiableModelAwareRepeater repeater = dataHolder.getValue(prefix + viewItem.getName());
289            repeater.getEntries().forEach(entry -> 
290            {
291                Object value = elementType.castValue(row.get(idValue));
292                if (value != null && value.equals(entry.getValue(attributeName)))
293                {
294                    indexList.add(entry.getPosition());
295                }
296            }); 
297        }
298        
299        Integer rowIndex = indexList.size() > 0 ? indexList.get(0) : 1;
300        
301        for (ViewItem child : children)
302        {
303            try
304            {
305                Object entryValues = _getValue(dataHolder, child, mappingValues, row, createAction, editAction, workflowName, language, errors, prefix + viewItem.getName() + "[" + rowIndex + "]" + ModelItem.ITEM_PATH_SEPARATOR);
306                repeaterValues.put(child.getName(), entryValues);
307            }
308            catch (Exception e)
309            {
310                errors.add(viewItem);
311                getLogger().error("Import from CSV file: error while trying to get values for view: {}", viewItem.getName(), e);
312            }
313        }
314        repeaterValuesList.add(repeaterValues);
315        
316        
317        if (indexList.size() == 1)
318        {
319            return SynchronizableRepeater.replace(repeaterValuesList, indexList);
320        }
321        // If several rows match the id, only replace the first but add a warning
322        else if (indexList.size() > 1)
323        {
324            errors.add(viewItem);
325            return SynchronizableRepeater.replace(repeaterValuesList, List.of(indexList.get(0)));
326        }
327        else
328        {
329            return SynchronizableRepeater.appendOrRemove(repeaterValuesList, Set.of());
330        }
331        
332    }
333    
334    private Object _getContentAttributeDefinitionValues(ViewItem viewItem, Map<String, Object> mapping, Map<String, String> row,
335            int createAction, int editAction, String workflowName, String language, ViewElement viewElement, ElementDefinition elementDefinition, List<ViewItem> errors) throws Exception
336    {
337        ViewElementAccessor viewElementAccessor = (ViewElementAccessor) viewElement;
338        ContentAttributeDefinition contentAttributeDefinition = (ContentAttributeDefinition) elementDefinition;
339        String contentTypeId = contentAttributeDefinition.getContentTypeId();
340        ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
341        @SuppressWarnings("unchecked")
342        Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewItem.getName());
343        @SuppressWarnings("unchecked")
344        Map<String, Object> mappingValues = (Map<String, Object>) (nestedMap.get("values"));
345        String attributeName = (String) nestedMap.get("id");
346        String idValue = (String) mappingValues.get(attributeName);
347        if (row.get(idValue) != null)
348        {
349            Map<String, Object> values = new HashMap<>();
350            ModifiableWorkflowAwareContent attachedContent = _getOrCreateContent(mappingValues, row, contentType, workflowName, createAction, language, viewElementAccessor.getModelViewItem(attributeName));
351            for (ViewItem nestedViewItem : viewElementAccessor.getViewItems())
352            {
353                try
354                {
355                    if (!_isId(nestedViewItem, attributeName))
356                    {
357                        Object value = _getValue(attachedContent, nestedViewItem, mappingValues, row, createAction, editAction, workflowName, language, errors, "");
358                        if (value != null)
359                        {
360                            values.put(nestedViewItem.getName(), value);
361                        }
362                    }
363                }
364                catch (Exception e)
365                {
366                    errors.add(viewItem);
367                    getLogger().error("Import from CSV file: error while trying to get values for view: {}", viewItem.getName(), e);
368                }
369            } 
370
371            if (!values.isEmpty())
372            {
373                _editContent(editAction, values, attachedContent);
374            }
375            
376            //If content is multiple, we keep the old value list and we check if content was already inside, and add it otherwise
377            if (contentAttributeDefinition.isMultiple())
378            {
379                SynchronizableValue syncValue = new SynchronizableValue(List.of(attachedContent));
380                syncValue.setMode(Mode.APPEND);
381                return syncValue;
382            }
383            else
384            {
385                return attachedContent;
386            } 
387        }
388        return null;
389    }
390
391    private Object _getAttributeDefinitionValues(Map<String, Object> mapping, Map<String, String> row, ElementDefinition elementDefinition, String language)
392    {
393        ElementType elementType = elementDefinition.getType();
394        String elementName = elementDefinition.getName();
395        String elementColumn = (String) mapping.get(elementName);
396        String valueAsString = row.get(elementColumn);
397        Object value;
398        if (elementType instanceof AbstractMultilingualStringElementType && !valueAsString.contains(":"))
399        {
400            MultilingualString multilingualString = new MultilingualString();
401            multilingualString.add(new Locale(language), valueAsString);
402            value = multilingualString;
403        }
404        else
405        {
406            value = elementType.castValue(valueAsString);
407        }
408        if (elementDefinition.isMultiple())
409        {
410
411            SynchronizableValue syncValue = new SynchronizableValue(List.of(value));
412            syncValue.setMode(Mode.APPEND);
413            return syncValue;
414        }
415        else
416        {
417            return value;
418        }
419    }
420
421    private ModifiableWorkflowAwareContent _getOrCreateContent(Map<String, Object> mapping, Map<String, String> row, ContentType contentType, String workflowName, int createAction, String language, ViewItem viewItem) throws ContentImportException, WorkflowException
422    {
423        ViewElement viewElement = (ViewElement) viewItem;
424        ElementDefinition elementDefinition = viewElement.getDefinition();
425        String attributeName = elementDefinition.getName();
426        String attributePath = (String) mapping.get(attributeName);
427        String value = row.get(attributePath);
428
429        if (value == null)
430        {
431            throw new ContentImportException("Identifier field is empty for content " + contentType.getLabel());
432        }
433        
434        Expression idExpression;
435        if (elementDefinition.getType() instanceof MultilingualStringRepositoryElementType)
436        {
437            idExpression = new MultilingualStringExpression(attributeName, Operator.EQ, value, language);
438        }
439        else
440        {
441            idExpression = new StringExpression(attributeName, Operator.EQ, value);
442        }
443        
444        Expression expression = new AndExpression(
445                new ContentTypeExpression(Operator.EQ, contentType.getId()),
446                idExpression);
447
448        String xPathQuery = ContentQueryHelper.getContentXPathQuery(expression);
449        AmetysObjectIterable<ModifiableDefaultContent> matchingContents = _resolver.query(xPathQuery);
450        if (matchingContents.getSize() > 1)
451        {
452            throw new ContentImportException("More than one content found for type " + contentType.getLabel() + " with " 
453                    + attributeName + " as identifier and " + value + " as value");
454        }
455        else if (matchingContents.getSize() == 1)
456        {
457            return matchingContents.iterator().next();
458        }
459        
460        Map<String, Object> result;
461        String title;
462        if (mapping.containsKey("title"))
463        {
464            title = row.get(mapping.get("title"));
465        }
466        else
467        {
468            title = _i18nUtils.translate(contentType.getDefaultTitle(), language);
469        }
470        
471        String contentName = FilterNameHelper.filterName(title);
472        if (contentType.isMultilingual())
473        {
474
475            Map<String, Object> inputs = new HashMap<>();
476            inputs.put(CreateContentFunction.CONTENT_LANGUAGE_KEY, language);
477            result = _contentWorkflowHelper.createContent(workflowName, createAction, contentName, Map.of(language, title), new String[] {contentType.getId()}, null, null, null, inputs);
478        }
479        else
480        {
481            result = _contentWorkflowHelper.createContent(workflowName, createAction, contentName, title, new String[] {contentType.getId()}, null, language);
482        }
483        ModifiableWorkflowAwareContent content = (ModifiableWorkflowAwareContent) result.get(AbstractContentWorkflowComponent.CONTENT_KEY);
484        if (elementDefinition.getType() instanceof AbstractMultilingualStringElementType)
485        {
486            value = language + ":" + value;
487        }
488        content.setValue(attributeName, elementDefinition.getType().castValue(value));
489        return content;
490    } 
491}