001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.synchronize;
017
018import java.util.ArrayList;
019import java.util.Collection;
020import java.util.HashMap;
021import java.util.HashSet;
022import java.util.LinkedHashMap;
023import java.util.List;
024import java.util.Map;
025import java.util.Optional;
026import java.util.Set;
027import java.util.stream.Collectors;
028import java.util.stream.Stream;
029
030import org.apache.avalon.framework.configuration.Configuration;
031import org.apache.avalon.framework.configuration.ConfigurationException;
032import org.apache.avalon.framework.service.ServiceException;
033import org.apache.avalon.framework.service.ServiceManager;
034import org.apache.commons.lang3.StringUtils;
035import org.slf4j.Logger;
036
037import org.ametys.cms.content.ContentHelper;
038import org.ametys.cms.contenttype.ContentType;
039import org.ametys.cms.repository.Content;
040import org.ametys.cms.repository.ModifiableContent;
041import org.ametys.cms.repository.WorkflowAwareContent;
042import org.ametys.cms.workflow.AbstractContentWorkflowComponent;
043import org.ametys.cms.workflow.EditContentFunction;
044import org.ametys.plugins.contentio.synchronize.workflow.EditSynchronizedContentFunction;
045import org.ametys.plugins.repository.AmetysObjectIterable;
046import org.ametys.plugins.repository.version.VersionableAmetysObject;
047import org.ametys.plugins.workflow.AbstractWorkflowComponent;
048import org.ametys.runtime.model.View;
049
050import com.opensymphony.workflow.WorkflowException;
051
052/**
053 * Abstract implementation of {@link SynchronizableContentsCollection}.
054 */
055public abstract class AbstractSimpleSynchronizableContentsCollection extends AbstractSynchronizableContentsCollection
056{
057    /** The extension point for Synchronizing Content Operators */
058    protected SynchronizingContentOperatorExtensionPoint _synchronizingContentOperatorEP;
059    
060    /** The content helper */
061    protected ContentHelper _contentHelper;
062    
063    private List<String> _handledContents;
064    
065    @Override
066    public void service(ServiceManager manager) throws ServiceException
067    {
068        super.service(manager);
069        _contentHelper = (ContentHelper) manager.lookup(ContentHelper.ROLE);
070        _synchronizingContentOperatorEP = (SynchronizingContentOperatorExtensionPoint) manager.lookup(SynchronizingContentOperatorExtensionPoint.ROLE);
071    }
072    
073    @Override
074    public void configure(Configuration configuration) throws ConfigurationException
075    {
076        super.configure(configuration);
077        _handledContents = new ArrayList<>();
078    }
079
080    @Override
081    public List<ModifiableContent> populate(Logger logger)
082    {
083        _handledContents.clear();
084        List<ModifiableContent> populatedContents = super.populate(logger);
085        _handledContents.clear();
086        return populatedContents;
087    }
088    
089    @Override
090    protected List<ModifiableContent> _internalPopulate(Logger logger)
091    {
092        return _importOrSynchronizeContents(new HashMap<>(), false, logger);
093    }
094    
095    /**
096     * Adds the given content as handled (i.e. will not be removed if _removalSync is true) 
097     * @param id The id of the content
098     */
099    protected void _handleContent(String id)
100    {
101        _handledContents.add(id);
102    }
103    
104    /**
105     * Returns true if the given content is handled
106     * @param id The content to test
107     * @return true if the given content is handled
108     */
109    protected boolean _isHandled(String id)
110    {
111        return _handledContents.contains(id);
112    }
113    
114    /**
115     * Imports or synchronizes a content for each available language
116     * @param idValue The unique identifier of the content
117     * @param remoteValues The remote values
118     * @param forceImport To force import and ignoring the synchronize existing contents only option
119     * @param logger The logger
120     * @return The list of synchronized or imported contents
121     */
122    protected List<ModifiableContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
123    {
124        List<ModifiableContent> contents = new ArrayList<>();
125        
126        for (String lang : getLanguages())
127        {
128            _importOrSynchronizeContent(idValue, lang, remoteValues, forceImport, logger)
129                .ifPresent(contents::add);
130        }
131        
132        return contents;
133    }
134    
135    /**
136     * Imports or synchronizes a content for a given language
137     * @param idValue The unique identifier of the content
138     * @param lang The language of content to import or synchronize
139     * @param remoteValues The remote values
140     * @param forceImport To force import and ignoring the synchronize existing contents only option
141     * @param logger The logger
142     * @return The imported or synchronized content
143     */
144    protected Optional<ModifiableContent> _importOrSynchronizeContent(String idValue, String lang, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
145    {
146        try
147        {
148            ModifiableContent content = getContent(lang, idValue);
149            if (content != null)
150            {
151                return Optional.of(_synchronizeContent(content, remoteValues, logger));
152            }
153            else if (forceImport || !synchronizeExistingContentsOnly())
154            {
155                return Optional.ofNullable(_importContent(idValue, null, lang, remoteValues, logger));
156            }
157        }
158        catch (Exception e)
159        {
160            _nbError++;
161            logger.error("An error occurred while importing or synchronizing content", e);
162        }
163        
164        return Optional.empty();
165    }
166
167    @Override
168    public void synchronizeContent(ModifiableContent content, Logger logger) throws Exception
169    {
170        String idValue = content.getValue(getIdField());
171        
172        Map<String, Object> searchParameters = putIdParameter(idValue);
173        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(searchParameters, logger);
174        if (!results.isEmpty())
175        {
176            try
177            {
178                _synchronizeContent(content, results.get(idValue), logger);
179            }
180            catch (Exception e)
181            {
182                _nbError++;
183                logger.error("An error occurred while importing or synchronizing content", e);
184                throw e;
185            }
186        }
187        else
188        {
189            logger.warn("The content {} ({}) with synchronization code '{}' doesn't exist anymore in the datasource from SCC '{}'", content.getTitle(), content.getId(), idValue, getId());
190        }
191    }
192
193    /**
194     * Synchronize a content with remove values.
195     * @param content The content to synchronize
196     * @param remoteValues Values to synchronize
197     * @param logger The logger
198     * @return The synchronized content
199     * @throws Exception if an error occurs
200     */
201    protected ModifiableContent _synchronizeContent(ModifiableContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
202    {
203        long startTime = System.currentTimeMillis();
204        
205        String contentTitle = content.getTitle();
206        String lang = content.getLanguage();
207        
208        // Update content
209        logger.info("Start synchronizing content '{}' for language {}", contentTitle, lang);
210
211        _ensureTitleIsPresent(content, remoteValues, logger);
212        
213        boolean hasChanged = _fillContent(remoteValues, content, Map.of(), false, logger);
214        if (hasChanged)
215        {
216            _nbSynchronizedContents++;
217            logger.info("Some changes were detected for content '{}' and language {}", contentTitle, lang);
218        }
219        else
220        {
221            _nbNotChangedContents++;
222            logger.info("No changes detected for content '{}' and language {}", contentTitle, lang);
223        }
224        
225        // Do additional operation on the content
226        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
227        if (synchronizingContentOperator != null)
228        {
229            synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
230        }
231        else
232        {
233            logger.warn("Cannot find synchronizing content operator with id '{}'. No additional operation has been done.", getSynchronizingContentOperator());
234        }
235        
236        long endTime = System.currentTimeMillis();
237        logger.info("End synchronization of content '{}' for language {} in {} ms", contentTitle, lang, endTime - startTime);
238        
239        return content;
240    }
241    
242    @Override
243    public List<ModifiableContent> importContent(String idValue, Map<String, Object> additionalParameters, Logger logger) throws Exception
244    {
245        List<ModifiableContent> createdContents = new ArrayList<>();
246        
247        Map<String, Object> searchParameters = putIdParameter(idValue);
248        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(searchParameters, logger);
249        if (!results.isEmpty())
250        {
251            for (String lang : getLanguages())
252            {
253                try
254                {
255                    createdContents.add(_importContent(idValue, additionalParameters, lang, results.get(idValue), logger));
256                }
257                catch (Exception e)
258                {
259                    _nbError++;
260                    logger.error("An error occurred while importing or synchronizing content", e);
261                }
262            }
263        }
264        
265        return createdContents;
266    }
267    
268    /**
269     * Set search parameters for the ID value.
270     * @param idValue Value to search
271     * @return Map with the search parameters
272     */
273    protected abstract Map<String, Object> putIdParameter(String idValue);
274
275    /**
276     * Import a content from remote values.
277     * @param idValue Id (for import/synchronization) of the content to import
278     * @param additionalParameters Specific parameters for import
279     * @param lang Lang of the content
280     * @param remoteValues Values of the content
281     * @param logger The logger
282     * @return The content created by import, or null
283     * @throws Exception if an error occurs.
284     */
285    protected ModifiableContent _importContent(String idValue, Map<String, Object> additionalParameters, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
286    {
287        long startTime = System.currentTimeMillis();
288
289        // Calculate contentTitle
290        String contentTitle = Optional.ofNullable(remoteValues.get(Content.ATTRIBUTE_TITLE))
291                .map(Collection::stream)
292                .orElseGet(Stream::empty)
293                .filter(String.class::isInstance)
294                .map(String.class::cast)
295                .filter(StringUtils::isNotBlank)
296                .findFirst()
297                .orElse(idValue);
298
299        // Create new content
300        logger.info("Start importing content '{}' for language {}", contentTitle, lang);
301
302        ModifiableContent content = createContentAction(lang, contentTitle, logger);
303        if (content != null)
304        {
305            _sccHelper.updateSCCProperty(content, getId());
306            _fillContent(remoteValues, content, additionalParameters, true, logger);
307
308            if (content instanceof WorkflowAwareContent)
309            {
310                // Validate content if allowed
311                validateContent((WorkflowAwareContent) content, logger);
312            }
313
314            _nbCreatedContents++;
315
316            // Do additional operation on the content
317            SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
318            synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
319
320            long endTime = System.currentTimeMillis();
321            logger.info("End import of content '{}' for language {} in {} ms", content.getId(), lang, endTime - startTime);
322        }
323        
324        return content;
325    }
326    
327    private void _ensureTitleIsPresent(Content content, Map<String, List<Object>> remoteValues, Logger logger)
328    {
329        if (remoteValues.containsKey(Content.ATTRIBUTE_TITLE))
330        {
331            boolean atLeastOneTitle =  remoteValues.get(Content.ATTRIBUTE_TITLE)
332                                                   .stream()
333                                                   .filter(String.class::isInstance)
334                                                   .map(String.class::cast)
335                                                   .anyMatch(StringUtils::isNotBlank);
336            if (atLeastOneTitle)
337            {
338                return;
339            }
340        }
341        
342        // Force to current title
343        logger.warn("The remote value of '{}' is empty for the content {}. The '{}' attribute is mandatory, the current title will remain.", Content.ATTRIBUTE_TITLE, content, Content.ATTRIBUTE_TITLE);
344        remoteValues.put(Content.ATTRIBUTE_TITLE, List.of(content.getTitle()));
345    }
346
347    @Override
348    public ModifiableContent getContent(String lang, String idValue)
349    {
350        String query = _getContentPathQuery(lang, idValue, getContentType());
351        AmetysObjectIterable<ModifiableContent> contents = _resolver.query(query);
352        
353        if (contents.getSize() > 0)
354        {
355            return contents.iterator().next();
356        }
357        return null;
358    }
359    
360    /**
361     * Creates content action with result from request
362     * @param lang The language
363     * @param contentTitle The content title
364     * @param logger The logger
365     * @return The content id, or null of a workflow error occurs
366     */
367    protected ModifiableContent createContentAction(String lang, String contentTitle, Logger logger)
368    {
369        return createContentAction(getContentType(), getWorkflowName(), getInitialActionId(), lang, contentTitle, logger);
370    }
371    
372    /**
373     * Fill the content with remote values.
374     * @param remoteValues The remote values
375     * @param content The content to synchronize
376     * @param additionalParameters Additional parameters
377     * @param create <code>true</code> if content is creating, false if it is updated
378     * @param logger The logger
379     * @return <code>true</code> if the content has been modified, <code>false</code> otherwise
380     * @throws Exception if an error occurs
381     */
382    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableContent content, Map<String, Object> additionalParameters, boolean create, Logger logger) throws Exception
383    {
384        if (content instanceof WorkflowAwareContent)
385        {
386            // Transform remote value to get values with the cardinality corresponding to the model
387            Map<String, Object> contentValues = _transformRemoteValuesCardinality(remoteValues, getContentType());
388            String idValue = (String) contentValues.get(getIdField());
389            
390            // Add additional values
391            contentValues.putAll(getAdditionalAttributeValues(idValue, content, additionalParameters, create, logger));
392            if (create)
393            {
394                // Remove title from values: the title has been set by the create content action
395                contentValues.remove(Content.ATTRIBUTE_TITLE);
396            }
397            
398            Set<String> notSynchronizedContentIds = getNotSynchronizedRelatedContentIds(content, contentValues, additionalParameters, content.getLanguage(), logger);
399            
400            return _editContent((WorkflowAwareContent) content, Optional.empty(), contentValues, additionalParameters, create, notSynchronizedContentIds, logger);
401        }
402        
403        return false;
404    }
405    
406    /**
407     * Synchronize the content with given values.
408     * @param content The content to synchronize
409     * @param view the view containing the item to edit
410     * @param values the values
411     * @param additionalParameters Additional parameters
412     * @param create <code>true</code> if content is creating, false if it is updated
413     * @param notSynchronizedContentIds the ids of the contents related to the given content but that are not part of the synchronization
414     * @param logger The logger
415     * @return <code>true</code> if the content has been modified, <code>false</code> otherwise
416     * @throws WorkflowException if an error occurs
417     */
418    protected boolean _editContent(WorkflowAwareContent content, Optional<View> view, Map<String, Object> values, Map<String, Object> additionalParameters, boolean create, Set<String> notSynchronizedContentIds, Logger logger) throws WorkflowException
419    {
420        Map<String, Object> inputs = new HashMap<>();
421        inputs.put(EditSynchronizedContentFunction.SCC_KEY, this);
422        inputs.put(EditSynchronizedContentFunction.SCC_LOGGER_KEY, logger);
423        inputs.put(EditSynchronizedContentFunction.ADDITIONAL_PARAMS_KEY, additionalParameters);
424        inputs.put(EditSynchronizedContentFunction.SYNCHRO_INVERT_EDIT_ACTION_ID_KEY, getSynchronizeActionId());
425        inputs.put(EditSynchronizedContentFunction.NOT_SYNCHRONIZED_RELATED_CONTENT_IDS_KEY, notSynchronizedContentIds);
426
427
428        Map<String, Object> parameters = new HashMap<>();
429        parameters.put(EditContentFunction.VALUES_KEY, values);
430        parameters.put(EditContentFunction.QUIT, true);
431        parameters.put(EditSynchronizedContentFunction.IMPORT, create);
432        inputs.put(AbstractWorkflowComponent.CONTEXT_PARAMETERS_KEY, parameters);
433        
434        Map<String, Object> actionResult = _contentWorkflowHelper.doAction(content, getSynchronizeActionId(), inputs);
435        return (boolean) actionResult.getOrDefault(AbstractContentWorkflowComponent.HAS_CHANGED_KEY, false);
436    }
437    
438    /**
439     * Validates a content after import
440     * @param content The content to validate
441     * @param logger The logger
442     */
443    protected void validateContent(WorkflowAwareContent content, Logger logger)
444    {
445        if (validateAfterImport())
446        {
447            validateContent(content, getValidateActionId(), logger);
448        }
449    }
450    
451    @Override
452    public Map<String, Map<String, Object>> search(Map<String, Object> searchParameters, int offset, int limit, List<Object> sort, Logger logger)
453    {
454        // Search
455        Map<String, Map<String, Object>> results = internalSearch(_removeEmptyParameters(searchParameters), offset, limit, sort, logger);
456                
457        return results;
458    }
459
460    /**
461     * Search values and return the result without any treatment.
462     * @param searchParameters Search parameters to restrict the search
463     * @param offset Begin of the search
464     * @param limit Number of results
465     * @param sort Sort of results (ignored for LDAP results)
466     * @param logger The logger
467     * @return Map of results without any treatment.
468     */
469    protected abstract Map<String, Map<String, Object>> internalSearch(Map<String, Object> searchParameters, int offset, int limit, List<Object> sort, Logger logger);
470
471    /**
472     * Search values and return the result organized by attributes and transformed by the {@link SynchronizingContentOperator} if exists.
473     * @param searchParameters Search parameters to restrict the search
474     * @param logger The logger
475     * @return Map of results organized by attributes.
476     */
477    protected Map<String, Map<String, List<Object>>> getTransformedRemoteValues(Map<String, Object> searchParameters, Logger logger)
478    {
479        Map<String, Map<String, List<Object>>> remoteValues = getRemoteValues(searchParameters, logger);
480        return _transformRemoteValues(remoteValues, logger);
481    }
482    
483    /**
484     * Search values and return the result organized by attributes
485     * @param searchParameters Search parameters to restrict the search
486     * @param logger The logger
487     * @return Map of results organized by attributes.
488     */
489    protected abstract Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> searchParameters, Logger logger);
490
491    /**
492     * Transform the given remote values by the {@link SynchronizingContentOperator} if exists.
493     * @param remoteValues The remote values
494     * @param logger The logger
495     * @return the transformed values
496     */
497    protected Map<String, Map<String, List<Object>>> _transformRemoteValues(Map<String, Map<String, List<Object>>> remoteValues, Logger logger)
498    {
499        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
500        if (synchronizingContentOperator != null)
501        {
502            Map<String, Map<String, List<Object>>> transformedRemoteValues = new LinkedHashMap<>();
503            ContentType contentType = _contentTypeEP.getExtension(getContentType());
504            for (String key : remoteValues.keySet())
505            {
506                transformedRemoteValues.put(key, synchronizingContentOperator.transform(contentType, remoteValues.get(key), logger));
507            }
508            
509            return transformedRemoteValues;
510        }
511        else
512        {
513            logger.warn("Cannot find synchronizing content operator with id '{}'. No transformation has applied on remote values", getSynchronizingContentOperator());
514            return remoteValues; // no transformation
515        }
516    }
517    
518    /**
519     * Retrieves additional values to synchronize for a content
520     * @param idValue id value of the content
521     * @param content The content
522     * @param additionalParameters Additional parameters
523     * @param create <code>true</code> if the content has been newly created, <code>false</code> otherwise
524     * @param logger The logger
525     * @return the values to add
526     */
527    protected Map<String, Object> getAdditionalAttributeValues(String idValue, Content content, Map<String, Object> additionalParameters, boolean create, Logger logger)
528    {
529        // No additional values by default
530        return new LinkedHashMap<>();
531    }
532    
533    /**
534     * Retrieves the ids of the contents related to the given content but that are not part of the synchronization
535     * @param content content
536     * @param contentValues the content values that will be set
537     * @param additionalParameters Additional parameters
538     * @param lang Language of the content
539     * @param logger The logger
540     * @return the ids of the contents that are not part of the synchronization
541     */
542    protected Set<String> getNotSynchronizedRelatedContentIds(Content content, Map<String, Object> contentValues, Map<String, Object> additionalParameters, String lang, Logger logger)
543    {
544        // All contents are synchronized by default
545        return new HashSet<>();
546    }
547    
548    @Override
549    public void updateSyncInformations(ModifiableContent content, String syncCode, Logger logger) throws Exception
550    {
551        _sccHelper.updateSCCProperty(content, getId());
552        content.setValue(getIdField(), syncCode);
553        content.saveChanges();
554        
555        if (content instanceof VersionableAmetysObject)
556        {
557            ((VersionableAmetysObject) content).checkpoint();
558        }
559    }
560
561    @Override
562    public int getTotalCount(Map<String, Object> searchParameters, Logger logger)
563    {
564        return search(searchParameters, 0, Integer.MAX_VALUE, null, logger).size();
565    }
566
567    /**
568     * Import or synchronize several contents from search params.
569     * @param searchParameters Search parameters
570     * @param forceImport To force import and ignoring the synchronize existing contents only option
571     * @param logger The logger
572     * @return The {@link List} of imported or synchronized {@link ModifiableContent}
573     */
574    protected List<ModifiableContent> _importOrSynchronizeContents(Map<String, Object> searchParameters, boolean forceImport, Logger logger)
575    {
576        List<ModifiableContent> contents = new ArrayList<>();
577        
578        Map<String, Map<String, List<Object>>> remoteValuesByContent = getTransformedRemoteValues(searchParameters, logger);
579        for (String idValue : remoteValuesByContent.keySet())
580        {
581            Map<String, List<Object>> remoteValues = remoteValuesByContent.get(idValue);
582            _handleContent(idValue);
583            contents.addAll(_importOrSynchronizeContent(idValue, remoteValues, forceImport, logger));
584        }
585        
586        return contents;
587    }
588    
589    @Override
590    protected List<Content> _getContentsToRemove(AmetysObjectIterable<ModifiableContent> contents)
591    {
592        return contents.stream()
593            .filter(content -> !_isHandled(content.getValue(getIdField())))
594            .collect(Collectors.toList());
595    }
596}