001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.synchronize;
017
018import java.util.ArrayList;
019import java.util.Collections;
020import java.util.HashMap;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.stream.Collectors;
025
026import org.apache.avalon.framework.configuration.Configuration;
027import org.apache.avalon.framework.configuration.ConfigurationException;
028import org.apache.avalon.framework.service.ServiceException;
029import org.apache.avalon.framework.service.ServiceManager;
030import org.apache.commons.lang3.StringUtils;
031import org.slf4j.Logger;
032
033import org.ametys.cms.ObservationConstants;
034import org.ametys.cms.content.ContentHelper;
035import org.ametys.cms.contenttype.ContentType;
036import org.ametys.cms.languages.Language;
037import org.ametys.cms.languages.LanguagesManager;
038import org.ametys.cms.repository.Content;
039import org.ametys.cms.repository.DefaultContent;
040import org.ametys.cms.repository.ModifiableDefaultContent;
041import org.ametys.cms.repository.WorkflowAwareContent;
042import org.ametys.core.observation.Event;
043import org.ametys.plugins.repository.AmetysObjectIterable;
044import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
045
046import com.google.common.collect.ImmutableMap;
047
048/**
049 * Abstract implementation of {@link SynchronizableContentsCollection}.
050 */
051public abstract class AbstractSimpleSynchronizableContentsCollection extends AbstractSynchronizableContentsCollection
052{
053    /** The languges manager */
054    protected LanguagesManager _languagesManager;
055    /** The extension point for Synchronizing Content Operators */
056    protected SynchronizingContentOperatorExtensionPoint _synchronizingContentOperatorEP;
057    
058    /** SCC helper */
059    protected SynchronizableContentsCollectionHelper _sccHelper;
060    /** The content helper */
061    protected ContentHelper _contentHelper;
062    
063    private List<String> _handledContents;
064    
065    @Override
066    public void service(ServiceManager manager) throws ServiceException
067    {
068        super.service(manager);
069        _contentHelper = (ContentHelper) manager.lookup(ContentHelper.ROLE);
070        _languagesManager = (LanguagesManager) manager.lookup(LanguagesManager.ROLE);
071        _synchronizingContentOperatorEP = (SynchronizingContentOperatorExtensionPoint) manager.lookup(SynchronizingContentOperatorExtensionPoint.ROLE);
072        _sccHelper = (SynchronizableContentsCollectionHelper) manager.lookup(SynchronizableContentsCollectionHelper.ROLE);
073    }
074    
075    @Override
076    public void configure(Configuration configuration) throws ConfigurationException
077    {
078        super.configure(configuration);
079        _handledContents = new ArrayList<>();
080    }
081
082    @Override
083    public List<ModifiableDefaultContent> populate(Logger logger)
084    {
085        _handledContents.clear();
086        List<ModifiableDefaultContent> populatedContents = super.populate(logger);
087        _handledContents.clear();
088        return populatedContents;
089    }
090    
091    @Override
092    protected List<ModifiableDefaultContent> _internalPopulate(Logger logger)
093    {
094        return _importOrSynchronizeContents(new HashMap<>(), false, logger);
095    }
096    
097    /**
098     * Adds the given content as handled (i.e. will not be removed if _removalSync is true) 
099     * @param id The id of the content
100     */
101    protected void _handleContent(String id)
102    {
103        _handledContents.add(id);
104    }
105    
106    /**
107     * Returns true if the given content is handled
108     * @param id The content to test
109     * @return true if the given content is handled
110     */
111    protected boolean _isHandled(String id)
112    {
113        return _handledContents.contains(id);
114    }
115
116    /**
117     * Imports or synchronizes a content for each available language
118     * @param idValue The unique identifier of the content
119     * @param remoteValues The remote values
120     * @param forceImport To force import and ignoring the synchronize existing contents only option
121     * @param logger The logger
122     * @return The list of synchronized or imported contents
123     */
124    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
125    {
126        List<ModifiableDefaultContent> contents = new ArrayList<>();
127
128        Map<String, Language> languages = _languagesManager.getAvailableLanguages();
129        for (String lang : languages.keySet())
130        {
131            contents.addAll(_importOrSynchronizeContent(idValue, lang, remoteValues, forceImport, logger));
132        }
133        
134        return contents;
135    }
136    
137    /**
138     * Imports or synchronizes a content for a given language
139     * @param idValue The unique identifier of the content
140     * @param lang The language of content to import or synchronize
141     * @param remoteValues The remote values
142     * @param forceImport To force import and ignoring the synchronize existing contents only option
143     * @param logger The logger
144     * @return The list of imported and synchronized contents
145     */
146    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, String lang, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
147    {
148        List<ModifiableDefaultContent> contents = new ArrayList<>();
149        
150        try
151        {
152            ModifiableDefaultContent content = getContent(lang, idValue);
153            if (content != null)
154            {
155                contents.add(_synchronizeContent(content, remoteValues, logger));
156            }
157            else if (forceImport || !synchronizeExistingContentsOnly())
158            {
159                contents.add(_importContent(idValue, null, lang, remoteValues, logger));
160            }
161        }
162        catch (Exception e)
163        {
164            _nbError++;
165            logger.error("An error occurred while importing or synchronizing content", e);
166        }
167        
168        return contents;
169    }
170
171    @Override
172    public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception
173    {
174        String idValue = content.getMetadataHolder().getString(getIdField());
175        
176        Map<String, Object> parameters = putIdParameter(idValue);
177        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
178        if (!results.isEmpty())
179        {
180            try
181            {
182                _synchronizeContent(content, results.get(idValue), logger);
183            }
184            catch (Exception e)
185            {
186                _nbError++;
187                logger.error("An error occurred while importing or synchronizing content", e);
188                throw e;
189            }
190        }
191    }
192
193    /**
194     * Synchronize a content with remove values.
195     * @param content The content to synchronize
196     * @param remoteValues Values to synchronize
197     * @param logger The logger
198     * @return The synchronized content
199     * @throws Exception if an error occurs.
200     */
201    protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
202    {
203        long startTime = System.currentTimeMillis();
204        
205        String contentTitle = content.getTitle();
206        String lang = content.getLanguage();
207        
208        // Update content
209        logger.info("Start synchronizing content '{}' for language {}", contentTitle, lang);
210
211        _ensureTitleIsPresent(content, remoteValues, logger);
212        
213        boolean hasChanges = _fillContent(remoteValues, content, false, logger);
214        hasChanges = additionalSynchronizeOperations(content, remoteValues, logger) || hasChanges;
215        
216        if (hasChanges)
217        {
218            boolean success = applyChanges(content, logger);
219            if (success)
220            {
221                _nbSynchronizedContents++;
222                logger.info("Some changes were detected for content '{}' and language {}", contentTitle, lang);
223            }
224        }
225        else
226        {
227            _nbNotChangedContents++;
228            logger.info("No changes detected for content '{}' and language {}", contentTitle, lang);
229        }
230        
231        // Do additional operation on the content
232        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
233        if (synchronizingContentOperator != null)
234        {
235            synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
236        }
237        else
238        {
239            logger.warn("Cannot find synchronizing content operator with id '" + getSynchronizingContentOperator() + "'. No additional operation has been done.");
240        }
241        
242        long endTime = System.currentTimeMillis();
243        logger.info("End synchronization of content '{}' for language {} in {} ms", contentTitle, lang, endTime - startTime);
244        
245        return content;
246    }
247    
248    @Override
249    public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception
250    {
251        List<ModifiableDefaultContent> createdContents = new ArrayList<>();
252        
253        Map<String, Object> parameters = putIdParameter(idValue);
254        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
255        if (!results.isEmpty())
256        {
257            Map<String, Language> languages = _languagesManager.getAvailableLanguages();
258            for (String lang : languages.keySet())
259            {
260                try
261                {
262                    createdContents.add(_importContent(idValue, importParams, lang, results.get(idValue), logger));
263                }
264                catch (Exception e)
265                {
266                    _nbError++;
267                    logger.error("An error occurred while importing or synchronizing content", e);
268                }
269            }
270        }
271        
272        return createdContents;
273    }
274    
275    /**
276     * Set search parameters for the ID value.
277     * @param idValue Value to search
278     * @return Map with the search parameters
279     */
280    protected abstract Map<String, Object> putIdParameter(String idValue);
281
282    /**
283     * Import a content from remote values.
284     * @param idValue Id (for import/synchronization) of the content to import
285     * @param importParams Specific parameters for import
286     * @param lang Lang of the content
287     * @param remoteValues Values of the content
288     * @param logger The logger
289     * @return The content created by import, or null
290     * @throws Exception if an error occurs.
291     */
292    protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
293    {
294        ModifiableDefaultContent content = getContent(lang, idValue);
295        if (content != null)
296        {
297            logger.warn("The content '{}' for language {} already exists and cannot be imported ", idValue, lang);
298        }
299        else
300        {
301            long startTime = System.currentTimeMillis();
302
303            // Calculate contentTitle
304            String contentTitle = idValue;
305            if (remoteValues.containsKey("title"))
306            {
307                List<Object> remoteTitles = remoteValues.get("title");
308                contentTitle = (String) remoteTitles.stream().filter(obj -> obj instanceof String && StringUtils.isNotEmpty((String) obj)).findFirst().orElse(idValue);
309            }
310            
311            // Create new content
312            logger.info("Start importing content '{}' for language {}", contentTitle, lang);
313    
314            content = createContentAction(lang, contentTitle, logger);
315            if (content != null)
316            {
317                // Synchronize content metadata
318                _fillContent(remoteValues, content, true, logger);
319                updateSCCProperty(content);
320                
321                additionalImportOperations(content, remoteValues, importParams, logger);
322                
323                content.saveChanges();
324                content.checkpoint();
325                
326                // Validate content if allowed
327                validateContent(content, logger);
328                
329                _nbCreatedContents++;
330
331                // Do additional operation on the content
332                SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
333                synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
334                
335                // Notify a content was imported
336                Map<String, Object> eventParams = new HashMap<>();
337                eventParams.put(ObservationConstants.ARGS_CONTENT, content);
338                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId());
339                _observationManager.notify(new Event(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, _currentUserProvider.getUser(), eventParams));
340                
341                long endTime = System.currentTimeMillis();
342                logger.info("End import of content '{}' for language {} in {} ms", content.getId(), lang, endTime - startTime);
343            }
344        }
345        
346        return content;
347    }
348    
349    /**
350     * Add specific fields to the content.
351     * @param content Content to update
352     * @param remoteValues Values of the content
353     * @param importParams Import parameters
354     * @param logger The logger
355     * @return <code>true</code> if there are changes
356     */
357    protected boolean additionalImportOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger)
358    {
359        // Do nothing by default
360        return false;
361    }
362
363    /**
364     * Add specific fields to the content.
365     * @param content Content to update
366     * @param remoteValues Values of the content
367     * @param logger The logger
368     * @return <code>true</code> if there are changes
369     */
370    protected boolean additionalSynchronizeOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger)
371    {
372        // Do nothing by default
373        return false;
374    }
375    
376    private void _ensureTitleIsPresent(Content content, Map<String, List<Object>> remoteValues, Logger logger)
377    {
378        if (remoteValues.containsKey("title"))
379        {
380            List<Object> titleValues = remoteValues.get("title");
381            boolean atLeastOneTitle = titleValues.stream()
382                                                 .filter(String.class::isInstance)
383                                                 .map(String.class::cast)
384                                                 .anyMatch(StringUtils::isNotBlank);
385            if (atLeastOneTitle)
386            {
387                return;
388            }
389        }
390        
391        // Force to current title
392        logger.warn("The remote value of 'title' is empty for the content {}. The 'title' metadata is mandatory, the current title will remain.", content);
393        remoteValues.put("title", Collections.singletonList(content.getTitle()));
394    }
395
396    @Override
397    public ModifiableDefaultContent getContent(String lang, String idValue)
398    {
399        String query = _getContentPathQuery(lang, idValue, getContentType());
400        AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(query);
401        
402        if (contents.getSize() > 0)
403        {
404            return contents.iterator().next();
405        }
406        return null;
407    }
408    
409    /**
410     * Creates content action with result from request
411     * @param lang The language
412     * @param contentTitle The content title
413     * @param logger The logger
414     * @return The content id, or null of a workflow error occured
415     */
416    protected ModifiableDefaultContent createContentAction(String lang, String contentTitle, Logger logger)
417    {
418        return createContentAction(getContentType(), getWorkflowName(), getInitialActionId(), lang, contentTitle, logger);
419    }
420    
421    /**
422     * Fill the content with remote values.
423     * @param remoteValues The remote values
424     * @param content The content to synchronize
425     * @param create <code>true</code> if content is creating, false if it is updated
426     * @param logger The logger
427     * @return <code>true</code> if changes were made
428     */
429    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
430    {
431        boolean hasChanges = false;
432        
433        if (content.isLocked())
434        {
435            logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner());
436        }
437        else
438        {
439            Map<String, Object> params = ImmutableMap.of("contentType", getContentType());
440            ContentType contentType = _contentTypeEP.getExtension(getContentType());
441            for (String metadataPath : remoteValues.keySet())
442            {
443                hasChanges = _synchronizeMetadata(content, contentType, metadataPath, metadataPath, remoteValues.get(metadataPath), getLocalAndExternalFields(params).contains(metadataPath), create, logger) || hasChanges;
444            }
445        }
446        
447        return hasChanges;
448    }
449    
450    /**
451     * Validates a content after import
452     * @param content The content to validate
453     * @param logger The logger
454     */
455    protected void validateContent(WorkflowAwareContent content, Logger logger)
456    {
457        if (validateAfterImport())
458        {
459            validateContent(content, getValidateActionId(), logger);
460        }
461    }
462    
463    /**
464     * Get the value of metadata holding the unique identifier of the synchronized content
465     * @param content The content
466     * @return The value
467     */
468    protected String _getIdFieldValue(DefaultContent content)
469    {
470        ModifiableCompositeMetadata metadataHolder = _getMetadataHolder(content.getMetadataHolder(), getIdField());
471        
472        String[] pathSegments = getIdField().split("/");
473        String metadataName = pathSegments[pathSegments.length - 1];
474        
475        return metadataHolder.getString(metadataName, null);
476    }
477    
478    @Override
479    public Map<String, Map<String, Object>> search(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger)
480    {
481        // Search
482        Map<String, Map<String, Object>> results = internalSearch(_removeEmptyParameters(parameters), offset, limit, sort, logger);
483                
484        return results;
485    }
486
487    /**
488     * Search values and return the result without any treatment.
489     * @param parameters Search parameters to restrict the search
490     * @param offset Begin of the search
491     * @param limit Number of results
492     * @param sort Sort of results (ignored for LDAP results)
493     * @param logger The logger
494     * @return Map of results without any treatment.
495     */
496    protected abstract Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger);
497
498    /**
499     * Search values and return the result organized by metadata and transformed by the {@link SynchronizingContentOperator} if exists.
500     * @param parameters Search parameters to restrict the search
501     * @param logger The logger
502     * @return Map of results organized by metadata.
503     */
504    protected Map<String, Map<String, List<Object>>> getTransformedRemoteValues(Map<String, Object> parameters, Logger logger)
505    {
506        Map<String, Map<String, List<Object>>> remoteValues = getRemoteValues(parameters, logger);
507        
508        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
509        if (synchronizingContentOperator != null)
510        {
511            Map<String, Map<String, List<Object>>> transformedRemoteValues = new LinkedHashMap<>();
512            for (String key : remoteValues.keySet())
513            {
514                transformedRemoteValues.put(key, synchronizingContentOperator.transform(remoteValues.get(key), logger));
515            }
516            
517            return transformedRemoteValues;
518        }
519        else
520        {
521            logger.warn("Cannot find synchronizing content operator with id '" + getSynchronizingContentOperator() + "'. No transformation has applied on remote values");
522            return remoteValues; // no transformation
523        }
524    }
525    
526    /**
527     * Search values and return the result organized by metadata
528     * @param parameters Search parameters to restrict the search
529     * @param logger The logger
530     * @return Map of results organized by metadata.
531     */
532    protected abstract Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger);
533    
534    @Override
535    public void updateSyncInformations(ModifiableDefaultContent content, String syncCode, Logger logger) throws Exception
536    {
537        updateSCCProperty(content);
538        content.getMetadataHolder().setMetadata(getIdField(), syncCode);
539        content.saveChanges();
540        content.checkpoint();
541    }
542
543    @Override
544    public int getTotalCount(Map<String, Object> parameters, Logger logger)
545    {
546        return search(parameters, 0, Integer.MAX_VALUE, null, logger).size();
547    }
548
549    /**
550     * Import or synchronize several contents from search params.
551     * @param searchParams Search parameters
552     * @param forceImport To force import and ignoring the synchronize existing contents only option
553     * @param logger The logger
554     * @return The {@link List} of imported or synchronized {@link ModifiableDefaultContent}
555     */
556    protected List<ModifiableDefaultContent> _importOrSynchronizeContents(Map<String, Object> searchParams, boolean forceImport, Logger logger)
557    {
558        List<ModifiableDefaultContent> contents = new ArrayList<>();
559        
560        Map<String, Map<String, List<Object>>> remoteValuesByContent = getTransformedRemoteValues(searchParams, logger);
561        for (String idValue : remoteValuesByContent.keySet())
562        {
563            Map<String, List<Object>> remoteValues = remoteValuesByContent.get(idValue);
564            _handleContent(idValue);
565            contents.addAll(_importOrSynchronizeContent(idValue, remoteValues, forceImport, logger));
566        }
567        
568        return contents;
569    }
570    
571    @Override
572    protected List<Content> _getContentsToRemove(AmetysObjectIterable<ModifiableDefaultContent> contents)
573    {
574        return contents.stream()
575            .filter(content -> !_isHandled(_getIdFieldValue(content)))
576            .collect(Collectors.toList());
577    }
578}