001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.synchronize;
017
018import java.util.ArrayList;
019import java.util.Collections;
020import java.util.HashMap;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.stream.Collectors;
025
026import org.apache.avalon.framework.configuration.Configuration;
027import org.apache.avalon.framework.configuration.ConfigurationException;
028import org.apache.avalon.framework.service.ServiceException;
029import org.apache.avalon.framework.service.ServiceManager;
030import org.apache.commons.lang3.StringUtils;
031import org.slf4j.Logger;
032
033import org.ametys.cms.ObservationConstants;
034import org.ametys.cms.content.ContentHelper;
035import org.ametys.cms.contenttype.ContentType;
036import org.ametys.cms.repository.Content;
037import org.ametys.cms.repository.DefaultContent;
038import org.ametys.cms.repository.ModifiableDefaultContent;
039import org.ametys.cms.repository.WorkflowAwareContent;
040import org.ametys.core.observation.Event;
041import org.ametys.plugins.repository.AmetysObjectIterable;
042import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
043
044import com.google.common.collect.ImmutableMap;
045
046/**
047 * Abstract implementation of {@link SynchronizableContentsCollection}.
048 */
049public abstract class AbstractSimpleSynchronizableContentsCollection extends AbstractSynchronizableContentsCollection
050{
051    /** The extension point for Synchronizing Content Operators */
052    protected SynchronizingContentOperatorExtensionPoint _synchronizingContentOperatorEP;
053    
054    /** SCC helper */
055    protected SynchronizableContentsCollectionHelper _sccHelper;
056    /** The content helper */
057    protected ContentHelper _contentHelper;
058    
059    private List<String> _handledContents;
060    
061    @Override
062    public void service(ServiceManager manager) throws ServiceException
063    {
064        super.service(manager);
065        _contentHelper = (ContentHelper) manager.lookup(ContentHelper.ROLE);
066        _synchronizingContentOperatorEP = (SynchronizingContentOperatorExtensionPoint) manager.lookup(SynchronizingContentOperatorExtensionPoint.ROLE);
067        _sccHelper = (SynchronizableContentsCollectionHelper) manager.lookup(SynchronizableContentsCollectionHelper.ROLE);
068    }
069    
070    @Override
071    public void configure(Configuration configuration) throws ConfigurationException
072    {
073        super.configure(configuration);
074        _handledContents = new ArrayList<>();
075    }
076
077    @Override
078    public List<ModifiableDefaultContent> populate(Logger logger)
079    {
080        _handledContents.clear();
081        List<ModifiableDefaultContent> populatedContents = super.populate(logger);
082        _handledContents.clear();
083        return populatedContents;
084    }
085    
086    @Override
087    protected List<ModifiableDefaultContent> _internalPopulate(Logger logger)
088    {
089        return _importOrSynchronizeContents(new HashMap<>(), false, logger);
090    }
091    
092    /**
093     * Adds the given content as handled (i.e. will not be removed if _removalSync is true) 
094     * @param id The id of the content
095     */
096    protected void _handleContent(String id)
097    {
098        _handledContents.add(id);
099    }
100    
101    /**
102     * Returns true if the given content is handled
103     * @param id The content to test
104     * @return true if the given content is handled
105     */
106    protected boolean _isHandled(String id)
107    {
108        return _handledContents.contains(id);
109    }
110
111    /**
112     * Imports or synchronizes a content for each available language
113     * @param idValue The unique identifier of the content
114     * @param remoteValues The remote values
115     * @param forceImport To force import and ignoring the synchronize existing contents only option
116     * @param logger The logger
117     * @return The list of synchronized or imported contents
118     */
119    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
120    {
121        List<ModifiableDefaultContent> contents = new ArrayList<>();
122
123        for (String lang : getLanguages())
124        {
125            contents.addAll(_importOrSynchronizeContent(idValue, lang, remoteValues, forceImport, logger));
126        }
127        
128        return contents;
129    }
130    
131    /**
132     * Imports or synchronizes a content for a given language
133     * @param idValue The unique identifier of the content
134     * @param lang The language of content to import or synchronize
135     * @param remoteValues The remote values
136     * @param forceImport To force import and ignoring the synchronize existing contents only option
137     * @param logger The logger
138     * @return The list of imported and synchronized contents
139     */
140    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, String lang, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
141    {
142        List<ModifiableDefaultContent> contents = new ArrayList<>();
143        
144        try
145        {
146            ModifiableDefaultContent content = getContent(lang, idValue);
147            if (content != null)
148            {
149                contents.add(_synchronizeContent(content, remoteValues, logger));
150            }
151            else if (forceImport || !synchronizeExistingContentsOnly())
152            {
153                contents.add(_importContent(idValue, null, lang, remoteValues, logger));
154            }
155        }
156        catch (Exception e)
157        {
158            _nbError++;
159            logger.error("An error occurred while importing or synchronizing content", e);
160        }
161        
162        return contents;
163    }
164
165    @Override
166    public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception
167    {
168        String idValue = content.getMetadataHolder().getString(getIdField());
169        
170        Map<String, Object> parameters = putIdParameter(idValue);
171        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
172        if (!results.isEmpty())
173        {
174            try
175            {
176                _synchronizeContent(content, results.get(idValue), logger);
177            }
178            catch (Exception e)
179            {
180                _nbError++;
181                logger.error("An error occurred while importing or synchronizing content", e);
182                throw e;
183            }
184        }
185    }
186
187    /**
188     * Synchronize a content with remove values.
189     * @param content The content to synchronize
190     * @param remoteValues Values to synchronize
191     * @param logger The logger
192     * @return The synchronized content
193     * @throws Exception if an error occurs.
194     */
195    protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
196    {
197        long startTime = System.currentTimeMillis();
198        
199        String contentTitle = content.getTitle();
200        String lang = content.getLanguage();
201        
202        // Update content
203        logger.info("Start synchronizing content '{}' for language {}", contentTitle, lang);
204
205        _ensureTitleIsPresent(content, remoteValues, logger);
206        
207        boolean hasChanges = _fillContent(remoteValues, content, false, logger);
208        hasChanges = additionalSynchronizeOperations(content, remoteValues, logger) || hasChanges;
209        
210        if (hasChanges)
211        {
212            boolean success = applyChanges(content, logger);
213            if (success)
214            {
215                _nbSynchronizedContents++;
216                logger.info("Some changes were detected for content '{}' and language {}", contentTitle, lang);
217            }
218        }
219        else
220        {
221            _nbNotChangedContents++;
222            logger.info("No changes detected for content '{}' and language {}", contentTitle, lang);
223        }
224        
225        // Do additional operation on the content
226        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
227        if (synchronizingContentOperator != null)
228        {
229            synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
230        }
231        else
232        {
233            logger.warn("Cannot find synchronizing content operator with id '{}'. No additional operation has been done.", getSynchronizingContentOperator());
234        }
235        
236        long endTime = System.currentTimeMillis();
237        logger.info("End synchronization of content '{}' for language {} in {} ms", contentTitle, lang, endTime - startTime);
238        
239        return content;
240    }
241    
242    @Override
243    public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception
244    {
245        List<ModifiableDefaultContent> createdContents = new ArrayList<>();
246        
247        Map<String, Object> parameters = putIdParameter(idValue);
248        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
249        if (!results.isEmpty())
250        {
251            for (String lang : getLanguages())
252            {
253                try
254                {
255                    createdContents.add(_importContent(idValue, importParams, lang, results.get(idValue), logger));
256                }
257                catch (Exception e)
258                {
259                    _nbError++;
260                    logger.error("An error occurred while importing or synchronizing content", e);
261                }
262            }
263        }
264        
265        return createdContents;
266    }
267    
268    /**
269     * Set search parameters for the ID value.
270     * @param idValue Value to search
271     * @return Map with the search parameters
272     */
273    protected abstract Map<String, Object> putIdParameter(String idValue);
274
275    /**
276     * Import a content from remote values.
277     * @param idValue Id (for import/synchronization) of the content to import
278     * @param importParams Specific parameters for import
279     * @param lang Lang of the content
280     * @param remoteValues Values of the content
281     * @param logger The logger
282     * @return The content created by import, or null
283     * @throws Exception if an error occurs.
284     */
285    protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
286    {
287        ModifiableDefaultContent content = getContent(lang, idValue);
288        if (content != null)
289        {
290            logger.warn("The content '{}' for language {} already exists and cannot be imported ", idValue, lang);
291        }
292        else
293        {
294            long startTime = System.currentTimeMillis();
295
296            // Calculate contentTitle
297            String contentTitle = idValue;
298            if (remoteValues.containsKey("title"))
299            {
300                List<Object> remoteTitles = remoteValues.get("title");
301                contentTitle = (String) remoteTitles.stream().filter(obj -> obj instanceof String && StringUtils.isNotEmpty((String) obj)).findFirst().orElse(idValue);
302            }
303            
304            // Create new content
305            logger.info("Start importing content '{}' for language {}", contentTitle, lang);
306    
307            content = createContentAction(lang, contentTitle, logger);
308            if (content != null)
309            {
310                // Synchronize content metadata
311                _fillContent(remoteValues, content, true, logger);
312                updateSCCProperty(content);
313                
314                additionalImportOperations(content, remoteValues, importParams, logger);
315                
316                content.saveChanges();
317                content.checkpoint();
318                
319                // Validate content if allowed
320                validateContent(content, logger);
321                
322                _nbCreatedContents++;
323
324                // Do additional operation on the content
325                SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
326                synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
327                
328                // Notify a content was imported
329                Map<String, Object> eventParams = new HashMap<>();
330                eventParams.put(ObservationConstants.ARGS_CONTENT, content);
331                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId());
332                _observationManager.notify(new Event(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, _currentUserProvider.getUser(), eventParams));
333                
334                long endTime = System.currentTimeMillis();
335                logger.info("End import of content '{}' for language {} in {} ms", content.getId(), lang, endTime - startTime);
336            }
337        }
338        
339        return content;
340    }
341    
342    /**
343     * Add specific fields to the content.
344     * @param content Content to update
345     * @param remoteValues Values of the content
346     * @param importParams Import parameters
347     * @param logger The logger
348     * @return <code>true</code> if there are changes
349     */
350    protected boolean additionalImportOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger)
351    {
352        return additionalCommonOperations(content, remoteValues, importParams, logger);
353    }
354
355    /**
356     * Add specific fields to the content.
357     * @param content Content to update
358     * @param remoteValues Values of the content
359     * @param logger The logger
360     * @return <code>true</code> if there are changes
361     */
362    protected boolean additionalSynchronizeOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger)
363    {
364        return additionalCommonOperations(content, remoteValues, null, logger);
365    }
366    
367    /**
368     * Add specific fields to the content during import or synchronization.
369     * @param content Content to update
370     * @param remoteValues Values of the content
371     * @param importParams the import params
372     * @param logger The logger
373     * @return <code>true</code> if there are changes
374     */
375    protected boolean additionalCommonOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger)
376    {
377        // Do nothing by default
378        return false;
379    }
380    
381    private void _ensureTitleIsPresent(Content content, Map<String, List<Object>> remoteValues, Logger logger)
382    {
383        if (remoteValues.containsKey("title"))
384        {
385            List<Object> titleValues = remoteValues.get("title");
386            boolean atLeastOneTitle = titleValues.stream()
387                                                 .filter(String.class::isInstance)
388                                                 .map(String.class::cast)
389                                                 .anyMatch(StringUtils::isNotBlank);
390            if (atLeastOneTitle)
391            {
392                return;
393            }
394        }
395        
396        // Force to current title
397        logger.warn("The remote value of 'title' is empty for the content {}. The 'title' metadata is mandatory, the current title will remain.", content);
398        remoteValues.put("title", Collections.singletonList(content.getTitle()));
399    }
400
401    @Override
402    public ModifiableDefaultContent getContent(String lang, String idValue)
403    {
404        String query = _getContentPathQuery(lang, idValue, getContentType());
405        AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(query);
406        
407        if (contents.getSize() > 0)
408        {
409            return contents.iterator().next();
410        }
411        return null;
412    }
413    
414    /**
415     * Creates content action with result from request
416     * @param lang The language
417     * @param contentTitle The content title
418     * @param logger The logger
419     * @return The content id, or null of a workflow error occured
420     */
421    protected ModifiableDefaultContent createContentAction(String lang, String contentTitle, Logger logger)
422    {
423        return createContentAction(getContentType(), getWorkflowName(), getInitialActionId(), lang, contentTitle, logger);
424    }
425    
426    /**
427     * Fill the content with remote values.
428     * @param remoteValues The remote values
429     * @param content The content to synchronize
430     * @param create <code>true</code> if content is creating, false if it is updated
431     * @param logger The logger
432     * @return <code>true</code> if changes were made
433     */
434    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
435    {
436        boolean hasChanges = false;
437        
438        if (content.isLocked())
439        {
440            logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner());
441        }
442        else
443        {
444            Map<String, Object> params = ImmutableMap.of("contentType", getContentType());
445            ContentType contentType = _contentTypeEP.getExtension(getContentType());
446            for (String metadataPath : remoteValues.keySet())
447            {
448                hasChanges = _synchronizeMetadata(content, contentType, metadataPath, metadataPath, remoteValues.get(metadataPath), getLocalAndExternalFields(params).contains(metadataPath), create, logger) || hasChanges;
449            }
450        }
451        
452        return hasChanges;
453    }
454    
455    /**
456     * Validates a content after import
457     * @param content The content to validate
458     * @param logger The logger
459     */
460    protected void validateContent(WorkflowAwareContent content, Logger logger)
461    {
462        if (validateAfterImport())
463        {
464            validateContent(content, getValidateActionId(), logger);
465        }
466    }
467    
468    /**
469     * Get the value of metadata holding the unique identifier of the synchronized content
470     * @param content The content
471     * @return The value
472     */
473    protected String _getIdFieldValue(DefaultContent content)
474    {
475        ModifiableCompositeMetadata metadataHolder = _getMetadataHolder(content.getMetadataHolder(), getIdField());
476        
477        String[] pathSegments = getIdField().split("/");
478        String metadataName = pathSegments[pathSegments.length - 1];
479        
480        return metadataHolder.getString(metadataName, null);
481    }
482    
483    @Override
484    public Map<String, Map<String, Object>> search(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger)
485    {
486        // Search
487        Map<String, Map<String, Object>> results = internalSearch(_removeEmptyParameters(parameters), offset, limit, sort, logger);
488                
489        return results;
490    }
491
492    /**
493     * Search values and return the result without any treatment.
494     * @param parameters Search parameters to restrict the search
495     * @param offset Begin of the search
496     * @param limit Number of results
497     * @param sort Sort of results (ignored for LDAP results)
498     * @param logger The logger
499     * @return Map of results without any treatment.
500     */
501    protected abstract Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger);
502
503    /**
504     * Search values and return the result organized by metadata and transformed by the {@link SynchronizingContentOperator} if exists.
505     * @param parameters Search parameters to restrict the search
506     * @param logger The logger
507     * @return Map of results organized by metadata.
508     */
509    protected Map<String, Map<String, List<Object>>> getTransformedRemoteValues(Map<String, Object> parameters, Logger logger)
510    {
511        Map<String, Map<String, List<Object>>> remoteValues = getRemoteValues(parameters, logger);
512        
513        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
514        if (synchronizingContentOperator != null)
515        {
516            Map<String, Map<String, List<Object>>> transformedRemoteValues = new LinkedHashMap<>();
517            for (String key : remoteValues.keySet())
518            {
519                transformedRemoteValues.put(key, synchronizingContentOperator.transform(remoteValues.get(key), logger));
520            }
521            
522            return transformedRemoteValues;
523        }
524        else
525        {
526            logger.warn("Cannot find synchronizing content operator with id '{}'. No transformation has applied on remote values", getSynchronizingContentOperator());
527            return remoteValues; // no transformation
528        }
529    }
530    
531    /**
532     * Search values and return the result organized by metadata
533     * @param parameters Search parameters to restrict the search
534     * @param logger The logger
535     * @return Map of results organized by metadata.
536     */
537    protected abstract Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger);
538    
539    @Override
540    public void updateSyncInformations(ModifiableDefaultContent content, String syncCode, Logger logger) throws Exception
541    {
542        updateSCCProperty(content);
543        content.getMetadataHolder().setMetadata(getIdField(), syncCode);
544        content.saveChanges();
545        content.checkpoint();
546    }
547
548    @Override
549    public int getTotalCount(Map<String, Object> parameters, Logger logger)
550    {
551        return search(parameters, 0, Integer.MAX_VALUE, null, logger).size();
552    }
553
554    /**
555     * Import or synchronize several contents from search params.
556     * @param searchParams Search parameters
557     * @param forceImport To force import and ignoring the synchronize existing contents only option
558     * @param logger The logger
559     * @return The {@link List} of imported or synchronized {@link ModifiableDefaultContent}
560     */
561    protected List<ModifiableDefaultContent> _importOrSynchronizeContents(Map<String, Object> searchParams, boolean forceImport, Logger logger)
562    {
563        List<ModifiableDefaultContent> contents = new ArrayList<>();
564        
565        Map<String, Map<String, List<Object>>> remoteValuesByContent = getTransformedRemoteValues(searchParams, logger);
566        for (String idValue : remoteValuesByContent.keySet())
567        {
568            Map<String, List<Object>> remoteValues = remoteValuesByContent.get(idValue);
569            _handleContent(idValue);
570            contents.addAll(_importOrSynchronizeContent(idValue, remoteValues, forceImport, logger));
571        }
572        
573        return contents;
574    }
575    
576    @Override
577    protected List<Content> _getContentsToRemove(AmetysObjectIterable<ModifiableDefaultContent> contents)
578    {
579        return contents.stream()
580            .filter(content -> !_isHandled(_getIdFieldValue(content)))
581            .collect(Collectors.toList());
582    }
583}