001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.synchronize;
017
018import java.util.ArrayList;
019import java.util.Collections;
020import java.util.HashMap;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.stream.Collectors;
025
026import org.apache.avalon.framework.configuration.Configuration;
027import org.apache.avalon.framework.configuration.ConfigurationException;
028import org.apache.avalon.framework.service.ServiceException;
029import org.apache.avalon.framework.service.ServiceManager;
030import org.apache.commons.lang3.StringUtils;
031import org.slf4j.Logger;
032
033import org.ametys.cms.ObservationConstants;
034import org.ametys.cms.content.ContentHelper;
035import org.ametys.cms.contenttype.ContentType;
036import org.ametys.cms.repository.Content;
037import org.ametys.cms.repository.DefaultContent;
038import org.ametys.cms.repository.ModifiableDefaultContent;
039import org.ametys.cms.repository.WorkflowAwareContent;
040import org.ametys.core.observation.Event;
041import org.ametys.plugins.repository.AmetysObjectIterable;
042import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
043
044/**
045 * Abstract implementation of {@link SynchronizableContentsCollection}.
046 */
047public abstract class AbstractSimpleSynchronizableContentsCollection extends AbstractSynchronizableContentsCollection
048{
049    /** The extension point for Synchronizing Content Operators */
050    protected SynchronizingContentOperatorExtensionPoint _synchronizingContentOperatorEP;
051    
052    /** SCC helper */
053    protected SynchronizableContentsCollectionHelper _sccHelper;
054    /** The content helper */
055    protected ContentHelper _contentHelper;
056    
057    private List<String> _handledContents;
058    
059    @Override
060    public void service(ServiceManager manager) throws ServiceException
061    {
062        super.service(manager);
063        _contentHelper = (ContentHelper) manager.lookup(ContentHelper.ROLE);
064        _synchronizingContentOperatorEP = (SynchronizingContentOperatorExtensionPoint) manager.lookup(SynchronizingContentOperatorExtensionPoint.ROLE);
065        _sccHelper = (SynchronizableContentsCollectionHelper) manager.lookup(SynchronizableContentsCollectionHelper.ROLE);
066    }
067    
068    @Override
069    public void configure(Configuration configuration) throws ConfigurationException
070    {
071        super.configure(configuration);
072        _handledContents = new ArrayList<>();
073    }
074
075    @Override
076    public List<ModifiableDefaultContent> populate(Logger logger)
077    {
078        _handledContents.clear();
079        List<ModifiableDefaultContent> populatedContents = super.populate(logger);
080        _handledContents.clear();
081        return populatedContents;
082    }
083    
084    @Override
085    protected List<ModifiableDefaultContent> _internalPopulate(Logger logger)
086    {
087        return _importOrSynchronizeContents(new HashMap<>(), false, logger);
088    }
089    
090    /**
091     * Adds the given content as handled (i.e. will not be removed if _removalSync is true) 
092     * @param id The id of the content
093     */
094    protected void _handleContent(String id)
095    {
096        _handledContents.add(id);
097    }
098    
099    /**
100     * Returns true if the given content is handled
101     * @param id The content to test
102     * @return true if the given content is handled
103     */
104    protected boolean _isHandled(String id)
105    {
106        return _handledContents.contains(id);
107    }
108
109    /**
110     * Imports or synchronizes a content for each available language
111     * @param idValue The unique identifier of the content
112     * @param remoteValues The remote values
113     * @param forceImport To force import and ignoring the synchronize existing contents only option
114     * @param logger The logger
115     * @return The list of synchronized or imported contents
116     */
117    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
118    {
119        List<ModifiableDefaultContent> contents = new ArrayList<>();
120
121        for (String lang : getLanguages())
122        {
123            contents.addAll(_importOrSynchronizeContent(idValue, lang, remoteValues, forceImport, logger));
124        }
125        
126        return contents;
127    }
128    
129    /**
130     * Imports or synchronizes a content for a given language
131     * @param idValue The unique identifier of the content
132     * @param lang The language of content to import or synchronize
133     * @param remoteValues The remote values
134     * @param forceImport To force import and ignoring the synchronize existing contents only option
135     * @param logger The logger
136     * @return The list of imported and synchronized contents
137     */
138    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, String lang, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
139    {
140        List<ModifiableDefaultContent> contents = new ArrayList<>();
141        
142        try
143        {
144            ModifiableDefaultContent content = getContent(lang, idValue);
145            if (content != null)
146            {
147                contents.add(_synchronizeContent(content, remoteValues, logger));
148            }
149            else if (forceImport || !synchronizeExistingContentsOnly())
150            {
151                contents.add(_importContent(idValue, null, lang, remoteValues, logger));
152            }
153        }
154        catch (Exception e)
155        {
156            _nbError++;
157            logger.error("An error occurred while importing or synchronizing content", e);
158        }
159        
160        return contents;
161    }
162
163    @Override
164    public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception
165    {
166        String idValue = content.getMetadataHolder().getString(getIdField());
167        
168        Map<String, Object> parameters = putIdParameter(idValue);
169        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
170        if (!results.isEmpty())
171        {
172            try
173            {
174                _synchronizeContent(content, results.get(idValue), logger);
175            }
176            catch (Exception e)
177            {
178                _nbError++;
179                logger.error("An error occurred while importing or synchronizing content", e);
180                throw e;
181            }
182        }
183    }
184
185    /**
186     * Synchronize a content with remove values.
187     * @param content The content to synchronize
188     * @param remoteValues Values to synchronize
189     * @param logger The logger
190     * @return The synchronized content
191     * @throws Exception if an error occurs.
192     */
193    protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
194    {
195        long startTime = System.currentTimeMillis();
196        
197        String contentTitle = content.getTitle();
198        String lang = content.getLanguage();
199        
200        // Update content
201        logger.info("Start synchronizing content '{}' for language {}", contentTitle, lang);
202
203        _ensureTitleIsPresent(content, remoteValues, logger);
204        
205        boolean hasChanges = _fillContent(remoteValues, content, false, logger);
206        hasChanges = additionalSynchronizeOperations(content, remoteValues, logger) || hasChanges;
207        
208        if (hasChanges)
209        {
210            boolean success = applyChanges(content, logger);
211            if (success)
212            {
213                _nbSynchronizedContents++;
214                logger.info("Some changes were detected for content '{}' and language {}", contentTitle, lang);
215            }
216        }
217        else
218        {
219            _nbNotChangedContents++;
220            logger.info("No changes detected for content '{}' and language {}", contentTitle, lang);
221        }
222        
223        // Do additional operation on the content
224        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
225        if (synchronizingContentOperator != null)
226        {
227            synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
228        }
229        else
230        {
231            logger.warn("Cannot find synchronizing content operator with id '{}'. No additional operation has been done.", getSynchronizingContentOperator());
232        }
233        
234        long endTime = System.currentTimeMillis();
235        logger.info("End synchronization of content '{}' for language {} in {} ms", contentTitle, lang, endTime - startTime);
236        
237        return content;
238    }
239    
240    @Override
241    public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception
242    {
243        List<ModifiableDefaultContent> createdContents = new ArrayList<>();
244        
245        Map<String, Object> parameters = putIdParameter(idValue);
246        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
247        if (!results.isEmpty())
248        {
249            for (String lang : getLanguages())
250            {
251                try
252                {
253                    createdContents.add(_importContent(idValue, importParams, lang, results.get(idValue), logger));
254                }
255                catch (Exception e)
256                {
257                    _nbError++;
258                    logger.error("An error occurred while importing or synchronizing content", e);
259                }
260            }
261        }
262        
263        return createdContents;
264    }
265    
266    /**
267     * Set search parameters for the ID value.
268     * @param idValue Value to search
269     * @return Map with the search parameters
270     */
271    protected abstract Map<String, Object> putIdParameter(String idValue);
272
273    /**
274     * Import a content from remote values.
275     * @param idValue Id (for import/synchronization) of the content to import
276     * @param importParams Specific parameters for import
277     * @param lang Lang of the content
278     * @param remoteValues Values of the content
279     * @param logger The logger
280     * @return The content created by import, or null
281     * @throws Exception if an error occurs.
282     */
283    protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
284    {
285        ModifiableDefaultContent content = getContent(lang, idValue);
286        if (content != null)
287        {
288            logger.warn("The content '{}' for language {} already exists and cannot be imported ", idValue, lang);
289        }
290        else
291        {
292            long startTime = System.currentTimeMillis();
293
294            // Calculate contentTitle
295            String contentTitle = idValue;
296            if (remoteValues.containsKey("title"))
297            {
298                List<Object> remoteTitles = remoteValues.get("title");
299                contentTitle = (String) remoteTitles.stream().filter(obj -> obj instanceof String && StringUtils.isNotEmpty((String) obj)).findFirst().orElse(idValue);
300            }
301            
302            // Create new content
303            logger.info("Start importing content '{}' for language {}", contentTitle, lang);
304    
305            content = createContentAction(lang, contentTitle, logger);
306            if (content != null)
307            {
308                // Synchronize content metadata
309                _fillContent(remoteValues, content, true, logger);
310                updateSCCProperty(content);
311                
312                additionalImportOperations(content, remoteValues, importParams, logger);
313                
314                content.saveChanges();
315                content.checkpoint();
316                
317                // Validate content if allowed
318                validateContent(content, logger);
319                
320                _nbCreatedContents++;
321
322                // Do additional operation on the content
323                SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
324                synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
325                
326                // Notify a content was imported
327                Map<String, Object> eventParams = new HashMap<>();
328                eventParams.put(ObservationConstants.ARGS_CONTENT, content);
329                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId());
330                _observationManager.notify(new Event(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, _currentUserProvider.getUser(), eventParams));
331                
332                long endTime = System.currentTimeMillis();
333                logger.info("End import of content '{}' for language {} in {} ms", content.getId(), lang, endTime - startTime);
334            }
335        }
336        
337        return content;
338    }
339    
340    /**
341     * Add specific fields to the content.
342     * @param content Content to update
343     * @param remoteValues Values of the content
344     * @param importParams Import parameters
345     * @param logger The logger
346     * @return <code>true</code> if there are changes
347     */
348    protected boolean additionalImportOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger)
349    {
350        return additionalCommonOperations(content, remoteValues, importParams, true, logger);
351    }
352
353    /**
354     * Add specific fields to the content.
355     * @param content Content to update
356     * @param remoteValues Values of the content
357     * @param logger The logger
358     * @return <code>true</code> if there are changes
359     */
360    protected boolean additionalSynchronizeOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger)
361    {
362        return additionalCommonOperations(content, remoteValues, Collections.EMPTY_MAP, false, logger);
363    }
364    
365    /**
366     * Add specific fields to the content during import or synchronization.
367     * @param content Content to update
368     * @param remoteValues Values of the content
369     * @param importParams the import params
370     * @param create true if the content has been newly created
371     * @param logger The logger
372     * @return <code>true</code> if there are changes
373     */
374    protected boolean additionalCommonOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, boolean create, Logger logger)
375    {
376        // Do nothing by default
377        return false;
378    }
379    
380    private void _ensureTitleIsPresent(Content content, Map<String, List<Object>> remoteValues, Logger logger)
381    {
382        if (remoteValues.containsKey("title"))
383        {
384            List<Object> titleValues = remoteValues.get("title");
385            boolean atLeastOneTitle = titleValues.stream()
386                                                 .filter(String.class::isInstance)
387                                                 .map(String.class::cast)
388                                                 .anyMatch(StringUtils::isNotBlank);
389            if (atLeastOneTitle)
390            {
391                return;
392            }
393        }
394        
395        // Force to current title
396        logger.warn("The remote value of 'title' is empty for the content {}. The 'title' metadata is mandatory, the current title will remain.", content);
397        remoteValues.put("title", Collections.singletonList(content.getTitle()));
398    }
399
400    @Override
401    public ModifiableDefaultContent getContent(String lang, String idValue)
402    {
403        String query = _getContentPathQuery(lang, idValue, getContentType());
404        AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(query);
405        
406        if (contents.getSize() > 0)
407        {
408            return contents.iterator().next();
409        }
410        return null;
411    }
412    
413    /**
414     * Creates content action with result from request
415     * @param lang The language
416     * @param contentTitle The content title
417     * @param logger The logger
418     * @return The content id, or null of a workflow error occured
419     */
420    protected ModifiableDefaultContent createContentAction(String lang, String contentTitle, Logger logger)
421    {
422        return createContentAction(getContentType(), getWorkflowName(), getInitialActionId(), lang, contentTitle, logger);
423    }
424    
425    /**
426     * Fill the content with remote values.
427     * @param remoteValues The remote values
428     * @param content The content to synchronize
429     * @param create <code>true</code> if content is creating, false if it is updated
430     * @param logger The logger
431     * @return <code>true</code> if changes were made
432     */
433    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
434    {
435        boolean hasChanges = false;
436        
437        if (content.isLocked())
438        {
439            logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner());
440        }
441        else
442        {
443            Map<String, Object> params = Map.of("contentTypes", List.of(getContentType()));
444            ContentType contentType = _contentTypeEP.getExtension(getContentType());
445            for (String metadataPath : remoteValues.keySet())
446            {
447                hasChanges = _synchronizeMetadata(content, contentType, metadataPath, metadataPath, remoteValues.get(metadataPath), getLocalAndExternalFields(params).contains(metadataPath), create, logger) || hasChanges;
448            }
449        }
450        
451        return hasChanges;
452    }
453    
454    /**
455     * Validates a content after import
456     * @param content The content to validate
457     * @param logger The logger
458     */
459    protected void validateContent(WorkflowAwareContent content, Logger logger)
460    {
461        if (validateAfterImport())
462        {
463            validateContent(content, getValidateActionId(), logger);
464        }
465    }
466    
467    /**
468     * Get the value of metadata holding the unique identifier of the synchronized content
469     * @param content The content
470     * @return The value
471     */
472    protected String _getIdFieldValue(DefaultContent content)
473    {
474        ModifiableCompositeMetadata metadataHolder = _getMetadataHolder(content.getMetadataHolder(), getIdField());
475        
476        String[] pathSegments = getIdField().split("/");
477        String metadataName = pathSegments[pathSegments.length - 1];
478        
479        return metadataHolder.getString(metadataName, null);
480    }
481    
482    @Override
483    public Map<String, Map<String, Object>> search(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger)
484    {
485        // Search
486        Map<String, Map<String, Object>> results = internalSearch(_removeEmptyParameters(parameters), offset, limit, sort, logger);
487                
488        return results;
489    }
490
491    /**
492     * Search values and return the result without any treatment.
493     * @param parameters Search parameters to restrict the search
494     * @param offset Begin of the search
495     * @param limit Number of results
496     * @param sort Sort of results (ignored for LDAP results)
497     * @param logger The logger
498     * @return Map of results without any treatment.
499     */
500    protected abstract Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger);
501
502    /**
503     * Search values and return the result organized by metadata and transformed by the {@link SynchronizingContentOperator} if exists.
504     * @param parameters Search parameters to restrict the search
505     * @param logger The logger
506     * @return Map of results organized by metadata.
507     */
508    protected Map<String, Map<String, List<Object>>> getTransformedRemoteValues(Map<String, Object> parameters, Logger logger)
509    {
510        Map<String, Map<String, List<Object>>> remoteValues = getRemoteValues(parameters, logger);
511        
512        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
513        if (synchronizingContentOperator != null)
514        {
515            Map<String, Map<String, List<Object>>> transformedRemoteValues = new LinkedHashMap<>();
516            ContentType contentType = _contentTypeEP.getExtension(getContentType());
517            for (String key : remoteValues.keySet())
518            {
519                transformedRemoteValues.put(key, synchronizingContentOperator.transform(contentType, remoteValues.get(key), logger));
520            }
521            
522            return transformedRemoteValues;
523        }
524        else
525        {
526            logger.warn("Cannot find synchronizing content operator with id '{}'. No transformation has applied on remote values", getSynchronizingContentOperator());
527            return remoteValues; // no transformation
528        }
529    }
530    
531    /**
532     * Search values and return the result organized by metadata
533     * @param parameters Search parameters to restrict the search
534     * @param logger The logger
535     * @return Map of results organized by metadata.
536     */
537    protected abstract Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger);
538    
539    @Override
540    public void updateSyncInformations(ModifiableDefaultContent content, String syncCode, Logger logger) throws Exception
541    {
542        updateSCCProperty(content);
543        content.getMetadataHolder().setMetadata(getIdField(), syncCode);
544        content.saveChanges();
545        content.checkpoint();
546    }
547
548    @Override
549    public int getTotalCount(Map<String, Object> parameters, Logger logger)
550    {
551        return search(parameters, 0, Integer.MAX_VALUE, null, logger).size();
552    }
553
554    /**
555     * Import or synchronize several contents from search params.
556     * @param searchParams Search parameters
557     * @param forceImport To force import and ignoring the synchronize existing contents only option
558     * @param logger The logger
559     * @return The {@link List} of imported or synchronized {@link ModifiableDefaultContent}
560     */
561    protected List<ModifiableDefaultContent> _importOrSynchronizeContents(Map<String, Object> searchParams, boolean forceImport, Logger logger)
562    {
563        List<ModifiableDefaultContent> contents = new ArrayList<>();
564        
565        Map<String, Map<String, List<Object>>> remoteValuesByContent = getTransformedRemoteValues(searchParams, logger);
566        for (String idValue : remoteValuesByContent.keySet())
567        {
568            Map<String, List<Object>> remoteValues = remoteValuesByContent.get(idValue);
569            _handleContent(idValue);
570            contents.addAll(_importOrSynchronizeContent(idValue, remoteValues, forceImport, logger));
571        }
572        
573        return contents;
574    }
575    
576    @Override
577    protected List<Content> _getContentsToRemove(AmetysObjectIterable<ModifiableDefaultContent> contents)
578    {
579        return contents.stream()
580            .filter(content -> !_isHandled(_getIdFieldValue(content)))
581            .collect(Collectors.toList());
582    }
583}