Source code

001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.synchronize;
017
018import java.util.ArrayList;
019import java.util.Collections;
020import java.util.HashMap;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.stream.Collectors;
025
026import org.apache.avalon.framework.configuration.Configuration;
027import org.apache.avalon.framework.configuration.ConfigurationException;
028import org.apache.avalon.framework.service.ServiceException;
029import org.apache.avalon.framework.service.ServiceManager;
030import org.apache.commons.lang3.StringUtils;
031import org.slf4j.Logger;
032
033import org.ametys.cms.ObservationConstants;
034import org.ametys.cms.content.ContentHelper;
035import org.ametys.cms.contenttype.ContentType;
036import org.ametys.cms.repository.Content;
037import org.ametys.cms.repository.DefaultContent;
038import org.ametys.cms.repository.ModifiableDefaultContent;
039import org.ametys.cms.repository.WorkflowAwareContent;
040import org.ametys.core.observation.Event;
041import org.ametys.plugins.repository.AmetysObjectIterable;
042import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
043
044/**
045 * Abstract implementation of {@link SynchronizableContentsCollection}.
046 */
047public abstract class AbstractSimpleSynchronizableContentsCollection extends AbstractSynchronizableContentsCollection
048{
049    /** The extension point for Synchronizing Content Operators */
050    protected SynchronizingContentOperatorExtensionPoint _synchronizingContentOperatorEP;
051    
052    /** The content helper */
053    protected ContentHelper _contentHelper;
054    
055    private List<String> _handledContents;
056    
057    @Override
058    public void service(ServiceManager manager) throws ServiceException
059    {
060        super.service(manager);
061        _contentHelper = (ContentHelper) manager.lookup(ContentHelper.ROLE);
062        _synchronizingContentOperatorEP = (SynchronizingContentOperatorExtensionPoint) manager.lookup(SynchronizingContentOperatorExtensionPoint.ROLE);
063    }
064    
065    @Override
066    public void configure(Configuration configuration) throws ConfigurationException
067    {
068        super.configure(configuration);
069        _handledContents = new ArrayList<>();
070    }
071
072    @Override
073    public List<ModifiableDefaultContent> populate(Logger logger)
074    {
075        _handledContents.clear();
076        List<ModifiableDefaultContent> populatedContents = super.populate(logger);
077        _handledContents.clear();
078        return populatedContents;
079    }
080    
081    @Override
082    protected List<ModifiableDefaultContent> _internalPopulate(Logger logger)
083    {
084        return _importOrSynchronizeContents(new HashMap<>(), false, logger);
085    }
086    
087    /**
088     * Adds the given content as handled (i.e. will not be removed if _removalSync is true) 
089     * @param id The id of the content
090     */
091    protected void _handleContent(String id)
092    {
093        _handledContents.add(id);
094    }
095    
096    /**
097     * Returns true if the given content is handled
098     * @param id The content to test
099     * @return true if the given content is handled
100     */
101    protected boolean _isHandled(String id)
102    {
103        return _handledContents.contains(id);
104    }
105
106    /**
107     * Imports or synchronizes a content for each available language
108     * @param idValue The unique identifier of the content
109     * @param remoteValues The remote values
110     * @param forceImport To force import and ignoring the synchronize existing contents only option
111     * @param logger The logger
112     * @return The list of synchronized or imported contents
113     */
114    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
115    {
116        List<ModifiableDefaultContent> contents = new ArrayList<>();
117
118        for (String lang : getLanguages())
119        {
120            contents.addAll(_importOrSynchronizeContent(idValue, lang, remoteValues, forceImport, logger));
121        }
122        
123        return contents;
124    }
125    
126    /**
127     * Imports or synchronizes a content for a given language
128     * @param idValue The unique identifier of the content
129     * @param lang The language of content to import or synchronize
130     * @param remoteValues The remote values
131     * @param forceImport To force import and ignoring the synchronize existing contents only option
132     * @param logger The logger
133     * @return The list of imported and synchronized contents
134     */
135    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, String lang, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
136    {
137        List<ModifiableDefaultContent> contents = new ArrayList<>();
138        
139        try
140        {
141            ModifiableDefaultContent content = getContent(lang, idValue);
142            if (content != null)
143            {
144                contents.add(_synchronizeContent(content, remoteValues, logger));
145            }
146            else if (forceImport || !synchronizeExistingContentsOnly())
147            {
148                contents.add(_importContent(idValue, null, lang, remoteValues, logger));
149            }
150        }
151        catch (Exception e)
152        {
153            _nbError++;
154            logger.error("An error occurred while importing or synchronizing content", e);
155        }
156        
157        return contents;
158    }
159
160    @Override
161    public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception
162    {
163        String idValue = content.getMetadataHolder().getString(getIdField());
164        
165        Map<String, Object> parameters = putIdParameter(idValue);
166        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
167        if (!results.isEmpty())
168        {
169            try
170            {
171                _synchronizeContent(content, results.get(idValue), logger);
172            }
173            catch (Exception e)
174            {
175                _nbError++;
176                logger.error("An error occurred while importing or synchronizing content", e);
177                throw e;
178            }
179        }
180        else
181        {
182            logger.warn("The content {} ({}) with synchronization code '{}' doesn't exist anymore in the datasource from SCC '{}'", content.getTitle(), content.getId(), idValue, getId());
183        }
184    }
185
186    /**
187     * Synchronize a content with remove values.
188     * @param content The content to synchronize
189     * @param remoteValues Values to synchronize
190     * @param logger The logger
191     * @return The synchronized content
192     * @throws Exception if an error occurs.
193     */
194    protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
195    {
196        long startTime = System.currentTimeMillis();
197        
198        String contentTitle = content.getTitle();
199        String lang = content.getLanguage();
200        
201        // Update content
202        logger.info("Start synchronizing content '{}' for language {}", contentTitle, lang);
203
204        _ensureTitleIsPresent(content, remoteValues, logger);
205        
206        boolean hasChanges = _fillContent(remoteValues, content, false, logger);
207        hasChanges = additionalSynchronizeOperations(content, remoteValues, logger) || hasChanges;
208        
209        if (hasChanges)
210        {
211            boolean success = applyChanges(content, logger);
212            if (success)
213            {
214                _nbSynchronizedContents++;
215                logger.info("Some changes were detected for content '{}' and language {}", contentTitle, lang);
216            }
217        }
218        else
219        {
220            _nbNotChangedContents++;
221            logger.info("No changes detected for content '{}' and language {}", contentTitle, lang);
222        }
223        
224        // Do additional operation on the content
225        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
226        if (synchronizingContentOperator != null)
227        {
228            synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
229        }
230        else
231        {
232            logger.warn("Cannot find synchronizing content operator with id '{}'. No additional operation has been done.", getSynchronizingContentOperator());
233        }
234        
235        long endTime = System.currentTimeMillis();
236        logger.info("End synchronization of content '{}' for language {} in {} ms", contentTitle, lang, endTime - startTime);
237        
238        return content;
239    }
240    
241    @Override
242    public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception
243    {
244        List<ModifiableDefaultContent> createdContents = new ArrayList<>();
245        
246        Map<String, Object> parameters = putIdParameter(idValue);
247        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
248        if (!results.isEmpty())
249        {
250            for (String lang : getLanguages())
251            {
252                try
253                {
254                    createdContents.add(_importContent(idValue, importParams, lang, results.get(idValue), logger));
255                }
256                catch (Exception e)
257                {
258                    _nbError++;
259                    logger.error("An error occurred while importing or synchronizing content", e);
260                }
261            }
262        }
263        
264        return createdContents;
265    }
266    
267    /**
268     * Set search parameters for the ID value.
269     * @param idValue Value to search
270     * @return Map with the search parameters
271     */
272    protected abstract Map<String, Object> putIdParameter(String idValue);
273
274    /**
275     * Import a content from remote values.
276     * @param idValue Id (for import/synchronization) of the content to import
277     * @param importParams Specific parameters for import
278     * @param lang Lang of the content
279     * @param remoteValues Values of the content
280     * @param logger The logger
281     * @return The content created by import, or null
282     * @throws Exception if an error occurs.
283     */
284    protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
285    {
286        ModifiableDefaultContent content = getContent(lang, idValue);
287        if (content != null)
288        {
289            logger.warn("The content '{}' for language {} already exists and cannot be imported ", idValue, lang);
290        }
291        else
292        {
293            long startTime = System.currentTimeMillis();
294
295            // Calculate contentTitle
296            String contentTitle = idValue;
297            if (remoteValues.containsKey("title"))
298            {
299                List<Object> remoteTitles = remoteValues.get("title");
300                contentTitle = (String) remoteTitles.stream().filter(obj -> obj instanceof String && StringUtils.isNotEmpty((String) obj)).findFirst().orElse(idValue);
301            }
302            
303            // Create new content
304            logger.info("Start importing content '{}' for language {}", contentTitle, lang);
305    
306            content = createContentAction(lang, contentTitle, logger);
307            if (content != null)
308            {
309                // Synchronize content metadata
310                _fillContent(remoteValues, content, true, logger);
311                updateSCCProperty(content);
312                
313                additionalImportOperations(content, remoteValues, importParams, logger);
314                
315                content.saveChanges();
316                content.checkpoint();
317                
318                // Validate content if allowed
319                validateContent(content, logger);
320                
321                _nbCreatedContents++;
322
323                // Do additional operation on the content
324                SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
325                synchronizingContentOperator.additionalOperation(content, remoteValues, logger);
326                
327                // Notify a content was imported
328                Map<String, Object> eventParams = new HashMap<>();
329                eventParams.put(ObservationConstants.ARGS_CONTENT, content);
330                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId());
331                _observationManager.notify(new Event(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, _currentUserProvider.getUser(), eventParams));
332                
333                long endTime = System.currentTimeMillis();
334                logger.info("End import of content '{}' for language {} in {} ms", content.getId(), lang, endTime - startTime);
335            }
336        }
337        
338        return content;
339    }
340    
341    /**
342     * Add specific fields to the content.
343     * @param content Content to update
344     * @param remoteValues Values of the content
345     * @param importParams Import parameters
346     * @param logger The logger
347     * @return <code>true</code> if there are changes
348     */
349    protected boolean additionalImportOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger)
350    {
351        return additionalCommonOperations(content, remoteValues, importParams, true, logger);
352    }
353
354    /**
355     * Add specific fields to the content.
356     * @param content Content to update
357     * @param remoteValues Values of the content
358     * @param logger The logger
359     * @return <code>true</code> if there are changes
360     */
361    protected boolean additionalSynchronizeOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger)
362    {
363        return additionalCommonOperations(content, remoteValues, Collections.EMPTY_MAP, false, logger);
364    }
365    
366    /**
367     * Add specific fields to the content during import or synchronization.
368     * @param content Content to update
369     * @param remoteValues Values of the content
370     * @param importParams the import params
371     * @param create true if the content has been newly created
372     * @param logger The logger
373     * @return <code>true</code> if there are changes
374     */
375    protected boolean additionalCommonOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, boolean create, Logger logger)
376    {
377        // Do nothing by default
378        return false;
379    }
380    
381    private void _ensureTitleIsPresent(Content content, Map<String, List<Object>> remoteValues, Logger logger)
382    {
383        if (remoteValues.containsKey("title"))
384        {
385            List<Object> titleValues = remoteValues.get("title");
386            boolean atLeastOneTitle = titleValues.stream()
387                                                 .filter(String.class::isInstance)
388                                                 .map(String.class::cast)
389                                                 .anyMatch(StringUtils::isNotBlank);
390            if (atLeastOneTitle)
391            {
392                return;
393            }
394        }
395        
396        // Force to current title
397        logger.warn("The remote value of 'title' is empty for the content {}. The 'title' metadata is mandatory, the current title will remain.", content);
398        remoteValues.put("title", Collections.singletonList(content.getTitle()));
399    }
400
401    @Override
402    public ModifiableDefaultContent getContent(String lang, String idValue)
403    {
404        String query = _getContentPathQuery(lang, idValue, getContentType());
405        AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(query);
406        
407        if (contents.getSize() > 0)
408        {
409            return contents.iterator().next();
410        }
411        return null;
412    }
413    
414    /**
415     * Creates content action with result from request
416     * @param lang The language
417     * @param contentTitle The content title
418     * @param logger The logger
419     * @return The content id, or null of a workflow error occured
420     */
421    protected ModifiableDefaultContent createContentAction(String lang, String contentTitle, Logger logger)
422    {
423        return createContentAction(getContentType(), getWorkflowName(), getInitialActionId(), lang, contentTitle, logger);
424    }
425    
426    /**
427     * Fill the content with remote values.
428     * @param remoteValues The remote values
429     * @param content The content to synchronize
430     * @param create <code>true</code> if content is creating, false if it is updated
431     * @param logger The logger
432     * @return <code>true</code> if changes were made
433     */
434    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
435    {
436        boolean hasChanges = false;
437        
438        if (content.isLocked())
439        {
440            logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner());
441        }
442        else
443        {
444            Map<String, Object> params = Map.of("contentTypes", List.of(getContentType()));
445            ContentType contentType = _contentTypeEP.getExtension(getContentType());
446            for (String metadataPath : remoteValues.keySet())
447            {
448                hasChanges = _synchronizeMetadata(content, contentType, metadataPath, metadataPath, remoteValues.get(metadataPath), getLocalAndExternalFields(params).contains(metadataPath), create, logger) || hasChanges;
449            }
450        }
451        
452        return hasChanges;
453    }
454    
455    /**
456     * Validates a content after import
457     * @param content The content to validate
458     * @param logger The logger
459     */
460    protected void validateContent(WorkflowAwareContent content, Logger logger)
461    {
462        if (validateAfterImport())
463        {
464            validateContent(content, getValidateActionId(), logger);
465        }
466    }
467    
468    /**
469     * Get the value of metadata holding the unique identifier of the synchronized content
470     * @param content The content
471     * @return The value
472     */
473    protected String _getIdFieldValue(DefaultContent content)
474    {
475        ModifiableCompositeMetadata metadataHolder = _getMetadataHolder(content.getMetadataHolder(), getIdField());
476        
477        String[] pathSegments = getIdField().split("/");
478        String metadataName = pathSegments[pathSegments.length - 1];
479        
480        return metadataHolder.getString(metadataName, null);
481    }
482    
483    @Override
484    public Map<String, Map<String, Object>> search(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger)
485    {
486        // Search
487        Map<String, Map<String, Object>> results = internalSearch(_removeEmptyParameters(parameters), offset, limit, sort, logger);
488                
489        return results;
490    }
491
492    /**
493     * Search values and return the result without any treatment.
494     * @param parameters Search parameters to restrict the search
495     * @param offset Begin of the search
496     * @param limit Number of results
497     * @param sort Sort of results (ignored for LDAP results)
498     * @param logger The logger
499     * @return Map of results without any treatment.
500     */
501    protected abstract Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger);
502
503    /**
504     * Search values and return the result organized by metadata and transformed by the {@link SynchronizingContentOperator} if exists.
505     * @param parameters Search parameters to restrict the search
506     * @param logger The logger
507     * @return Map of results organized by metadata.
508     */
509    protected Map<String, Map<String, List<Object>>> getTransformedRemoteValues(Map<String, Object> parameters, Logger logger)
510    {
511        Map<String, Map<String, List<Object>>> remoteValues = getRemoteValues(parameters, logger);
512        
513        SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator());
514        if (synchronizingContentOperator != null)
515        {
516            Map<String, Map<String, List<Object>>> transformedRemoteValues = new LinkedHashMap<>();
517            ContentType contentType = _contentTypeEP.getExtension(getContentType());
518            for (String key : remoteValues.keySet())
519            {
520                transformedRemoteValues.put(key, synchronizingContentOperator.transform(contentType, remoteValues.get(key), logger));
521            }
522            
523            return transformedRemoteValues;
524        }
525        else
526        {
527            logger.warn("Cannot find synchronizing content operator with id '{}'. No transformation has applied on remote values", getSynchronizingContentOperator());
528            return remoteValues; // no transformation
529        }
530    }
531    
532    /**
533     * Search values and return the result organized by metadata
534     * @param parameters Search parameters to restrict the search
535     * @param logger The logger
536     * @return Map of results organized by metadata.
537     */
538    protected abstract Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger);
539    
540    @Override
541    public void updateSyncInformations(ModifiableDefaultContent content, String syncCode, Logger logger) throws Exception
542    {
543        updateSCCProperty(content);
544        content.getMetadataHolder().setMetadata(getIdField(), syncCode);
545        content.saveChanges();
546        content.checkpoint();
547    }
548
549    @Override
550    public int getTotalCount(Map<String, Object> parameters, Logger logger)
551    {
552        return search(parameters, 0, Integer.MAX_VALUE, null, logger).size();
553    }
554
555    /**
556     * Import or synchronize several contents from search params.
557     * @param searchParams Search parameters
558     * @param forceImport To force import and ignoring the synchronize existing contents only option
559     * @param logger The logger
560     * @return The {@link List} of imported or synchronized {@link ModifiableDefaultContent}
561     */
562    protected List<ModifiableDefaultContent> _importOrSynchronizeContents(Map<String, Object> searchParams, boolean forceImport, Logger logger)
563    {
564        List<ModifiableDefaultContent> contents = new ArrayList<>();
565        
566        Map<String, Map<String, List<Object>>> remoteValuesByContent = getTransformedRemoteValues(searchParams, logger);
567        for (String idValue : remoteValuesByContent.keySet())
568        {
569            Map<String, List<Object>> remoteValues = remoteValuesByContent.get(idValue);
570            _handleContent(idValue);
571            contents.addAll(_importOrSynchronizeContent(idValue, remoteValues, forceImport, logger));
572        }
573        
574        return contents;
575    }
576    
577    @Override
578    protected List<Content> _getContentsToRemove(AmetysObjectIterable<ModifiableDefaultContent> contents)
579    {
580        return contents.stream()
581            .filter(content -> !_isHandled(_getIdFieldValue(content)))
582            .collect(Collectors.toList());
583    }
584}