001/*
002 *  Copyright 2016 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.contentio.synchronize;
017
018import java.io.IOException;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.HashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.stream.Collectors;
025
026import org.apache.avalon.framework.service.ServiceException;
027import org.apache.avalon.framework.service.ServiceManager;
028import org.apache.commons.lang3.StringUtils;
029import org.slf4j.Logger;
030
031import org.ametys.cms.repository.Content;
032import org.ametys.cms.repository.ContentQueryHelper;
033import org.ametys.cms.repository.ContentTypeExpression;
034import org.ametys.cms.repository.LanguageExpression;
035import org.ametys.cms.repository.ModifiableContent;
036import org.ametys.cms.repository.WorkflowAwareContent;
037import org.ametys.cms.workflow.ContentWorkflowHelper;
038import org.ametys.core.observation.Event;
039import org.ametys.core.observation.ObservationManager;
040import org.ametys.core.schedule.progression.ContainerProgressionTracker;
041import org.ametys.core.schedule.progression.ProgressionTrackerFactory;
042import org.ametys.core.schedule.progression.SimpleProgressionTracker;
043import org.ametys.core.user.CurrentUserProvider;
044import org.ametys.core.util.HttpUtils;
045import org.ametys.core.util.I18nUtils;
046import org.ametys.core.util.language.UserLanguagesManager;
047import org.ametys.core.util.mail.SendMailHelper;
048import org.ametys.plugins.repository.AmetysObjectIterable;
049import org.ametys.plugins.repository.query.expression.AndExpression;
050import org.ametys.plugins.repository.query.expression.Expression;
051import org.ametys.plugins.repository.query.expression.Expression.Operator;
052import org.ametys.plugins.repository.query.expression.OrExpression;
053import org.ametys.plugins.repository.query.expression.StringExpression;
054import org.ametys.plugins.workflow.AbstractWorkflowComponent;
055import org.ametys.plugins.workflow.AbstractWorkflowComponent.ConditionFailure;
056import org.ametys.runtime.config.Config;
057import org.ametys.runtime.i18n.I18nizableText;
058
059import com.opensymphony.workflow.InvalidActionException;
060import com.opensymphony.workflow.WorkflowException;
061
062import jakarta.mail.MessagingException;
063
064/**
065 * Abstract implementation of {@link SynchronizableContentsCollection}.
066 */
067public abstract class AbstractSynchronizableContentsCollection extends AbstractStaticSynchronizableContentsCollection
068{
069    /** SCC unique ID field */
070    protected static final String SCC_UNIQUE_ID = "scc$uniqueid";
071    
072    /** The i18n utils */
073    protected I18nUtils _i18nUtils;
074    /** The current user provider */
075    protected CurrentUserProvider _currentUserProvider;
076    /** The observation manager */
077    protected ObservationManager _observationManager;
078    /** The content workflow helper */
079    protected ContentWorkflowHelper _contentWorkflowHelper;
080    /** The user languages manager */
081    protected UserLanguagesManager _userLanguagesManager;
082    
083    /** Number of errors encountered */
084    protected int _nbError;
085    /** True if there is a global error during synchronization */
086    protected boolean _hasGlobalError;
087    
088    /** Number of created contents */
089    protected int _nbCreatedContents;
090    /** Number of synchronized contents */
091    protected int _nbSynchronizedContents;
092    /** Number of unchanged contents */
093    protected int _nbNotChangedContents;
094    /** Number of deleted contents */
095    protected int _nbDeletedContents;
096    
097    @Override
098    public void service(ServiceManager manager) throws ServiceException
099    {
100        super.service(manager);
101        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
102        _currentUserProvider = (CurrentUserProvider) manager.lookup(CurrentUserProvider.ROLE);
103        _observationManager = (ObservationManager) manager.lookup(ObservationManager.ROLE);
104        _contentWorkflowHelper = (ContentWorkflowHelper) manager.lookup(ContentWorkflowHelper.ROLE);
105        _userLanguagesManager = (UserLanguagesManager) manager.lookup(UserLanguagesManager.ROLE);
106    }
107    
108    @Override
109    public List<ModifiableContent> populate(Logger logger, ContainerProgressionTracker progressionTracker)
110    {
111        ContainerProgressionTracker internalPopulaitePT =  progressionTracker.addContainerStep("internalpopulate", new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_SCHEDULER_SYNCHRONIZE_COLLECTION_IMPORT_SYNCHRONIZE_CONTENTS_STEP_LABEL"), 2);
112        
113        SimpleProgressionTracker deleteUnexistingContentsPT = progressionTracker.addSimpleStep("deleteunexistingcontents", new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_SCHEDULER_SYNCHRONIZE_COLLECTION_DELETE_UNEXISTING_CONTENTS_STEP_LABEL"));
114        
115        SimpleProgressionTracker notifyPT = progressionTracker.addSimpleStep("notify", new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_SCHEDULER_SYNCHRONIZE_COLLECTION_NOTIFY_OBSERVERS_AND_SEND_MAILS_STEP_LABEL"));
116        
117        _nbCreatedContents = 0;
118        _nbSynchronizedContents = 0;
119        _nbNotChangedContents = 0;
120        _nbDeletedContents = 0;
121        _nbError = 0;
122        _hasGlobalError = false;
123        
124        logger.info("Start synchronization of collection '{}'", getId());
125        List<Long> times = new ArrayList<>();
126        times.add(System.currentTimeMillis());
127        
128        // Do populate
129        List<ModifiableContent> populatedContents = _internalPopulate(logger, internalPopulaitePT);
130        
131        if (!_hasGlobalError && removalSync())
132        {
133            // Delete old contents if source prevails
134            deleteUnexistingContents(logger);
135        }
136        deleteUnexistingContentsPT.increment();
137        
138        times.add(System.currentTimeMillis());
139        logger.info("[Synchronization of collection '{}'] Populated in {} ms", getId(), times.get(times.size() - 1) - times.get(times.size() - 2));
140        _logSynchronizationResult(logger);
141        
142        notifyPT.setSize(2);
143        
144        if (_hasSomethingChanged())
145        {
146            // Do not notify obeservers if there is no change
147            Map<String, Object> eventParams = new HashMap<>();
148            eventParams.put(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.ARGS_COLLECTION_ID, this.getId());
149            eventParams.put(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.ARGS_COLLECTION_CONTENT_TYPE, this.getContentType());
150            _observationManager.notify(new Event(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_COLLECTION_SYNCHRONIZED, _currentUserProvider.getUser(), eventParams));
151            
152            times.add(System.currentTimeMillis());
153            logger.info("[Synchronization of collection '{}'] Listeners notified in {} ms", getId(), times.get(times.size() - 1) - times.get(times.size() - 2));
154            
155        }
156        notifyPT.increment();
157        
158        if (_nbError > 0 && getReportMails().length() > 0)
159        {
160            try
161            {
162                logger.warn("{} contents were not created/updated because of an error.", _nbError);
163                sendErrorMail(_nbError);
164                
165                times.add(System.currentTimeMillis());
166                logger.info("[Synchronization of collection '{}'] Error mail sent in {} ms", getId(), times.get(times.size() - 1) - times.get(times.size() - 2));
167            }
168            catch (MessagingException | IOException e)
169            {
170                logger.warn("Unable to send mail", e);
171            }
172        }
173        notifyPT.increment();
174        
175        logger.info("[Synchronization of collection '{}'] Total in {} ms", getId(), times.get(times.size() - 1) - times.get(0));
176
177        return populatedContents;
178    }
179    
180    /**
181     * Internal implementation of {@link #populate(Logger, ContainerProgressionTracker)}
182     * @param logger The logger
183     * @return The list of created/synchronized contents
184     */
185    protected List<ModifiableContent> _internalPopulate(Logger logger)
186    {
187        return _internalPopulate(logger, ProgressionTrackerFactory.createContainerProgressionTracker("Internal populate", logger));
188    }
189    
190    /**
191     * Internal implementation of {@link #populate(Logger, ContainerProgressionTracker)}
192     * @param logger The logger
193     * @param progressionTracker The progression tracker
194     * @return The list of created/synchronized contents
195     */
196    protected abstract List<ModifiableContent> _internalPopulate(Logger logger, ContainerProgressionTracker progressionTracker);
197    
198    @Override
199    public void empty(Logger logger)
200    {
201        // Get all contents from the SCC
202        Expression collectionExpression = _sccHelper.getCollectionExpression(getId());
203        String xPathQuery = ContentQueryHelper.getContentXPathQuery(collectionExpression);
204        List<ModifiableContent> contentsToRemove = _resolver.<ModifiableContent>query(xPathQuery)
205            .stream()
206            .toList();
207        // Process to the deletion
208        _removeSCCOrDeleteContents(contentsToRemove, logger);
209    }
210    
211    /**
212     * Delete contents created by a previous synchronization which does not exist anymore in remote source.
213     * If the content belongs to several SCC, only the current SCC reference will be deleted.
214     * @param logger The logger
215     */
216    protected void deleteUnexistingContents(Logger logger)
217    {
218        String query = _getContentPathQuery(null, null, null, true);
219        AmetysObjectIterable<ModifiableContent> contents = _resolver.query(query);
220        
221        List<ModifiableContent> contentsToRemove = _getContentsToRemove(contents);
222
223        if (!contentsToRemove.isEmpty())
224        {
225            if (logger.isInfoEnabled())
226            {
227                contentsToRemove.stream().forEach(content -> logger.info("The content '{}' ({}) does not exist anymore in remote source: it will be deleted if possible.", content.getTitle(), content.getId()));
228            }
229            
230            _nbDeletedContents = _removeSCCOrDeleteContents(contentsToRemove, logger);
231        }
232    }
233    
234    /**
235     * Filter the contents to remove.
236     * @param contents The list of all the available contents
237     * @return The {@link List} of {@link Content} to remove.
238     */
239    protected abstract List<ModifiableContent> _getContentsToRemove(AmetysObjectIterable<ModifiableContent> contents);
240    
241    /**
242     * For each content, if the content has only this SCC, try to delete it, but if it has several SCC, remove the current collection from the SCC property.
243     * @param contents The contents to delete or to remove the SCC
244     * @param logger The logger
245     * @return the number of really deleted contents
246     */
247    protected int _removeSCCOrDeleteContents(List<ModifiableContent> contents, Logger logger)
248    {
249        logger.info("Remove SCC on contents with multiple SCC... {} contents for all the deletion process.", contents.size());
250        List<Content> contentsToReallyDelete = new ArrayList<>();
251        for (ModifiableContent content : contents)
252        {
253            // There is more than one SCC, only remove the SCC reference
254            if (_sccHelper.getSynchronizableCollectionIds(content).size() > 1)
255            {
256                _sccHelper.removeSCCProperty(content, getId());
257                content.saveChanges();
258            }
259            // It was the only SCC, completely remove the content
260            else
261            {
262                contentsToReallyDelete.add(content);
263            }
264        }
265
266        logger.info("Remove contents with single SCC... {} contents remaining.", contentsToReallyDelete.size());
267        int nbDeletedContents = _deleteContents(contentsToReallyDelete, logger);
268        logger.info("Contents deleting process ended. {} contents has been deleted.", nbDeletedContents);
269        
270        return nbDeletedContents;
271    }
272    
273    /**
274     * Delete contents.
275     * @param contentsToRemove List of contents to remove
276     * @param logger The logger
277     * @return the number of deleted contents
278     */
279    protected int _deleteContents(List<Content> contentsToRemove, Logger logger)
280    {
281        return _contentDAO.forceDeleteContentsWithLog(contentsToRemove, null, logger);
282    }
283    
284    /**
285     * Logs the result of the synchronization, containing
286     * <ul>
287     * <li>The number of created contents</li>
288     * <li>The number of synchronized contents</li>
289     * <li>The number of unchanged contents</li>
290     * <li>The number of deleted contents</li>
291     * </ul>
292     * @param logger the logger
293     */
294    protected void _logSynchronizationResult(Logger logger)
295    {
296        logger.info("{} contents were created", _nbCreatedContents);
297        logger.info("{} contents were updated", _nbSynchronizedContents);
298        logger.info("{} contents did not changed", _nbNotChangedContents);
299        logger.info("{} contents were deleted", _nbDeletedContents);
300    }
301    
302    /**
303     * Checks if some content have changed during the synchronization
304     * @return <code>true</code> if some contents have changed, <code>false</code> otherwise
305     */
306    protected boolean _hasSomethingChanged()
307    {
308        return _nbCreatedContents > 0 || _nbSynchronizedContents > 0 || _nbDeletedContents > 0;
309    }
310    
311    /**
312     * Sends the report mails
313     * @param nbError The number of error
314     * @throws MessagingException if a messaging error occurred
315     * @throws IOException if an error occurred building the message
316     */
317    protected void sendErrorMail(int nbError) throws MessagingException, IOException
318    {
319        String language = _userLanguagesManager.getDefaultLanguage();
320        
321        String pluginName = "plugin.contentio";
322        List<String> params = new ArrayList<>();
323        params.add(getId());
324        String subject = _i18nUtils.translate(new I18nizableText(pluginName, "PLUGINS_CONTENTIO_POPULATE_REPORT_MAIL_SUBJECT", params), language);
325        
326        params.clear();
327        params.add(String.valueOf(nbError));
328        params.add(getId());
329        String baseUrl = HttpUtils.sanitize(Config.getInstance().getValue("cms.url"));
330        params.add(baseUrl + "/_admin/index.html?uitool=uitool-admin-logs");
331        String body = _i18nUtils.translate(new I18nizableText(pluginName, "PLUGINS_CONTENTIO_POPULATE_REPORT_MAIL_BODY", params), language);
332        
333        SendMailHelper.newMail()
334                      .withSubject(subject)
335                      .withTextBody(body)
336                      .withRecipients(Arrays.asList(getReportMails().split("\\n")))
337                      .sendMail();
338    }
339    
340    /**
341     * Validates a content after import
342     * @param content The content to validate
343     * @param validationActionId Validation action ID to use for this content
344     * @param logger The logger
345     */
346    protected void validateContent(WorkflowAwareContent content, int validationActionId, Logger logger)
347    {
348        Map<String, Object> inputs = new HashMap<>();
349        
350        try
351        {
352            _contentWorkflowHelper.doAction(content, validationActionId, inputs);
353            logger.info("The content {} has been validated after import", content);
354        }
355        catch (WorkflowException | InvalidActionException e)
356        {
357            String failuresAsString = _getActionFailuresAsString(inputs);
358            logger.error("The content {} cannot be validated after import{}", content, failuresAsString, e);
359        }
360    }
361    
362    private String _getActionFailuresAsString(Map<String, Object> actionInputs)
363    {
364        String failuresAsString = "";
365        if (actionInputs.containsKey(AbstractWorkflowComponent.FAIL_CONDITIONS_KEY))
366        {
367            @SuppressWarnings("unchecked")
368            List<ConditionFailure> failures = (List<ConditionFailure>) actionInputs.get(AbstractWorkflowComponent.FAIL_CONDITIONS_KEY);
369            if (!failures.isEmpty())
370            {
371                failuresAsString = ", due to the following error(s):\n" + String.join("\n", failures.stream().map(ConditionFailure::text).toList());
372            }
373        }
374        
375        return failuresAsString;
376    }
377    
378    /**
379     * Creates content action with result from request
380     * @param contentType Type of the content to create
381     * @param workflowName Workflow to use for this content
382     * @param initialActionId Action ID for initialization
383     * @param lang The language
384     * @param contentTitle The content title
385     * @param logger The logger
386     * @return The content id, or null of a workflow error occured
387     */
388    protected ModifiableContent createContentAction(String contentType, String workflowName, int initialActionId, String lang, String contentTitle, Logger logger)
389    {
390        try
391        {
392            logger.info("Creating content '{}' with the content type '{}' for language {}", contentTitle, getContentType(), lang);
393            String desiredContentName = _contentPrefix + "-" + contentTitle + "-" + lang;
394            
395            Map<String, Object> inputs = _getAdditionalInputsForContentCreation();
396            Map<String, Object> result = _contentWorkflowHelper.createContent(
397                    workflowName,
398                    initialActionId,
399                    desiredContentName,
400                    contentTitle,
401                    new String[] {contentType},
402                    null,
403                    lang,
404                    inputs);
405            
406            return (ModifiableContent) result.get(Content.class.getName());
407        }
408        catch (WorkflowException e)
409        {
410            _nbError++;
411            logger.error("Failed to initialize workflow for content {} and language {}", contentTitle, lang, e);
412            return null;
413        }
414    }
415    
416    /**
417     * Retrieves additional inputs for content creation
418     * @return the additional inputs for content creation
419     */
420    protected Map<String, Object> _getAdditionalInputsForContentCreation()
421    {
422        // no additional inputs by default
423        return new HashMap<>();
424    }
425
426    /**
427     * Construct the query to retrieve the content.
428     * @param lang Lang
429     * @param idValue Synchronization value
430     * @param contentType Content type
431     * @param forceStrictCheck <code>true</code> to force strict mode to check the collection, otherwise it read the "checkCollection" option
432     * @return The {@link List} of {@link Expression}
433     */
434    protected List<Expression> _getExpressionsList(String lang, String idValue, String contentType, boolean forceStrictCheck)
435    {
436        List<Expression> expList = new ArrayList<>();
437        
438        if (forceStrictCheck)
439        {
440            expList.add(_sccHelper.getCollectionExpression(getId()));
441        }
442        else if (checkCollection())
443        {
444            expList.add(
445                getCompatibleSCC(true).stream()
446                    .map(_sccHelper::getCollectionExpression)
447                    .collect(Collectors.toCollection(OrExpression::new))
448            );
449        }
450        
451        if (StringUtils.isNotBlank(contentType))
452        {
453            expList.add(new ContentTypeExpression(Operator.EQ, contentType));
454        }
455        
456        if (StringUtils.isNotBlank(idValue))
457        {
458            expList.add(new StringExpression(getIdField(), Operator.EQ, idValue));
459        }
460        
461        if (StringUtils.isNotBlank(lang))
462        {
463            expList.add(new LanguageExpression(Operator.EQ, lang));
464        }
465        
466        return expList;
467    }
468    
469    /**
470     * Construct the query to retrieve the content.
471     * @param lang Lang
472     * @param idValue Synchronization value
473     * @param contentType Content type
474     * @param forceStrictCheck <code>true</code> to force strict mode to check the collection, otherwise it read the "checkCollection" option
475     * @return The XPATH query
476     */
477    protected String _getContentPathQuery(String lang, String idValue, String contentType, boolean forceStrictCheck)
478    {
479        List<Expression> expList = _getExpressionsList(lang, idValue, contentType, forceStrictCheck);
480        AndExpression andExp = new AndExpression(expList);
481        return ContentQueryHelper.getContentXPathQuery(andExp);
482    }
483    
484    /**
485     * Remove empty parameters to the map
486     * @param searchParameters the parameters
487     * @return the map of none empty parameters
488     */
489    protected Map<String, Object> _removeEmptyParameters(Map<String, Object> searchParameters)
490    {
491        Map<String, Object> result = new HashMap<>();
492        for (String parameterName : searchParameters.keySet())
493        {
494            Object parameterValue = searchParameters.get(parameterName);
495            if (_isParamNotEmpty(parameterValue))
496            {
497                result.put(parameterName, parameterValue);
498            }
499        }
500        
501        return result;
502    }
503    
504    /**
505     * Check if the parameter value is empty
506     * @param parameterValue the parameter value
507     * @return true if the parameter value is empty
508     */
509    protected boolean _isParamNotEmpty(Object parameterValue)
510    {
511        return parameterValue != null && !(parameterValue instanceof String && StringUtils.isBlank((String) parameterValue));
512    }
513    
514    public Map<String, Integer> getSynchronizationResult()
515    {
516        Map<String, Integer> result = new HashMap<>();
517        
518        result.put(RESULT_NB_CREATED_CONTENTS, _nbCreatedContents);
519        result.put(RESULT_NB_SYNCHRONIZED_CONTENTS, _nbSynchronizedContents);
520        result.put(RESULT_NB_NOT_CHANGED_CONTENTS, _nbNotChangedContents);
521        result.put(RESULT_NB_DELETED_CONTENTS, _nbDeletedContents);
522        
523        return result;
524    }
525}