001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.odfsync.cdmfr.components;
017
018import java.io.ByteArrayInputStream;
019import java.io.File;
020import java.io.FileInputStream;
021import java.io.IOException;
022import java.io.InputStream;
023import java.net.URL;
024import java.text.ParseException;
025import java.text.SimpleDateFormat;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Date;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.LinkedList;
032import java.util.List;
033import java.util.Map;
034import java.util.Objects;
035import java.util.Set;
036
037import javax.jcr.RepositoryException;
038import javax.xml.transform.TransformerException;
039
040import org.apache.avalon.framework.activity.Initializable;
041import org.apache.avalon.framework.component.Component;
042import org.apache.avalon.framework.configuration.Configurable;
043import org.apache.avalon.framework.configuration.Configuration;
044import org.apache.avalon.framework.configuration.ConfigurationException;
045import org.apache.avalon.framework.configuration.DefaultConfigurationBuilder;
046import org.apache.avalon.framework.context.ContextException;
047import org.apache.avalon.framework.context.Contextualizable;
048import org.apache.avalon.framework.service.ServiceException;
049import org.apache.avalon.framework.service.ServiceManager;
050import org.apache.avalon.framework.service.Serviceable;
051import org.apache.cocoon.Constants;
052import org.apache.cocoon.ProcessingException;
053import org.apache.cocoon.environment.Context;
054import org.apache.commons.collections4.CollectionUtils;
055import org.apache.commons.io.IOUtils;
056import org.apache.commons.lang3.StringUtils;
057import org.apache.excalibur.xml.dom.DOMParser;
058import org.apache.excalibur.xml.xpath.PrefixResolver;
059import org.apache.excalibur.xml.xpath.XPathProcessor;
060import org.slf4j.Logger;
061import org.w3c.dom.Document;
062import org.w3c.dom.Element;
063import org.w3c.dom.NamedNodeMap;
064import org.w3c.dom.Node;
065import org.w3c.dom.NodeList;
066import org.xml.sax.InputSource;
067import org.xml.sax.SAXException;
068
069import org.ametys.cms.ObservationConstants;
070import org.ametys.cms.content.external.ExternalizableMetadataHelper;
071import org.ametys.cms.content.external.ExternalizableMetadataProvider.ExternalizableMetadataStatus;
072import org.ametys.cms.contenttype.ContentType;
073import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
074import org.ametys.cms.contenttype.MetadataDefinition;
075import org.ametys.cms.contenttype.MetadataType;
076import org.ametys.cms.contenttype.RepeaterDefinition;
077import org.ametys.cms.repository.ContentQueryHelper;
078import org.ametys.cms.repository.ContentTypeExpression;
079import org.ametys.cms.repository.LanguageExpression;
080import org.ametys.cms.repository.ModifiableContent;
081import org.ametys.cms.repository.ModifiableDefaultContent;
082import org.ametys.cms.repository.comment.DefaultCommentManagerExtensionPoint;
083import org.ametys.core.observation.Event;
084import org.ametys.core.observation.ObservationManager;
085import org.ametys.core.user.CurrentUserProvider;
086import org.ametys.core.user.population.UserPopulationDAO;
087import org.ametys.core.util.URIUtils;
088import org.ametys.odf.ProgramItem;
089import org.ametys.odf.catalog.CatalogsManager;
090import org.ametys.odf.course.Course;
091import org.ametys.odf.course.CourseFactory;
092import org.ametys.odf.course.ShareableCourseHelper;
093import org.ametys.odf.courselist.CourseList;
094import org.ametys.odf.courselist.CourseListFactory;
095import org.ametys.odf.coursepart.CoursePart;
096import org.ametys.odf.coursepart.CoursePartFactory;
097import org.ametys.odf.enumeration.OdfReferenceTableEntry;
098import org.ametys.odf.enumeration.OdfReferenceTableHelper;
099import org.ametys.odf.observation.OdfObservationConstants;
100import org.ametys.odf.orgunit.OrgUnit;
101import org.ametys.odf.orgunit.OrgUnitFactory;
102import org.ametys.odf.orgunit.RootOrgUnitProvider;
103import org.ametys.odf.person.PersonFactory;
104import org.ametys.odf.program.ContainerFactory;
105import org.ametys.odf.program.ProgramFactory;
106import org.ametys.odf.program.ProgramPart;
107import org.ametys.odf.program.SubProgramFactory;
108import org.ametys.odf.program.TraversableProgramPart;
109import org.ametys.odf.translation.TranslationHelper;
110import org.ametys.plugins.contentio.ContentImporterHelper;
111import org.ametys.plugins.contentio.synchronize.BaseSynchroComponent;
112import org.ametys.plugins.odfsync.cdmfr.CDMFrSyncExtensionPoint;
113import org.ametys.plugins.odfsync.cdmfr.transformers.CDMFrSyncTransformer;
114import org.ametys.plugins.repository.AmetysObjectIterable;
115import org.ametys.plugins.repository.AmetysObjectResolver;
116import org.ametys.plugins.repository.AmetysRepositoryException;
117import org.ametys.plugins.repository.metadata.CompositeMetadata;
118import org.ametys.plugins.repository.metadata.ModifiableBinaryMetadata;
119import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
120import org.ametys.plugins.repository.metadata.ModifiableFile;
121import org.ametys.plugins.repository.metadata.ModifiableResource;
122import org.ametys.plugins.repository.metadata.ModifiableRichText;
123import org.ametys.plugins.repository.query.expression.AndExpression;
124import org.ametys.plugins.repository.query.expression.Expression;
125import org.ametys.plugins.repository.query.expression.Expression.Operator;
126import org.ametys.plugins.repository.query.expression.StringExpression;
127import org.ametys.runtime.config.Config;
128
129/**
130 * Abstract class of a component to import a CDM-fr input stream.
131 */
132public abstract class AbstractImportCDMFrComponent implements ImportCDMFrComponent, Serviceable, Initializable, Contextualizable, Configurable, Component
133{
134    /** Tag to identify a program */
135    protected static final String _TAG_PROGRAM = "program";
136
137    /** Tag to identify a subprogram */
138    protected static final String _TAG_SUBPROGRAM = "subProgram";
139    
140    /** Tag to identify a container */
141    protected static final String _TAG_CONTAINER = "container";
142    
143    /** Tag to identify a courseList */
144    protected static final String _TAG_COURSELIST = "coursesReferences";
145    
146    /** Tag to identify a coursePart */
147    protected static final String _TAG_COURSEPART = "coursePart";
148
149    /** The synchronize workflow action id */
150    private static final int SYNCHRONIZE_WORKFLOW_ACTION_ID = 800;
151    
152    private static final ContentWorkflowDescription _PROGRAM_WF_DESCRIPTION = new ContentWorkflowDescription(ProgramFactory.PROGRAM_CONTENT_TYPE, "program", 1, 4);
153    private static final ContentWorkflowDescription _SUBPROGRAM_WF_DESCRIPTION = new ContentWorkflowDescription(SubProgramFactory.SUBPROGRAM_CONTENT_TYPE, "subprogram", 1, 4);
154    private static final ContentWorkflowDescription _CONTAINER_WF_DESCRIPTION = new ContentWorkflowDescription(ContainerFactory.CONTAINER_CONTENT_TYPE, "container", 1, 4);
155    private static final ContentWorkflowDescription _COURSELIST_WF_DESCRIPTION = new ContentWorkflowDescription(CourseListFactory.COURSE_LIST_CONTENT_TYPE, "courselist", 1, 4);
156    private static final ContentWorkflowDescription _COURSE_WF_DESCRIPTION = new ContentWorkflowDescription(CourseFactory.COURSE_CONTENT_TYPE, "course", 1, 4);
157    private static final ContentWorkflowDescription _COURSEPART_WF_DESCRIPTION = new ContentWorkflowDescription(CoursePartFactory.COURSE_PART_CONTENT_TYPE, "course-part", 1, -1);
158    private static final ContentWorkflowDescription _ORGUNIT_WF_DESCRIPTION = new ContentWorkflowDescription(OrgUnitFactory.ORGUNIT_CONTENT_TYPE, "orgunit", 1, 4);
159    private static final ContentWorkflowDescription _PERSON_WF_DESCRIPTION = new ContentWorkflowDescription(PersonFactory.PERSON_CONTENT_TYPE, "person", 1, 4);
160
161    private static final PrefixResolver _PREFIX_RESOLVER = new DocbookPrefixResolver();
162    
163    /** The Cocoon context */
164    protected Context _cocoonContext;
165    
166    /** The DOM parser */
167    protected DOMParser _domParser;
168
169    /** The XPath processor */
170    protected XPathProcessor _xPathProcessor;
171
172    /** Extension point to transform CDM-fr */
173    protected CDMFrSyncExtensionPoint _cdmFrSyncExtensionPoint;
174
175    /** Default language configured for ODF */
176    protected String _odfLang;
177    
178    /** The catalog manager */
179    protected CatalogsManager _catalogsManager;
180
181    /** The ametys object resolver */
182    protected AmetysObjectResolver _resolver;
183
184    /** The ODF TableRef Helper */
185    protected OdfReferenceTableHelper _odfRefTableHelper;
186
187    /** The content type extension point */
188    protected ContentTypeExtensionPoint _contentTypeEP;
189
190    /** The current user provider */
191    protected CurrentUserProvider _currentUserProvider;
192
193    /** The observation manager */
194    protected ObservationManager _observationManager;
195    
196    /** The root orgunit provider */
197    protected RootOrgUnitProvider _rootOUProvider;
198
199    /** The base SCC component */
200    protected BaseSynchroComponent _synchroComponent;
201    
202    /** The shareable course helper */
203    protected ShareableCourseHelper _shareableCourseHelper;
204    
205    /** List of synchronized contents */
206    protected Map<String, Integer> _importedContents;
207    
208    /** List of synchronized contents having differences */
209    protected Set<String> _synchronizedContents;
210    
211    /** List of synchronized contents (to avoid a treatment twice or more) */
212    protected Set<String> _updatedContents;
213
214    /** Number of errors encountered */
215    protected int _nbError;
216    /** Number of created contents */
217    protected int _nbCreatedContents;
218    /** Number of synchronized contents */
219    protected int _nbSynchronizedContents;
220    /** Number of unchanged contents */
221    protected int _nbNotChangedContents;
222    /** The prefix of the contents */
223    protected String _contentPrefix;
224    /** Synchronized fields by content type */
225    protected Map<String, Set<String>> _syncFieldsByContentType;
226
227    public void initialize() throws Exception
228    {
229        _odfLang = Config.getInstance().getValue("odf.programs.lang");
230    }
231
232    @Override
233    public void contextualize(org.apache.avalon.framework.context.Context context) throws ContextException
234    {
235        _cocoonContext = (Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
236    }
237
238    public void configure(Configuration configuration) throws ConfigurationException
239    {
240        _parseSynchronizedFields();
241    }
242    
243    @Override
244    public void service(ServiceManager manager) throws ServiceException
245    {
246        _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
247        _domParser = (DOMParser) manager.lookup(DOMParser.ROLE);
248        _xPathProcessor = (XPathProcessor) manager.lookup(XPathProcessor.ROLE);
249        _cdmFrSyncExtensionPoint = (CDMFrSyncExtensionPoint) manager.lookup(CDMFrSyncExtensionPoint.ROLE);
250        _catalogsManager = (CatalogsManager) manager.lookup(CatalogsManager.ROLE);
251        _odfRefTableHelper = (OdfReferenceTableHelper) manager.lookup(OdfReferenceTableHelper.ROLE);
252        _contentTypeEP = (ContentTypeExtensionPoint) manager.lookup(ContentTypeExtensionPoint.ROLE);
253        _currentUserProvider = (CurrentUserProvider) manager.lookup(CurrentUserProvider.ROLE);
254        _observationManager = (ObservationManager) manager.lookup(ObservationManager.ROLE);
255        _rootOUProvider = (RootOrgUnitProvider) manager.lookup(RootOrgUnitProvider.ROLE);
256        _synchroComponent = (BaseSynchroComponent) manager.lookup(BaseSynchroComponent.ROLE);
257        _shareableCourseHelper = (ShareableCourseHelper) manager.lookup(ShareableCourseHelper.ROLE);
258    }
259
260    @Override
261    public String getIdField()
262    {
263        return "cdmfrSyncCode";
264    }
265
266    /**
267     * Get the synchronized metadata from the configuration file
268     * @throws ConfigurationException if the configuration is not valid.
269     */
270    private void _parseSynchronizedFields() throws ConfigurationException
271    {
272        _syncFieldsByContentType = new HashMap<>();
273        
274        @SuppressWarnings("resource")
275        InputStream is = null;
276        try
277        {
278            File apogeeMapping = new File(_cocoonContext.getRealPath("/WEB-INF/param/odf-synchro.xml"));
279            if (!apogeeMapping.isFile())
280            {
281                is = getClass().getResourceAsStream("/org/ametys/plugins/odfsync/cdmfr/odf-synchro.xml");
282            }
283            else
284            {
285                is = new FileInputStream(apogeeMapping);
286            }
287            Configuration cfg = new DefaultConfigurationBuilder().build(is);
288
289            Configuration[] cTypesConf = cfg.getChildren("content-type");
290            for (Configuration cTypeConf : cTypesConf)
291            {
292                String contentType = cTypeConf.getAttribute("id");
293                Set<String> syncMetadata = _configureSynchronizedFields(cTypeConf, "");
294                _syncFieldsByContentType.put(contentType, syncMetadata);
295            }
296        }
297        catch (Exception e)
298        {
299            throw new ConfigurationException("Error while parsing odf-synchro.xml", e);
300        }
301        finally
302        {
303            IOUtils.closeQuietly(is);
304        }
305    }
306
307    private Set<String> _configureSynchronizedFields(Configuration configuration, String prefix) throws ConfigurationException
308    {
309        Set<String> syncMetadata = new HashSet<>();
310        Configuration[] metaConf = configuration.getChildren("metadata");
311        
312        if (metaConf.length > 0)
313        {
314            for (Configuration metadata : metaConf)
315            {
316                if (metadata.getChildren("metadata").length > 0)
317                {
318                    // composite
319                    syncMetadata.addAll(_configureSynchronizedFields(metadata, prefix + metadata.getAttribute("name") + "/"));
320                }
321                else
322                {
323                    syncMetadata.add(prefix + metadata.getAttribute("name"));
324                }
325            }
326        }
327        else if (configuration.getAttribute("name", null) != null)
328        {
329            syncMetadata.add(prefix + configuration.getAttribute("name"));
330        }
331        
332        return syncMetadata;
333    }
334    
335    @Override
336    @SuppressWarnings("unchecked")
337    public synchronized Map<String, Object> handleInputStream(InputStream input, Map<String, Object> parameters, Logger logger) throws ProcessingException
338    {
339        List<ModifiableDefaultContent> importedPrograms = new ArrayList<>();
340        
341        _importedContents = (Map<String, Integer>) parameters.getOrDefault("importedContents", new HashMap<>());
342        _synchronizedContents = (Set<String>) parameters.getOrDefault("synchronizedContents", new HashSet<>());
343        _updatedContents = (Set<String>) parameters.getOrDefault("updatedContents", new HashSet<>());
344        _nbCreatedContents = (int) parameters.getOrDefault("nbCreatedContents", 0);
345        _nbSynchronizedContents = (int) parameters.getOrDefault("nbSynchronizedContents", 0);
346        _nbNotChangedContents = (int) parameters.getOrDefault("nbNotChangedContents", 0);
347        _nbError = (int) parameters.getOrDefault("nbError", 0);
348        _contentPrefix = (String) parameters.getOrDefault("contentPrefix", "cdmfr-");
349        additionalParameters(parameters);
350        
351        Map<String, Object> resultMap = new HashMap<>();
352        
353        try
354        {
355            Document doc = _domParser.parseDocument(new InputSource(input));
356            doc = transformDocument(doc, new HashMap<String, Object>(), logger);
357
358            if (doc != null)
359            {
360                String defaultLang = _getXPathString(doc, "CDM/@language", _odfLang);
361                
362                NodeList nodes = doc.getElementsByTagName(_TAG_PROGRAM);
363                
364                for (int i = 0; i < nodes.getLength(); i++)
365                {
366                    Node contentNode = nodes.item(i);
367                    String syncCode = _xPathProcessor.evaluateAsString(contentNode, "@CDMid");
368                    String contentLang = _getXPathString(contentNode, "@language", defaultLang);
369                    contentLang = StringUtils.substring(contentLang, 0, 2).toLowerCase(); // on keep the language from the locale
370                    
371                    String catalog = getCatalogName(contentNode);
372
373                    importedPrograms.add(_importOrSynchronizeContent(doc, contentNode, getProgramWfDescription(), syncCode, contentLang, catalog, syncCode, logger));
374                }
375                
376                // Apply changes (synchronize action)
377                boolean ignoreRights = ignoreRights();
378                for (String contentId : _synchronizedContents)
379                {
380                    ModifiableDefaultContent content = _resolver.resolveById(contentId);
381                    _synchroComponent.applyChanges(content, SYNCHRONIZE_WORKFLOW_ACTION_ID, org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, ignoreRights, logger);
382                }
383                
384                // Validate contents
385                if (validateAfterImport())
386                {
387                    for (String contentId : _importedContents.keySet())
388                    {
389                        ModifiableDefaultContent content = _resolver.resolveById(contentId);
390                        Integer validationActionId = _importedContents.get(contentId);
391                        if (validationActionId > 0)
392                        {
393                            _synchroComponent.validateContent(content, validationActionId, ignoreRights, logger);
394                        }
395                    }
396                }
397            }
398        }
399        catch (IOException | ProcessingException e)
400        {
401            throw new ProcessingException("An error occured while transforming the stream.", e);
402        }
403        catch (SAXException e)
404        {
405            throw new ProcessingException("An error occured while parsing the stream.", e);
406        }
407        catch (RepositoryException e)
408        {
409            throw new ProcessingException("An error occured while applying changes on contents.", e);
410        }
411
412        resultMap.put("importedContents", _importedContents);
413        resultMap.put("synchronizedContents", _synchronizedContents);
414        resultMap.put("updatedContents", _updatedContents);
415        resultMap.put("nbCreatedContents", _nbCreatedContents);
416        resultMap.put("nbSynchronizedContents", _nbSynchronizedContents);
417        resultMap.put("nbNotChangedContents", _nbNotChangedContents);
418        resultMap.put("nbError", _nbError);
419        resultMap.put("importedPrograms", importedPrograms);
420
421        return resultMap;
422    }
423
424    /**
425     * True to validate the contents after import
426     * @return True to validate the contents after import
427     */
428    protected abstract boolean validateAfterImport();
429
430    /**
431     * When returns true, a content created by a previous synchro will be removed if it does not exist anymore during the current synchro.
432     * @return true if a content created by a previous synchro has to be removed if it does not exist anymore during the current synchro.
433     */
434    protected abstract boolean removalSync();
435    
436    /**
437     * Transform the document depending of it structure.
438     * @param document Document to transform.
439     * @param parameters Optional parameters for transformation
440     * @param logger The logger
441     * @return The transformed document.
442     * @throws IOException if an error occurs.
443     * @throws SAXException if an error occurs.
444     * @throws ProcessingException if an error occurs.
445     */
446    protected Document transformDocument(Document document, Map<String, Object> parameters, Logger logger) throws IOException, SAXException, ProcessingException
447    {
448        CDMFrSyncTransformer transformer = _cdmFrSyncExtensionPoint.getTransformer(document);
449        if (transformer == null)
450        {
451            logger.error("Cannot match a CDM-fr transformer to this file structure.");
452            return null;
453        }
454        
455        return transformer.transform(document, parameters);
456    }
457
458    /**
459     * Get the name of catalog to use for import
460     * @param contentNode The node of program
461     * @return The catalog to used
462     */
463    protected String getCatalogName(Node contentNode)
464    {
465        String defaultCatalog = _catalogsManager.getDefaultCatalogName();
466        
467        String contentCatalog = _getXPathString(contentNode, "catalog", defaultCatalog);
468        if (_catalogsManager.getCatalog(contentCatalog) == null)
469        {
470            // Catalog is empty or do not exist, use the default catalog
471            return defaultCatalog;
472        }
473        
474        return contentCatalog;
475    }
476    
477    /**
478     * Get or create the content from the synchronization code, the lang, the catalog and the content type.
479     * @param title The title
480     * @param lang The lang
481     * @param catalog The catalog
482     * @param syncCode The synchronization code
483     * @param wfDescription The workflow description
484     * @param logger The logger
485     * @return the retrieved or created content
486     * @throws RepositoryException if an error occurs
487     */
488    protected ModifiableDefaultContent _getOrCreateContent(String title, String lang, String catalog, String syncCode, ContentWorkflowDescription wfDescription, Logger logger) throws RepositoryException
489    {
490        ModifiableDefaultContent receivedContent = _getContent(lang, catalog, syncCode, wfDescription);
491        if (receivedContent != null)
492        {
493            return receivedContent;
494        }
495        
496        Map<String, Object> resultMap = _synchroComponent.createContentAction(wfDescription.getContentType(), wfDescription.getWorkflowName(), wfDescription.getInitialActionId(), lang, title, _contentPrefix, logger);
497        if ((boolean) resultMap.getOrDefault("error", false))
498        {
499            _nbError++;
500        }
501        
502        ModifiableDefaultContent content = (ModifiableDefaultContent) resultMap.get("content");
503        
504        if (content != null)
505        {
506            ExternalizableMetadataHelper.setMetadata(content.getMetadataHolder(), getIdField(), syncCode);
507            if (catalog != null && (content instanceof ProgramItem || content instanceof CoursePart))
508            {
509                ExternalizableMetadataHelper.setMetadata(content.getMetadataHolder(), ProgramItem.CATALOG, catalog);
510            }
511            additionalOperationsBeforeSave(content, logger);
512            content.saveChanges();
513            _importedContents.put(content.getId(), wfDescription.getValidationActionId());
514            _nbCreatedContents++;
515        }
516        return content;
517    }
518
519    /**
520     * Get the content from the synchronization code, the lang, the catalog and the content type.
521     * @param lang The lang
522     * @param catalog The catalog
523     * @param syncCode The synchronization code
524     * @param wfDescription The workflow description
525     * @return the retrieved content
526     */
527    protected ModifiableDefaultContent _getContent(String lang, String catalog, String syncCode, ContentWorkflowDescription wfDescription)
528    {
529        List<Expression> expList = getExpressionsList(lang, syncCode, wfDescription.getContentType(), catalog);
530        AndExpression andExp = new AndExpression(expList.toArray(new Expression[expList.size()]));
531        String xPathQuery = ContentQueryHelper.getContentXPathQuery(andExp);
532
533        AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(xPathQuery);
534
535        if (contents.getSize() > 0)
536        {
537            return contents.iterator().next();
538        }
539        
540        return null;
541    }
542    
543    /**
544     * Additional parameters for specific treatments.
545     * @param parameters The parameters map to get
546     */
547    protected abstract void additionalParameters(Map<String, Object> parameters);
548    
549    /**
550     * Additional operation to do on the content before saving it.
551     * @param content The content
552     * @param logger The logger
553     * @throws RepositoryException if an error occurs
554     */
555    protected abstract void additionalOperationsBeforeSave(ModifiableDefaultContent content, Logger logger) throws RepositoryException;
556    
557    /**
558     * Import or synchronize the content.
559     * @param doc XML document
560     * @param contentNode Node of the content
561     * @param wfDescription The workflow description
562     * @param title The title
563     * @param lang The lang
564     * @param catalog The catalog
565     * @param syncCode The synchronization code
566     * @param logger The logger
567     * @return The imported or synchronized content
568     */
569    protected ModifiableDefaultContent _importOrSynchronizeContent(Document doc, Node contentNode, ContentWorkflowDescription wfDescription, String title, String lang, String catalog, String syncCode, Logger logger)
570    {
571        ModifiableDefaultContent content = null;
572        try
573        {
574            content = _getOrCreateContent(title, lang, catalog, syncCode, wfDescription, logger);
575            if (content != null)
576            {
577                _synchronizeContent(doc, contentNode, content, wfDescription.getContentType(), lang, catalog, syncCode, logger);
578            }
579        }
580        catch (RepositoryException e)
581        {
582            _nbError++;
583            logger.error("An error occurred while importing or synchronizing content", e);
584        }
585        return content;
586    }
587    
588    /**
589     * Synchronize content 
590     * @param doc The root document
591     * @param contentNode the DOM content node
592     * @param content The content to synchronize
593     * @param contentTypeId The content type ID
594     * @param lang Parent program language (to select the good courses in the CDM-FR file)
595     * @param catalog The catalog of parent program
596     * @param syncCode The synchronization code
597     * @param logger The logger
598     */
599    protected void _synchronizeContent(Document doc, Node contentNode, ModifiableDefaultContent content, String contentTypeId, String lang, String catalog, String syncCode, Logger logger)
600    {
601        if (_updatedContents.add(content.getId()))
602        {
603            logger.info("Synchronization of the content '{}' with the content type '{}'", content.getTitle(), contentTypeId);
604            
605            List<ModifiableDefaultContent> children = new LinkedList<>();
606            List<ModifiableDefaultContent> courseParts = new LinkedList<>();
607            boolean removeOldCourseParts = false;
608            
609            if (content.isLocked())
610            {
611                logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner());
612            }
613            else
614            {
615                boolean hasChanges = false;
616                ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
617                int courseListPosition = 0;
618                
619                NodeList metadataNodes = contentNode.getChildNodes();
620                for (int i = 0; i < metadataNodes.getLength(); i++)
621                {
622                    Node metadataNode = metadataNodes.item(i);
623                    String metadataName = metadataNode.getLocalName();
624                    
625                    if (metadataName.equals(_TAG_SUBPROGRAM))
626                    {
627                        String subContentSyncCode = _xPathProcessor.evaluateAsString(metadataNode, "@CDMid");
628                        String title = _xPathProcessor.evaluateAsString(metadataNode, "title");
629                        ModifiableDefaultContent subProgram = _importOrSynchronizeContent(doc, metadataNode, getSubProgramWfDescription(), title, lang, catalog, subContentSyncCode, logger);
630                        CollectionUtils.addIgnoreNull(children, subProgram);
631                    }
632                    else if (metadataName.equals(_TAG_CONTAINER))
633                    {
634                        String subContentSyncCode = _xPathProcessor.evaluateAsString(metadataNode, "@CDMid");
635                        String title = _xPathProcessor.evaluateAsString(metadataNode, "title");
636                        ModifiableDefaultContent container = _importOrSynchronizeContent(doc, metadataNode, getContainerWfDescription(), title, lang, catalog, subContentSyncCode, logger);
637                        CollectionUtils.addIgnoreNull(children, container);
638                    }
639                    else if (metadataName.equals(_TAG_COURSELIST))
640                    {
641                        courseListPosition++;
642                        // For courseList from another source than Ametys, there is no unique code, then a code is generated with the parent ID and the position in the parent : [parentId]-[position]
643                        String subContentSyncCode = _getXPathString(metadataNode, "@code", syncCode + "-" + courseListPosition);
644                        String title = _getXPathString(metadataNode, "@name", "Liste d'éléments pédagogiques");
645                        ModifiableDefaultContent courseList = _importOrSynchronizeContent(doc, metadataNode, getCourseListWfDescription(), title, lang, catalog, subContentSyncCode, logger);
646                        if (courseList != null)
647                        {
648                            _synchronizeCourseList(doc, metadataNode, courseList, lang, catalog, logger);
649                            children.add(courseList);
650                        }
651                    }
652                    else if (metadataName.equals(_TAG_COURSEPART))
653                    {
654                        String subContentSyncCode = _xPathProcessor.evaluateAsString(metadataNode, "code");
655                        if (StringUtils.isEmpty(subContentSyncCode))
656                        {
657                            subContentSyncCode = org.ametys.core.util.StringUtils.generateKey().toUpperCase();
658                            removeOldCourseParts = true;
659                        }
660                        String title = _xPathProcessor.evaluateAsString(metadataNode, "title");
661                        ModifiableDefaultContent coursePart = _importOrSynchronizeContent(doc, metadataNode, getCoursePartWfDescription(), title, lang, catalog, subContentSyncCode, logger);
662                        CollectionUtils.addIgnoreNull(courseParts, coursePart);
663                    }
664                    // Explicitely ignore catalog metadata which is forced at content creation
665                    else if (!metadataName.equals(ProgramItem.CATALOG))
666                    {
667                        hasChanges = _synchronizeMetadata(doc, metadataNode, content, metadataName, metadataName, contentType, lang, catalog, logger) || hasChanges;
668                    }
669                }
670                
671                hasChanges = _setRelations(content, children, courseParts, removeOldCourseParts, logger) || hasChanges;
672                
673                // Create translation links
674                _linkTranslationsIfExist(content, contentTypeId);
675                
676                _saveContentChanges(content, contentTypeId, hasChanges, logger);
677            }
678        }
679    }
680
681    /**
682     * Synchronize a course list, it has attributes to synchronize.
683     * @param doc The XML document
684     * @param courseListNode The XML node of the course list
685     * @param courseList The course list content
686     * @param lang The lang
687     * @param catalog The catalog
688     * @param logger The logger
689     */
690    protected void _synchronizeCourseList(Document doc, Node courseListNode, ModifiableDefaultContent courseList, String lang, String catalog, Logger logger)
691    {
692        boolean hasChanges = false;
693        
694        ContentType contentType = _contentTypeEP.getExtension(getCourseListWfDescription().getContentType());
695        NamedNodeMap attributes = courseListNode.getAttributes();
696        for (int i = 0; i < attributes.getLength(); i++)
697        {
698            Node attributeNode = attributes.item(i);
699            String attributeName = attributeNode.getLocalName();
700            // Explicitely ignore catalog metadata which is forced at content creation
701            if (!attributeName.equals(ProgramItem.CATALOG))
702            {
703                hasChanges = _synchronizeMetadata(doc, attributeNode, courseList, attributeName, attributeName, contentType, lang, catalog, logger) || hasChanges;
704            }
705        }
706
707        List<ModifiableDefaultContent> courses = new LinkedList<>();
708        NodeList itemNodes = courseListNode.getChildNodes();
709        for (int i = 0; i < itemNodes.getLength(); i++)
710        {
711            String syncCode = itemNodes.item(i).getTextContent().trim();
712            
713            Node courseNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "course[@CDMid = '" + syncCode + "' and @language = '" + lang + "']");
714            if (courseNode == null)
715            {
716                courseNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "course[@CDMid = '" + syncCode + "']");
717            }
718            
719            if (courseNode != null)
720            {
721                String elpLang = _getXPathString(courseNode, "@language", lang);
722
723                // Check the catalog is the same as the parent program
724                String courseCatalog = getCatalogName(courseNode);
725                if (courseCatalog != null && !courseCatalog.equals(catalog))
726                {
727                    String cdmCode = _xPathProcessor.evaluateAsString(courseNode, "cdmCode");
728                    logger.error("The course '{}' belongs to a different catalog than the one from the imported/synchronized program : '{}' vs '{}'. No synchronization will be done on this course.", cdmCode, courseCatalog, catalog);
729                }
730                else
731                {
732                    String title = _xPathProcessor.evaluateAsString(courseNode, "title");
733                    ModifiableDefaultContent course = _importOrSynchronizeContent(doc, courseNode, getCourseWfDescription(), title, elpLang, catalog, syncCode, logger);
734                    if (course != null)
735                    {
736                        courses.add(course);
737                        hasChanges = _shareableCourseHelper.initializeShareableFields((Course) course, (CourseList) courseList,  UserPopulationDAO.SYSTEM_USER_IDENTITY, true) || hasChanges;
738                    }
739                }
740            }
741        }
742        
743        if (!courses.isEmpty())
744        {
745            hasChanges = ExternalizableMetadataHelper.setMetadata(courseList.getMetadataHolder(), CourseList.CHILD_COURSES, courses.toArray(new ModifiableDefaultContent[courses.size()])) || hasChanges;
746            for (ModifiableDefaultContent course : courses)
747            {
748                hasChanges = _synchroComponent.updateRelation(course.getMetadataHolder(), Course.PARENT_COURSE_LISTS, courseList, false) || hasChanges;
749            }
750        }
751        
752        _saveContentChanges(courseList, getCourseListWfDescription().getContentType(), hasChanges, logger);
753    }
754
755    private void _fetchImages(Node docbookNode, ModifiableDefaultContent content, ModifiableRichText richText, String metadataPath, Logger logger)
756    {
757        NodeList unfetchUrls = _xPathProcessor.selectNodeList(docbookNode, ".//docbook:mediaobject/docbook:imageobject/docbook:imagedata", _PREFIX_RESOLVER);
758        
759        for (int i = 0; i < unfetchUrls.getLength(); i++)
760        {
761            Element href = (Element) unfetchUrls.item(i);
762            String imageUrl = href.getAttribute("fileref");
763            
764            if (DefaultCommentManagerExtensionPoint.URL_VALIDATOR.matcher(imageUrl).matches())
765            {
766                @SuppressWarnings("resource")
767                InputStream is = null;
768                try
769                {
770                    URL url = new URL(imageUrl);
771                    is = url.openStream();
772                    
773                    String path = url.getPath();
774                    String fileName = path.substring(path.lastIndexOf("/") + 1);
775                    
776                    ModifiableFile file = null;
777                    if (richText.getAdditionalDataFolder().hasFile(fileName))
778                    {
779                        file = richText.getAdditionalDataFolder().getFile(fileName);
780                    }
781                    else
782                    {
783                        file = richText.getAdditionalDataFolder().addFile(fileName); 
784                    }
785                    
786                    ModifiableResource resource = file.getResource();
787                    resource.setInputStream(is);
788                    resource.setLastModified(new Date());
789                    
790                    String mimeType = _cocoonContext.getMimeType(fileName);
791                    if (mimeType == null)
792                    {
793                        mimeType = "application/unknown";
794                    }
795                    resource.setMimeType(mimeType);
796                    
797                    String fetchUrl = content.getId() + "@" + metadataPath + ";" + fileName;
798                    href.setAttribute("fileref", fetchUrl);
799                }
800                catch (Exception e)
801                {
802                    logger.warn("Unable to retrieve remote image '{}'.", imageUrl, e);
803                }
804                finally
805                {
806                    IOUtils.closeQuietly(is);
807                }
808            }
809        }
810    }
811    
812    /**
813     * Synchronize a metadata (can be a composite or a repeater).
814     * @param doc The XML document
815     * @param metadataNode The metadata node
816     * @param content The content
817     * @param logicalMetadataPath The logical metadata path (to retrieve the definition)
818     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
819     * @param contentType The content type
820     * @param lang The lang
821     * @param catalog The catalog
822     * @param logger The logger
823     * @return <code>true</code> if changes occurs
824     */
825    protected boolean _synchronizeMetadata(Document doc, Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, ContentType contentType, String lang, String catalog, Logger logger)
826    {
827        List<String> untypedValues = null;
828        MetadataDefinition metadataDef = contentType.getMetadataDefinitionByPath(logicalMetadataPath);
829        Map<String, Object> params = Map.of("contentTypes", List.of(contentType.getId()));
830        boolean synchronize = getLocalAndExternalFields(params).contains(logicalMetadataPath);
831        if (metadataDef != null)
832        {
833            if (metadataDef instanceof RepeaterDefinition)
834            {
835                return _handleRepeaterMetadata(doc, metadataNode, content, logicalMetadataPath, completeMetadataPath, contentType, lang, catalog, logger);
836            }
837            else if (metadataDef.getType() == MetadataType.COMPOSITE)
838            {
839                return _handleCompositeMetadata(doc, metadataNode, content, logicalMetadataPath, completeMetadataPath, contentType, lang, catalog, logger);
840            }
841            else if (metadataDef.getType() == MetadataType.RICH_TEXT)
842            {
843                return _handleRichTextMetadata(content, logicalMetadataPath, completeMetadataPath, metadataNode, synchronize, logger);
844            }
845            else if (metadataDef.getType() == MetadataType.BINARY)
846            {
847                return _handleBinaryMetadata(metadataNode, content, completeMetadataPath, synchronize, logger);
848            }
849            else if (metadataDef.getType() == MetadataType.FILE)
850            {
851                return _handleFileMetadata(metadataNode, content, logicalMetadataPath, completeMetadataPath, synchronize, contentType, logger);
852            }
853            else if (metadataDef.getType() == MetadataType.GEOCODE)
854            {
855                return _handleGeocodeMetadata(metadataNode, content, completeMetadataPath, synchronize);
856            }
857            else if (metadataDef.isMultiple())
858            {
859                NodeList itemNodes = metadataNode.getChildNodes();
860                untypedValues = new ArrayList<>();
861                for (int j = 0; j < itemNodes.getLength(); j++)
862                {
863                    String metadataValue = itemNodes.item(j).getTextContent().trim();
864                    if (StringUtils.isNotEmpty(metadataValue))
865                    {
866                        untypedValues.add(metadataValue);
867                    }
868                }
869            }
870            else
871            {
872                String metadataValue = metadataNode.getTextContent().trim();
873                if (StringUtils.isNotEmpty(metadataValue))
874                {
875                    untypedValues = List.of(metadataValue);
876                }
877            }
878        }
879
880        Map<String, Boolean> resultMap = _synchroComponent.synchronizeMetadata(content, contentType, logicalMetadataPath, completeMetadataPath, _handleMetadataValues(doc, content, metadataDef, untypedValues, lang, catalog, logger), synchronize, _importedContents.containsKey(content.getId()), logger);
881        if (resultMap.getOrDefault("error", Boolean.FALSE))
882        {
883            _nbError++;
884        }
885        return resultMap.getOrDefault("hasChanges", Boolean.FALSE).booleanValue();
886    }
887    
888    /**
889     * Synchronize geocode metadata
890     * @param metadataNode The metadata node
891     * @param content The content
892     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
893     * @param synchronize true if the data is external
894     * @return true if some changes has been made
895     */
896    protected boolean _handleGeocodeMetadata(Node metadataNode, ModifiableDefaultContent content, String completeMetadataPath, boolean synchronize)
897    {
898        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
899        String[] arrayPath = completeMetadataPath.split("/");
900        String metadataName = arrayPath[arrayPath.length - 1];
901        
902        return _handleGeocodeMetadata(metadataNode, content, metadataHolder, metadataName, synchronize);
903    }
904    
905    /**
906     * Synchronize geocode metadata
907     * @param metadataNode The metadata node
908     * @param content The content
909     * @param metadataHolder the metadata holder
910     * @param metadataName the metadata name
911     * @param synchronize true if the data is external
912     * @return true if some changes has been made
913     */
914    protected boolean _handleGeocodeMetadata(Node metadataNode, ModifiableDefaultContent content, ModifiableCompositeMetadata metadataHolder, String metadataName, boolean synchronize)
915    {
916        if (!metadataNode.hasChildNodes())
917        {
918            return _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize);
919        }
920
921        boolean hasChanges = false;
922        ExternalizableMetadataStatus status = synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL;
923        ModifiableCompositeMetadata geoCode = ExternalizableMetadataHelper.getCompositeMetadata(metadataHolder, metadataName, status, true);
924        if (synchronize && _importedContents.containsKey(content.getId()))
925        {
926            hasChanges = ExternalizableMetadataHelper.updateStatus(metadataHolder, metadataName, status) || hasChanges;
927        }
928        
929        NodeList coordinates = metadataNode.getChildNodes();
930        for (int j = 0; j < coordinates.getLength(); j++)
931        {
932            Node coordinate = coordinates.item(j);
933            hasChanges = ExternalizableMetadataHelper.setMetadata(geoCode, coordinate.getLocalName(), coordinate.getTextContent().trim()) || hasChanges;
934        }
935        return hasChanges;
936    }
937
938    /**
939     * Synchronize file metadata
940     * @param metadataNode The metadata node
941     * @param content The content
942     * @param logicalMetadataPath The logical metadata path (to retrieve the definition) 
943     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
944     * @param synchronize true if the data is external
945     * @param contentType the content type
946     * @param logger the logger
947     * @return true if some changes has been made
948     */
949    protected boolean _handleFileMetadata(Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, boolean synchronize, ContentType contentType, Logger logger)
950    {
951        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
952        String[] arrayPath = completeMetadataPath.split("/");
953        String metadataName = arrayPath[arrayPath.length - 1];
954        
955        return _handleFileMetadata(metadataNode, content, logicalMetadataPath, metadataHolder, metadataName, synchronize, contentType, logger);
956    }
957    
958    /**
959     * Synchronize file metadata
960     * @param metadataNode The metadata node
961     * @param content The content
962     * @param logicalMetadataPath The logical metadata path (to retrieve the definition) 
963     * @param metadataHolder the metadata holder
964     * @param metadataName the metadata name
965     * @param synchronize true if the data is external
966     * @param contentType the content type
967     * @param logger the logger
968     * @return true if some changes has been made
969     */
970    protected boolean _handleFileMetadata(Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, ModifiableCompositeMetadata metadataHolder, String metadataName, boolean synchronize, ContentType contentType, Logger logger)
971    {
972        boolean hasChanges = false;
973        String value = metadataNode.getTextContent().trim();
974        boolean metadataExists = metadataHolder.hasMetadata(metadataName);
975        CompositeMetadata.MetadataType metadataType = metadataExists ? metadataHolder.getType(metadataName) : null;
976        if (DefaultCommentManagerExtensionPoint.URL_VALIDATOR.matcher(value).matches())
977        {
978            if (metadataExists && metadataType != CompositeMetadata.MetadataType.BINARY)
979            {
980                hasChanges = _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize) || hasChanges;
981            }
982            hasChanges = _handleBinaryMetadata(metadataNode, content, metadataHolder, metadataName, synchronize, logger) || hasChanges;
983        }
984        else
985        {
986            if (metadataExists && metadataType != CompositeMetadata.MetadataType.STRING)
987            {
988                hasChanges = _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize) || hasChanges;
989            }
990            
991            Map<String, Boolean> resultMap = _synchroComponent.synchronizeMetadata(content, contentType, logicalMetadataPath, metadataHolder, metadataName, List.of(value), synchronize, _importedContents.containsKey(content.getId()), logger);
992            if (resultMap.getOrDefault("error", Boolean.FALSE))
993            {
994                _nbError++;
995            }
996            hasChanges = resultMap.getOrDefault("hasChanges", Boolean.FALSE).booleanValue() || hasChanges;
997        }
998        return hasChanges;
999    }
1000    
1001    /**
1002     * Synchronize binary metadata
1003     * @param metadataNode The metadata node
1004     * @param content The content
1005     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
1006     * @param synchronize true if the data is external
1007     * @param logger the logger
1008     * @return true if some changes has been made
1009     */
1010    protected boolean _handleBinaryMetadata(Node metadataNode, ModifiableDefaultContent content, String completeMetadataPath, boolean synchronize, Logger logger)
1011    {
1012        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
1013        String[] arrayPath = completeMetadataPath.split("/");
1014        String metadataName = arrayPath[arrayPath.length - 1];
1015        
1016        return _handleBinaryMetadata(metadataNode, content, metadataHolder, metadataName, synchronize, logger);
1017    }
1018    
1019    /**
1020     * Synchronize binary metadata
1021     * @param metadataNode The metadata node
1022     * @param content The content
1023     * @param metadataHolder the metadata holder
1024     * @param metadataName the metadata name
1025     * @param synchronize true if the data is external
1026     * @param logger the logger
1027     * @return true if some changes has been made
1028     */
1029    protected boolean _handleBinaryMetadata(Node metadataNode, ModifiableDefaultContent content, ModifiableCompositeMetadata metadataHolder, String metadataName, boolean synchronize, Logger logger)
1030    {
1031        boolean hasChanges = false;
1032        
1033        String value = metadataNode.getTextContent().trim();
1034        
1035        if (StringUtils.isEmpty(value))
1036        {
1037            return _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize);
1038        }
1039        
1040        try
1041        {
1042            URL url = new URL(value);
1043            
1044            byte[] bytes;
1045            try (InputStream is = url.openStream())
1046            {
1047                bytes = IOUtils.toByteArray(is);
1048            }
1049
1050            ExternalizableMetadataStatus status = synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL;
1051            ModifiableBinaryMetadata binaryMetadata = ExternalizableMetadataHelper.getBinaryMetadata(metadataHolder, metadataName, status, true);
1052
1053            byte[] oldValue = new byte[0];
1054            try (InputStream is = binaryMetadata.getInputStream())
1055            {
1056                oldValue = IOUtils.toByteArray(is);
1057            }
1058            catch (AmetysRepositoryException e)
1059            {
1060                logger.debug("The old value of '{}' should not be initialized on the content '{}'.", metadataName, content.getId(), e);
1061            }
1062
1063            if (!Objects.deepEquals(bytes, oldValue))
1064            {
1065                String path = url.getPath();
1066                String filename = path.substring(path.lastIndexOf("/") + 1);
1067                String mimeType = _cocoonContext.getMimeType(filename);
1068                if (mimeType == null)
1069                {
1070                    mimeType = "application/unknown";
1071                }
1072                binaryMetadata.setFilename(URIUtils.decode(filename));
1073                binaryMetadata.setMimeType(mimeType);
1074                binaryMetadata.setLastModified(new Date());
1075                binaryMetadata.setInputStream(new ByteArrayInputStream(bytes));
1076    
1077                hasChanges = true;
1078            }
1079
1080            if (synchronize && _importedContents.containsKey(content.getId()))
1081            {
1082                hasChanges = ExternalizableMetadataHelper.updateStatus(metadataHolder, metadataName, status) || hasChanges;
1083            }
1084        }
1085        catch (IOException e)
1086        {
1087            logger.error("Unable to retrieve remote file input stream", e);
1088        }
1089
1090        return hasChanges;
1091    }
1092    
1093    private boolean _handleRepeaterMetadata(Document doc, Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, ContentType contentType, String lang, String catalog, Logger logger)
1094    {
1095        boolean hasChanges = false;
1096        
1097        ModifiableCompositeMetadata repeater;
1098        if (completeMetadataPath.indexOf("/") == -1)
1099        {
1100            repeater = content.getMetadataHolder().getCompositeMetadata(completeMetadataPath, true);
1101        }
1102        else
1103        {
1104            ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
1105            repeater = metadataHolder.getCompositeMetadata(StringUtils.substringAfterLast(completeMetadataPath, "/"), true);
1106        }
1107        
1108        // Remove locale entries (unable to compare locales and remotes ...)
1109        String[] metadataNames = repeater.getMetadataNames();
1110        for (String entryName : metadataNames)
1111        {
1112            repeater.removeMetadata(entryName);
1113            hasChanges = true;
1114        }
1115        
1116        // Create new entries from remote data
1117        NodeList entryNodes = metadataNode.getChildNodes();
1118        for (int i = 0; i < entryNodes.getLength(); i++)
1119        {
1120            Node entryNode = entryNodes.item(i);
1121            String entryName = entryNode.getAttributes().getNamedItem("name").getTextContent().trim();
1122            
1123            NodeList childNodes = entryNode.getChildNodes();
1124            for (int j = 0; j < childNodes.getLength(); j++)
1125            {
1126                Node childNode = childNodes.item(j);
1127                String subMetadataName = childNode.getLocalName();
1128                
1129                hasChanges = _synchronizeMetadata(doc, childNode, content, logicalMetadataPath + "/" + subMetadataName, completeMetadataPath + "/" + entryName + "/" + subMetadataName, contentType, lang, catalog, logger) || hasChanges;
1130            }
1131        }
1132        
1133        return hasChanges;
1134    }
1135    
1136    private boolean _handleCompositeMetadata(Document doc, Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, ContentType contentType, String lang, String catalog, Logger logger)
1137    {
1138        boolean hasChanges = false;
1139        NodeList subMetadataNodes = metadataNode.getChildNodes();
1140        for (int i = 0; i < subMetadataNodes.getLength(); i++)
1141        {
1142            Node subMetadataNode = subMetadataNodes.item(i);
1143            hasChanges = _synchronizeMetadata(doc, subMetadataNode, content, logicalMetadataPath + "/" + subMetadataNode.getLocalName(), completeMetadataPath + "/" + subMetadataNode.getLocalName(), contentType, lang, catalog, logger) || hasChanges;
1144        }
1145        return hasChanges;
1146    }
1147    
1148    private boolean _handleRichTextMetadata(ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, Node metadataNode, boolean synchronize, Logger logger)
1149    {
1150        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
1151        String[] arrayPath = completeMetadataPath.split("/");
1152        String metadataName = arrayPath[arrayPath.length - 1];
1153
1154        if (metadataNode.hasChildNodes())
1155        {
1156            boolean hasChanges = false;
1157            try
1158            {
1159                ExternalizableMetadataStatus status = synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL;
1160                ModifiableRichText richText = ExternalizableMetadataHelper.getRichText(metadataHolder, metadataName, status, true);
1161                
1162                _fetchImages(metadataNode.getFirstChild(), content, richText, completeMetadataPath, logger);
1163                
1164                String docbook = ContentImporterHelper.serializeNode(metadataNode.getFirstChild());
1165                try (ByteArrayInputStream is =  new ByteArrayInputStream(docbook.getBytes("UTF-8")))
1166                {
1167                    richText.setInputStream(is);
1168                    richText.setMimeType("text/xml");
1169                    richText.setLastModified(new Date()); 
1170
1171                    if (synchronize && _importedContents.containsKey(content.getId()))
1172                    {
1173                        hasChanges = ExternalizableMetadataHelper.updateStatus(metadataHolder, metadataName, status) || hasChanges;
1174                    }
1175                    
1176                    hasChanges = true;
1177                }
1178                catch (IOException e)
1179                {
1180                    logger.error("An error occured while parsing the rich text '{}' of the content '{}'", logicalMetadataPath, content.getTitle(), e);
1181                }
1182            }
1183            catch (TransformerException e)
1184            {
1185                logger.error("Error serializing a docbook node.", e);
1186            }
1187            return hasChanges;
1188        }
1189        
1190        return ExternalizableMetadataHelper.removeMetadataIfExists(metadataHolder, metadataName);
1191    }
1192    
1193    /**
1194     * Handle metadata values
1195     * @param doc the document
1196     * @param content the content
1197     * @param metadataDef the metadata definition
1198     * @param untypedValues the metadata values, untyped
1199     * @param lang the language
1200     * @param catalog the catalog
1201     * @param logger the logger
1202     * @return the list of handle value, typed
1203     */
1204    protected List<Object> _handleMetadataValues(Document doc, ModifiableDefaultContent content, MetadataDefinition metadataDef, List<String> untypedValues, String lang, String catalog, Logger logger)
1205    {
1206        if (untypedValues == null)
1207        {
1208            return null;
1209        }
1210        
1211        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
1212        List<Object> typedValues = new ArrayList<>();
1213        
1214        for (String metadataValue : untypedValues)
1215        {
1216            switch (metadataDef.getType())
1217            {
1218                case DATE:
1219                    try
1220                    {
1221                        typedValues.add(dateFormat.parse(metadataValue));
1222                    }
1223                    catch (ParseException e)
1224                    {
1225                        logger.warn("Unable to parse the metadata '{}' of the content '{}' with the value '{}', it should respect the following format '{}'.", metadataDef.getName(), content.getId(), metadataValue, dateFormat);
1226                    }
1227                    break;
1228                case BOOLEAN:
1229                    typedValues.add(Boolean.valueOf(metadataValue));
1230                    break;
1231                case CONTENT:
1232                    Object value = _handleMetadataValuesAsContent(doc, metadataDef, metadataValue, lang, catalog, logger);
1233                    if (value != null)
1234                    {
1235                        typedValues.add(value);
1236                    }
1237                    break;
1238                case LONG:
1239                    typedValues.add(Long.valueOf(metadataValue));
1240                    break;
1241                case DOUBLE:
1242                    typedValues.add(Double.valueOf(metadataValue));
1243                    break;
1244                default:
1245                    if (content instanceof CourseList && metadataDef.getName().equals(CourseList.CHOICE_TYPE))
1246                    {
1247                        typedValues.add(metadataValue.toUpperCase());
1248                    }
1249                    else
1250                    {
1251                        typedValues.add(metadataValue);
1252                    }
1253                    break;
1254            }
1255        }
1256        
1257        return typedValues;
1258    }
1259    
1260    private Object _handleMetadataValuesAsContent(Document doc, MetadataDefinition metadataDef, String metadataValue, String lang, String catalog, Logger logger)
1261    {
1262        String refContentTypeId = metadataDef.getContentType();
1263        if (refContentTypeId != null)
1264        {
1265            ContentType refContentType = _contentTypeEP.getExtension(refContentTypeId);
1266            if (refContentTypeId.equals(PersonFactory.PERSON_CONTENT_TYPE))
1267            {
1268                Node personNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "person[@CDMid ='" + metadataValue + "']");
1269                ModifiableDefaultContent person = _importOrSynchronizeContent(doc, personNode, getPersonWfDescription(), metadataValue, lang, null, metadataValue, logger);
1270                if (person != null)
1271                {
1272                    return person;
1273                }
1274            }
1275            else if (refContentTypeId.equals(OrgUnitFactory.ORGUNIT_CONTENT_TYPE))
1276            {
1277                Node ouNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "orgunit[@CDMid ='" + metadataValue + "']");
1278                ModifiableDefaultContent orgUnit = _importOrSynchronizeContent(doc, ouNode, getOrgUnitWfDescription(), metadataValue, lang, null, metadataValue, logger);
1279                if (orgUnit != null)
1280                {
1281                    return orgUnit;
1282                }
1283            }
1284            else if (metadataDef.getName().equals("courseHolder"))
1285            {
1286                List<Expression> expList = getExpressionsList(lang, metadataValue, getCourseWfDescription().getContentType(), catalog);
1287                AndExpression andExp = new AndExpression(expList.toArray(new Expression[expList.size()]));
1288                String xPathQuery = ContentQueryHelper.getContentXPathQuery(andExp);
1289                AmetysObjectIterable<CoursePart> contents = _resolver.query(xPathQuery);
1290                if (contents.getSize() > 0)
1291                {
1292                    return contents.iterator().next();
1293                }
1294                else
1295                {
1296                    logger.warn("There is no course part corresponding to the CDM ID '{}'.", metadataValue);
1297                }
1298            }
1299            else if (refContentType.isReferenceTable() && _odfRefTableHelper.isTableReference(refContentTypeId))
1300            {
1301                String entryId = _getIdFromCDMThenCode(refContentTypeId, metadataValue);
1302                if (StringUtils.isNotEmpty(entryId))
1303                {
1304                    return _resolver.resolveById(entryId);
1305                }
1306                else
1307                {
1308                    logger.warn("There is no entry corresponding to the CDM-fr or Ametys code '{}' in the reference table '{}'.", metadataValue, refContentTypeId);
1309                }
1310            }
1311        }
1312        else
1313        {
1314            logger.warn("Cannot match data '{}' of content type '{}' because it is not typed.", metadataDef.getName(), metadataDef.getReferenceContentType());
1315        }
1316        
1317        return null;
1318    }
1319
1320    /**
1321     * Search for translated contents
1322     * @param importedContent The imported content
1323     * @param contentType The content type
1324     */
1325    protected void _linkTranslationsIfExist(ModifiableContent importedContent, String contentType)
1326    {
1327        if (importedContent instanceof ProgramItem)
1328        {
1329            Map<String, String> translations = new HashMap<>();
1330    
1331            List<Expression> expList = getExpressionsList(importedContent.getLanguage(), importedContent.getMetadataHolder().getString(getIdField()), contentType, importedContent.getMetadataHolder().getString(ProgramItem.CATALOG, null));
1332            AndExpression andExp = new AndExpression(expList.toArray(new Expression[expList.size()]));
1333            String xPathQuery = ContentQueryHelper.getContentXPathQuery(andExp);
1334    
1335            AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(xPathQuery);
1336            
1337            for (ModifiableDefaultContent content : contents)
1338            {
1339                translations.put(content.getLanguage(), content.getId());
1340                
1341                Map<String, String> translations2 = TranslationHelper.getTranslations(content);
1342                translations2.put(importedContent.getLanguage(), importedContent.getId());
1343                TranslationHelper.setTranslations(content, translations2);
1344    
1345                Map<String, Object> eventParams = new HashMap<>();
1346                eventParams.put(ObservationConstants.ARGS_CONTENT, content);
1347                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId());
1348                _observationManager.notify(new Event(OdfObservationConstants.ODF_CONTENT_TRANSLATED, _currentUserProvider.getUser(), eventParams));
1349            }
1350            
1351            TranslationHelper.setTranslations(importedContent, translations);
1352        }
1353    }
1354
1355    /**
1356     * Save the changes of the content.
1357     * @param content Content to save
1358     * @param contentTypeId The content type (for logs)
1359     * @param hasChanges If there are changes to save
1360     * @param logger The logger
1361     */
1362    protected void _saveContentChanges(ModifiableDefaultContent content, String contentTypeId, boolean hasChanges, Logger logger)
1363    {
1364        if (hasChanges)
1365        {
1366            content.saveChanges();
1367            _synchronizedContents.add(content.getId());
1368            if (!_importedContents.containsKey(content.getId()))
1369            {
1370                _nbSynchronizedContents++;
1371            }
1372        }
1373        else
1374        {
1375            logger.info("No changes detected for content '{}' with the content type '{}'", content.getTitle(), contentTypeId);
1376            if (!_importedContents.containsKey(content.getId()))
1377            {
1378                _nbNotChangedContents++;
1379            }
1380        }
1381    }
1382    
1383    /**
1384     * Set relations for the content.
1385     * @param content The content to update
1386     * @param children Children to set
1387     * @param courseParts {@link List} of {@link CoursePart} to set for {@link Course} content
1388     * @param removeOldCourseParts If the course parts doesn't have a code, we remove all the old {@link CoursePart}s before adding the new ones
1389     * @param logger The logger
1390     * @return <code>true</code> if changes occurs
1391     */
1392    protected boolean _setRelations(ModifiableDefaultContent content, List<ModifiableDefaultContent> children, List<ModifiableDefaultContent> courseParts, boolean removeOldCourseParts, Logger logger)
1393    {
1394        boolean hasChanges = false;
1395        
1396        if (!courseParts.isEmpty() && content instanceof Course)
1397        {
1398            if (removeOldCourseParts)
1399            {
1400                hasChanges = ExternalizableMetadataHelper.removeMetadataIfExists(content.getMetadataHolder(), Course.CHILD_COURSE_PARTS) || hasChanges;
1401            }
1402            hasChanges = _updateDoubleRelation(content, courseParts, Course.CHILD_COURSE_PARTS, CoursePart.PARENT_COURSES, logger) || hasChanges;
1403        }
1404        
1405        if (!children.isEmpty())
1406        {
1407            hasChanges = _setChildren(content, children, logger) || hasChanges;
1408        }
1409        
1410        // Set the orgUnit parent (if no parent is set)
1411        if (content instanceof OrgUnit)
1412        {
1413            hasChanges = _setOrgUnitParent(content, logger) || hasChanges;
1414        }
1415        
1416        return hasChanges;
1417    }
1418    
1419    /**
1420     * Set children for the given content.
1421     * @param content Content to add the children
1422     * @param children Children to add
1423     * @param logger The logger
1424     * @return <code>true</code> if changes occurs
1425     */
1426    protected boolean _setChildren(ModifiableDefaultContent content, List<ModifiableDefaultContent> children, Logger logger)
1427    {
1428        boolean hasChanges = false;
1429        
1430        String metadataName = null;
1431        String invertMetadataName = null;
1432        if (content instanceof CourseList)
1433        {
1434            metadataName = CourseList.CHILD_COURSES;
1435            invertMetadataName = Course.PARENT_COURSE_LISTS;
1436        }
1437        else if (content instanceof TraversableProgramPart)
1438        {
1439            metadataName = TraversableProgramPart.CHILD_PROGRAM_PARTS;
1440            invertMetadataName = ProgramPart.PARENT_PROGRAM_PARTS;
1441        }
1442        else if (content instanceof Course)
1443        {
1444            metadataName = Course.CHILD_COURSE_LISTS;
1445            invertMetadataName = CourseList.PARENT_COURSES;
1446        }
1447        
1448        if (metadataName != null)
1449        {
1450            hasChanges = _updateDoubleRelation(content, children, metadataName, invertMetadataName, logger);
1451        }
1452        
1453        return hasChanges;
1454    }
1455    
1456    /**
1457     * Set the orgUnit parent to rootOrgUnit.
1458     * @param orgUnit The orgunit to link
1459     * @param logger The logger
1460     * @return <code>true</code> if changes occurs
1461     */
1462    protected boolean _setOrgUnitParent(ModifiableDefaultContent orgUnit, Logger logger)
1463    {
1464        boolean hasChanges = false;
1465        
1466        // Set the orgUnit parent (if no parent is set)
1467        ModifiableCompositeMetadata holder = orgUnit.getMetadataHolder();
1468        if (!holder.hasMetadata(OrgUnit.PARENT_ORGUNIT))
1469        {
1470            OrgUnit rootOrgUnit = _rootOUProvider.getRoot();
1471            
1472            hasChanges = ExternalizableMetadataHelper.setMetadata(holder, OrgUnit.PARENT_ORGUNIT, rootOrgUnit) || hasChanges;
1473            hasChanges = _synchroComponent.updateRelation(rootOrgUnit.getMetadataHolder(), OrgUnit.CHILD_ORGUNITS, orgUnit, false) || hasChanges;
1474
1475            try
1476            {
1477                _synchroComponent.applyChanges(rootOrgUnit, 22, ObservationConstants.EVENT_CONTENT_MODIFIED, ignoreRights(), logger);
1478            }
1479            catch (RepositoryException e)
1480            {
1481                logger.error("An error occured during updating root org unit after synchronizing the content '{}'.", orgUnit.getId(), logger);
1482            }
1483        }
1484        
1485        return hasChanges;
1486    }
1487
1488    /**
1489     * Get the content ID from the CDM code, if there is no match with the CDM code, then we search with the code.
1490     * If nothing is found we return null. 
1491     * @param tableRefId The reference table ID
1492     * @param cdmCode The CDM code
1493     * @return A content ID or <code>null</code>
1494     */
1495    protected String _getIdFromCDMThenCode(String tableRefId, String cdmCode)
1496    {
1497        OdfReferenceTableEntry entry = _odfRefTableHelper.getItemFromCDM(tableRefId, cdmCode);
1498        if (entry == null)
1499        {
1500            entry = _odfRefTableHelper.getItemFromCode(tableRefId, cdmCode);
1501        }
1502        return entry != null ? entry.getId() : null;
1503    }
1504    
1505    private boolean _updateDoubleRelation(ModifiableDefaultContent content, List<ModifiableDefaultContent> children, String metadataName, String invertMetadataName, Logger logger)
1506    {
1507        boolean hasChanges = false;
1508        ModifiableCompositeMetadata holder = content.getMetadataHolder();
1509        
1510        // "Normal" relation
1511        if (removalSync() || !holder.hasMetadata(metadataName))
1512        {
1513            hasChanges = ExternalizableMetadataHelper.setMetadata(holder, metadataName, children.toArray(new ModifiableDefaultContent[children.size()]));
1514        }
1515        else
1516        {
1517            List<String> oldValues = new ArrayList<>(Arrays.asList(holder.getStringArray(metadataName)));
1518            
1519            for (ModifiableDefaultContent child : children)
1520            {
1521                hasChanges = _synchroComponent.updateRelation(holder, metadataName, child, false) || hasChanges;
1522                oldValues.remove(child.getId());
1523            }
1524            
1525            if (logger.isWarnEnabled())
1526            {
1527                for (String oldValue : oldValues)
1528                {
1529                    // Warn the old contents
1530                    ModifiableDefaultContent child = _resolver.resolveById(oldValue);
1531                    logger.warn("The ODF content '{}' ({}) is not linked anymore to the content '{}', it should be manually removed.", child.getTitle(), oldValue, content.getTitle());
1532                }
1533            }
1534        }
1535        
1536        // Invert relation
1537        for (ModifiableDefaultContent child : children)
1538        {
1539            hasChanges = _synchroComponent.updateRelation(child.getMetadataHolder(), invertMetadataName, content, false) || hasChanges;
1540        }
1541        
1542        return hasChanges;
1543    }
1544    
1545    private String _getXPathString(Node metadataNode, String xPath, String defaultValue)
1546    {
1547        String value = _xPathProcessor.evaluateAsString(metadataNode, xPath);
1548        if (StringUtils.isEmpty(value))
1549        {
1550            value = defaultValue;
1551        }
1552        return value;
1553    }
1554
1555    /**
1556     * If true, bypass the rights check during the import process
1557     * @return True if the rights check are bypassed during the import process
1558     */
1559    protected boolean ignoreRights()
1560    {
1561        return false;
1562    }
1563    
1564    /**
1565     * Get the program workflow description.
1566     * @return A {@link ContentWorkflowDescription} containing informations about the program workflow
1567     */
1568    protected ContentWorkflowDescription getProgramWfDescription()
1569    {
1570        return _PROGRAM_WF_DESCRIPTION;
1571    }
1572
1573    /**
1574     * Get the subprogram workflow description.
1575     * @return A {@link ContentWorkflowDescription} containing informations about the subprogram workflow
1576     */
1577    protected ContentWorkflowDescription getSubProgramWfDescription()
1578    {
1579        return _SUBPROGRAM_WF_DESCRIPTION;
1580    }
1581
1582    /**
1583     * Get the container workflow description.
1584     * @return A {@link ContentWorkflowDescription} containing informations about the container workflow
1585     */
1586    protected ContentWorkflowDescription getContainerWfDescription()
1587    {
1588        return _CONTAINER_WF_DESCRIPTION;
1589    }
1590
1591    /**
1592     * Get the course list workflow description.
1593     * @return A {@link ContentWorkflowDescription} containing informations about the course list workflow
1594     */
1595    protected ContentWorkflowDescription getCourseListWfDescription()
1596    {
1597        return _COURSELIST_WF_DESCRIPTION;
1598    }
1599
1600    /**
1601     * Get the course workflow description.
1602     * @return A {@link ContentWorkflowDescription} containing informations about the course workflow
1603     */
1604    protected ContentWorkflowDescription getCourseWfDescription()
1605    {
1606        return _COURSE_WF_DESCRIPTION;
1607    }
1608
1609    /**
1610     * Get the course part workflow description.
1611     * @return A {@link ContentWorkflowDescription} containing informations about the course part workflow
1612     */
1613    protected ContentWorkflowDescription getCoursePartWfDescription()
1614    {
1615        return _COURSEPART_WF_DESCRIPTION;
1616    }
1617
1618    /**
1619     * Get the orgunit workflow description.
1620     * @return A {@link ContentWorkflowDescription} containing informations about the orgunit workflow
1621     */
1622    protected ContentWorkflowDescription getOrgUnitWfDescription()
1623    {
1624        return _ORGUNIT_WF_DESCRIPTION;
1625    }
1626
1627    /**
1628     * Get the person workflow description.
1629     * @return A {@link ContentWorkflowDescription} containing informations about the person workflow
1630     */
1631    protected ContentWorkflowDescription getPersonWfDescription()
1632    {
1633        return _PERSON_WF_DESCRIPTION;
1634    }
1635    
1636    /**
1637     * Internal object to describe content workflow elements.
1638     */
1639    protected static class ContentWorkflowDescription
1640    {
1641        private String _contentType;
1642        private String _workflowName;
1643        private int _initialActionId;
1644        private int _validationActionId;
1645        
1646        ContentWorkflowDescription(String contentType, String workflowName, int initialActionId, int validationActionId)
1647        {
1648            _contentType = contentType;
1649            _workflowName = workflowName;
1650            _initialActionId = initialActionId;
1651            _validationActionId = validationActionId;
1652        }
1653        
1654        /**
1655         * Get the content type.
1656         * @return the content type ID
1657         */
1658        public String getContentType()
1659        {
1660            return _contentType;
1661        }
1662        
1663        /**
1664         * Get the workflow name.
1665         * @return the workflow name
1666         */
1667        public String getWorkflowName()
1668        {
1669            return _workflowName;
1670        }
1671        
1672        /**
1673         * Get the initial action ID.
1674         * @return the initial action ID
1675         */
1676        public int getInitialActionId()
1677        {
1678            return _initialActionId;
1679        }
1680        
1681        /**
1682         * Get the validation action ID.
1683         * @return the validation action ID
1684         */
1685        public int getValidationActionId()
1686        {
1687            return _validationActionId;
1688        }
1689    }
1690    
1691    private static class DocbookPrefixResolver implements PrefixResolver
1692    {
1693        private Map<String, String> _ns = new HashMap<>();
1694        
1695        public DocbookPrefixResolver()
1696        {
1697            _ns.put("docbook", "http://docbook.org/ns/docbook");
1698        }
1699        
1700        @Override
1701        public String prefixToNamespace(String prefix)
1702        {
1703            return _ns.get(prefix);
1704        }
1705    }
1706    
1707    @Override
1708    public List<Expression> getExpressionsList(String lang, String idValue, String contentType, String catalog)
1709    {
1710        List<Expression> expList = new ArrayList<>();
1711        
1712        if (StringUtils.isNotBlank(contentType))
1713        {
1714            expList.add(new ContentTypeExpression(Operator.EQ, contentType));
1715        }
1716        
1717        if (StringUtils.isNotBlank(idValue))
1718        {
1719            expList.add(new StringExpression(getIdField(), Operator.EQ, idValue));
1720        }
1721        
1722        if (StringUtils.isNotBlank(catalog))
1723        {
1724            expList.add(new StringExpression(ProgramItem.CATALOG, Operator.EQ, catalog));
1725        }
1726        
1727        if (StringUtils.isNotBlank(lang))
1728        {
1729            expList.add(new LanguageExpression(Operator.EQ, lang));
1730        }
1731        
1732        return expList;
1733    }
1734
1735    @Override
1736    public Set<String> getLocalAndExternalFields(Map<String, Object> additionalParameters)
1737    {
1738        if (additionalParameters == null || !additionalParameters.containsKey("contentTypes"))
1739        {
1740            throw new IllegalArgumentException("Content types shouldn't be null.");
1741        }
1742
1743        @SuppressWarnings("unchecked")
1744        List<String> contentTypeIds = (List<String>) additionalParameters.get("contentTypes");
1745        Set<String> allSyncFields = new HashSet<>();
1746        
1747        for (String contentTypeId : contentTypeIds)
1748        {
1749            Set<String> syncFields = _syncFieldsByContentType.computeIfAbsent(contentTypeId, k -> new HashSet<>());
1750            allSyncFields.addAll(syncFields);
1751        }
1752        
1753        return allSyncFields;
1754    }
1755}