001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.odfsync.cdmfr.components;
017
018import java.io.ByteArrayInputStream;
019import java.io.File;
020import java.io.FileInputStream;
021import java.io.IOException;
022import java.io.InputStream;
023import java.net.URL;
024import java.net.URLDecoder;
025import java.text.ParseException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Date;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.LinkedList;
032import java.util.List;
033import java.util.Map;
034import java.util.Objects;
035import java.util.Set;
036
037import javax.jcr.RepositoryException;
038import javax.xml.transform.TransformerException;
039
040import org.apache.avalon.framework.activity.Initializable;
041import org.apache.avalon.framework.component.Component;
042import org.apache.avalon.framework.configuration.Configurable;
043import org.apache.avalon.framework.configuration.Configuration;
044import org.apache.avalon.framework.configuration.ConfigurationException;
045import org.apache.avalon.framework.configuration.DefaultConfigurationBuilder;
046import org.apache.avalon.framework.context.ContextException;
047import org.apache.avalon.framework.context.Contextualizable;
048import org.apache.avalon.framework.service.ServiceException;
049import org.apache.avalon.framework.service.ServiceManager;
050import org.apache.avalon.framework.service.Serviceable;
051import org.apache.cocoon.Constants;
052import org.apache.cocoon.ProcessingException;
053import org.apache.cocoon.environment.Context;
054import org.apache.commons.collections4.CollectionUtils;
055import org.apache.commons.io.IOUtils;
056import org.apache.commons.lang3.StringUtils;
057import org.apache.excalibur.xml.dom.DOMParser;
058import org.apache.excalibur.xml.xpath.PrefixResolver;
059import org.apache.excalibur.xml.xpath.XPathProcessor;
060import org.slf4j.Logger;
061import org.w3c.dom.Document;
062import org.w3c.dom.Element;
063import org.w3c.dom.NamedNodeMap;
064import org.w3c.dom.Node;
065import org.w3c.dom.NodeList;
066import org.xml.sax.InputSource;
067import org.xml.sax.SAXException;
068
069import org.ametys.cms.ObservationConstants;
070import org.ametys.cms.content.external.ExternalizableMetadataHelper;
071import org.ametys.cms.content.external.ExternalizableMetadataProvider.ExternalizableMetadataStatus;
072import org.ametys.cms.contenttype.ContentType;
073import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
074import org.ametys.cms.contenttype.MetadataDefinition;
075import org.ametys.cms.contenttype.MetadataType;
076import org.ametys.cms.contenttype.RepeaterDefinition;
077import org.ametys.cms.repository.ContentQueryHelper;
078import org.ametys.cms.repository.ContentTypeExpression;
079import org.ametys.cms.repository.LanguageExpression;
080import org.ametys.cms.repository.ModifiableContent;
081import org.ametys.cms.repository.ModifiableDefaultContent;
082import org.ametys.cms.repository.comment.DefaultCommentManagerExtensionPoint;
083import org.ametys.core.observation.Event;
084import org.ametys.core.observation.ObservationManager;
085import org.ametys.core.user.CurrentUserProvider;
086import org.ametys.core.user.population.UserPopulationDAO;
087import org.ametys.odf.ProgramItem;
088import org.ametys.odf.catalog.CatalogsManager;
089import org.ametys.odf.cdmfr.CDMHelper;
090import org.ametys.odf.course.Course;
091import org.ametys.odf.course.CourseFactory;
092import org.ametys.odf.course.ShareableCourseHelper;
093import org.ametys.odf.courselist.CourseList;
094import org.ametys.odf.courselist.CourseListFactory;
095import org.ametys.odf.coursepart.CoursePart;
096import org.ametys.odf.coursepart.CoursePartFactory;
097import org.ametys.odf.enumeration.OdfReferenceTableEntry;
098import org.ametys.odf.enumeration.OdfReferenceTableHelper;
099import org.ametys.odf.observation.OdfObservationConstants;
100import org.ametys.odf.orgunit.OrgUnit;
101import org.ametys.odf.orgunit.OrgUnitFactory;
102import org.ametys.odf.orgunit.RootOrgUnitProvider;
103import org.ametys.odf.person.PersonFactory;
104import org.ametys.odf.program.ContainerFactory;
105import org.ametys.odf.program.ProgramFactory;
106import org.ametys.odf.program.ProgramPart;
107import org.ametys.odf.program.SubProgramFactory;
108import org.ametys.odf.program.TraversableProgramPart;
109import org.ametys.odf.translation.TranslationHelper;
110import org.ametys.plugins.contentio.ContentImporterHelper;
111import org.ametys.plugins.contentio.synchronize.BaseSynchroComponent;
112import org.ametys.plugins.odfsync.cdmfr.CDMFrSyncExtensionPoint;
113import org.ametys.plugins.odfsync.cdmfr.transformers.CDMFrSyncTransformer;
114import org.ametys.plugins.repository.AmetysObjectIterable;
115import org.ametys.plugins.repository.AmetysObjectResolver;
116import org.ametys.plugins.repository.AmetysRepositoryException;
117import org.ametys.plugins.repository.metadata.CompositeMetadata;
118import org.ametys.plugins.repository.metadata.ModifiableBinaryMetadata;
119import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
120import org.ametys.plugins.repository.metadata.ModifiableFile;
121import org.ametys.plugins.repository.metadata.ModifiableResource;
122import org.ametys.plugins.repository.metadata.ModifiableRichText;
123import org.ametys.plugins.repository.query.expression.AndExpression;
124import org.ametys.plugins.repository.query.expression.Expression;
125import org.ametys.plugins.repository.query.expression.Expression.Operator;
126import org.ametys.plugins.repository.query.expression.StringExpression;
127import org.ametys.runtime.config.Config;
128
129import com.google.common.collect.ImmutableList;
130import com.google.common.collect.ImmutableMap;
131
132/**
133 * Abstract class of a component to import a CDM-fr input stream.
134 */
135public abstract class AbstractImportCDMFrComponent implements ImportCDMFrComponent, Serviceable, Initializable, Contextualizable, Configurable, Component
136{
137    /** Tag to identify a program */
138    protected static final String _TAG_PROGRAM = "program";
139
140    /** Tag to identify a subprogram */
141    protected static final String _TAG_SUBPROGRAM = "subProgram";
142    
143    /** Tag to identify a container */
144    protected static final String _TAG_CONTAINER = "container";
145    
146    /** Tag to identify a courseList */
147    protected static final String _TAG_COURSELIST = "coursesReferences";
148    
149    /** Tag to identify a coursePart */
150    protected static final String _TAG_COURSEPART = "coursePart";
151
152    /** The synchronize workflow action id */
153    private static final int SYNCHRONIZE_WORKFLOW_ACTION_ID = 800;
154    
155    private static final ContentWorkflowDescription _PROGRAM_WF_DESCRIPTION = new ContentWorkflowDescription(ProgramFactory.PROGRAM_CONTENT_TYPE, "program", 1, 4);
156    private static final ContentWorkflowDescription _SUBPROGRAM_WF_DESCRIPTION = new ContentWorkflowDescription(SubProgramFactory.SUBPROGRAM_CONTENT_TYPE, "subprogram", 1, 4);
157    private static final ContentWorkflowDescription _CONTAINER_WF_DESCRIPTION = new ContentWorkflowDescription(ContainerFactory.CONTAINER_CONTENT_TYPE, "container", 1, 4);
158    private static final ContentWorkflowDescription _COURSELIST_WF_DESCRIPTION = new ContentWorkflowDescription(CourseListFactory.COURSE_LIST_CONTENT_TYPE, "courselist", 1, 4);
159    private static final ContentWorkflowDescription _COURSE_WF_DESCRIPTION = new ContentWorkflowDescription(CourseFactory.COURSE_CONTENT_TYPE, "course", 1, 4);
160    private static final ContentWorkflowDescription _COURSEPART_WF_DESCRIPTION = new ContentWorkflowDescription(CoursePartFactory.COURSE_PART_CONTENT_TYPE, "course-part", 1, -1);
161    private static final ContentWorkflowDescription _ORGUNIT_WF_DESCRIPTION = new ContentWorkflowDescription(OrgUnitFactory.ORGUNIT_CONTENT_TYPE, "orgunit", 1, 4);
162    private static final ContentWorkflowDescription _PERSON_WF_DESCRIPTION = new ContentWorkflowDescription(PersonFactory.PERSON_CONTENT_TYPE, "person", 1, 4);
163
164    private static final PrefixResolver _PREFIX_RESOLVER = new DocbookPrefixResolver();
165    
166    /** The Cocoon context */
167    protected Context _cocoonContext;
168    
169    /** The DOM parser */
170    protected DOMParser _domParser;
171
172    /** The XPath processor */
173    protected XPathProcessor _xPathProcessor;
174
175    /** Extension point to transform CDM-fr */
176    protected CDMFrSyncExtensionPoint _cdmFrSyncExtensionPoint;
177
178    /** Default language configured for ODF */
179    protected String _odfLang;
180    
181    /** The catalog manager */
182    protected CatalogsManager _catalogsManager;
183
184    /** The ametys object resolver */
185    protected AmetysObjectResolver _resolver;
186
187    /** The ODF TableRef Helper */
188    protected OdfReferenceTableHelper _odfRefTableHelper;
189
190    /** The content type extension point */
191    protected ContentTypeExtensionPoint _contentTypeEP;
192
193    /** The current user provider */
194    protected CurrentUserProvider _currentUserProvider;
195
196    /** The observation manager */
197    protected ObservationManager _observationManager;
198    
199    /** The root orgunit provider */
200    protected RootOrgUnitProvider _rootOUProvider;
201
202    /** The base SCC component */
203    protected BaseSynchroComponent _synchroComponent;
204    
205    /** The shareable course helper */
206    protected ShareableCourseHelper _shareableCourseHelper;
207    
208    /** List of synchronized contents */
209    protected Map<String, Integer> _importedContents;
210    
211    /** List of synchronized contents having differences */
212    protected Set<String> _synchronizedContents;
213    
214    /** List of synchronized contents (to avoid a treatment twice or more) */
215    protected Set<String> _updatedContents;
216
217    /** Number of errors encountered */
218    protected int _nbError;
219    /** Number of created contents */
220    protected int _nbCreatedContents;
221    /** Number of synchronized contents */
222    protected int _nbSynchronizedContents;
223    /** Number of unchanged contents */
224    protected int _nbNotChangedContents;
225    /** The prefix of the contents */
226    protected String _contentPrefix;
227    /** Synchronized fields by content type */
228    protected Map<String, Set<String>> _syncFieldsByContentType;
229
230    public void initialize() throws Exception
231    {
232        _odfLang = Config.getInstance().getValue("odf.programs.lang");
233    }
234
235    @Override
236    public void contextualize(org.apache.avalon.framework.context.Context context) throws ContextException
237    {
238        _cocoonContext = (Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
239    }
240
241    public void configure(Configuration configuration) throws ConfigurationException
242    {
243        _parseSynchronizedFields();
244    }
245    
246    @Override
247    public void service(ServiceManager manager) throws ServiceException
248    {
249        _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
250        _domParser = (DOMParser) manager.lookup(DOMParser.ROLE);
251        _xPathProcessor = (XPathProcessor) manager.lookup(XPathProcessor.ROLE);
252        _cdmFrSyncExtensionPoint = (CDMFrSyncExtensionPoint) manager.lookup(CDMFrSyncExtensionPoint.ROLE);
253        _catalogsManager = (CatalogsManager) manager.lookup(CatalogsManager.ROLE);
254        _odfRefTableHelper = (OdfReferenceTableHelper) manager.lookup(OdfReferenceTableHelper.ROLE);
255        _contentTypeEP = (ContentTypeExtensionPoint) manager.lookup(ContentTypeExtensionPoint.ROLE);
256        _currentUserProvider = (CurrentUserProvider) manager.lookup(CurrentUserProvider.ROLE);
257        _observationManager = (ObservationManager) manager.lookup(ObservationManager.ROLE);
258        _rootOUProvider = (RootOrgUnitProvider) manager.lookup(RootOrgUnitProvider.ROLE);
259        _synchroComponent = (BaseSynchroComponent) manager.lookup(BaseSynchroComponent.ROLE);
260        _shareableCourseHelper = (ShareableCourseHelper) manager.lookup(ShareableCourseHelper.ROLE);
261    }
262
263    @Override
264    public String getIdField()
265    {
266        return "cdmfrSyncCode";
267    }
268
269    /**
270     * Get the synchronized metadata from the configuration file
271     * @throws ConfigurationException if the configuration is not valid.
272     */
273    private void _parseSynchronizedFields() throws ConfigurationException
274    {
275        _syncFieldsByContentType = new HashMap<>();
276        
277        @SuppressWarnings("resource")
278        InputStream is = null;
279        try
280        {
281            File apogeeMapping = new File(_cocoonContext.getRealPath("/WEB-INF/param/odf-synchro.xml"));
282            if (!apogeeMapping.isFile())
283            {
284                is = getClass().getResourceAsStream("/org/ametys/plugins/odfsync/cdmfr/odf-synchro.xml");
285            }
286            else
287            {
288                is = new FileInputStream(apogeeMapping);
289            }
290            Configuration cfg = new DefaultConfigurationBuilder().build(is);
291
292            Configuration[] cTypesConf = cfg.getChildren("content-type");
293            for (Configuration cTypeConf : cTypesConf)
294            {
295                String contentType = cTypeConf.getAttribute("id");
296                Set<String> syncMetadata = _configureSynchronizedFields(cTypeConf, "");
297                _syncFieldsByContentType.put(contentType, syncMetadata);
298            }
299        }
300        catch (Exception e)
301        {
302            throw new ConfigurationException("Error while parsing odf-synchro.xml", e);
303        }
304        finally
305        {
306            IOUtils.closeQuietly(is);
307        }
308    }
309
310    private Set<String> _configureSynchronizedFields(Configuration configuration, String prefix) throws ConfigurationException
311    {
312        Set<String> syncMetadata = new HashSet<>();
313        Configuration[] metaConf = configuration.getChildren("metadata");
314        
315        if (metaConf.length > 0)
316        {
317            for (Configuration metadata : metaConf)
318            {
319                if (metadata.getChildren("metadata").length > 0)
320                {
321                    // composite
322                    syncMetadata.addAll(_configureSynchronizedFields(metadata, prefix + metadata.getAttribute("name") + "/"));
323                }
324                else
325                {
326                    syncMetadata.add(prefix + metadata.getAttribute("name"));
327                }
328            }
329        }
330        else if (configuration.getAttribute("name", null) != null)
331        {
332            syncMetadata.add(prefix + configuration.getAttribute("name"));
333        }
334        
335        return syncMetadata;
336    }
337    
338    @Override
339    @SuppressWarnings("unchecked")
340    public synchronized Map<String, Object> handleInputStream(InputStream input, Map<String, Object> parameters, Logger logger) throws ProcessingException
341    {
342        List<ModifiableDefaultContent> importedPrograms = new ArrayList<>();
343        
344        _importedContents = (Map<String, Integer>) parameters.getOrDefault("importedContents", new HashMap<>());
345        _synchronizedContents = (Set<String>) parameters.getOrDefault("synchronizedContents", new HashSet<>());
346        _updatedContents = (Set<String>) parameters.getOrDefault("updatedContents", new HashSet<>());
347        _nbCreatedContents = (int) parameters.getOrDefault("nbCreatedContents", 0);
348        _nbSynchronizedContents = (int) parameters.getOrDefault("nbSynchronizedContents", 0);
349        _nbNotChangedContents = (int) parameters.getOrDefault("nbNotChangedContents", 0);
350        _nbError = (int) parameters.getOrDefault("nbError", 0);
351        _contentPrefix = (String) parameters.getOrDefault("contentPrefix", "cdmfr-");
352        additionalParameters(parameters);
353        
354        Map<String, Object> resultMap = new HashMap<>();
355        
356        try
357        {
358            Document doc = _domParser.parseDocument(new InputSource(input));
359            doc = transformDocument(doc, new HashMap<String, Object>(), logger);
360
361            if (doc != null)
362            {
363                String defaultLang = _getXPathString(doc, "CDM/@language", _odfLang);
364                
365                NodeList nodes = doc.getElementsByTagName(_TAG_PROGRAM);
366                
367                for (int i = 0; i < nodes.getLength(); i++)
368                {
369                    Node contentNode = nodes.item(i);
370                    String syncCode = _xPathProcessor.evaluateAsString(contentNode, "@CDMid");
371                    String contentLang = _getXPathString(contentNode, "@language", defaultLang);
372                    contentLang = StringUtils.substring(contentLang, 0, 2).toLowerCase(); // on keep the language from the locale
373                    
374                    String catalog = getCatalogName(contentNode);
375
376                    importedPrograms.add(_importOrSynchronizeContent(doc, contentNode, getProgramWfDescription(), syncCode, contentLang, catalog, syncCode, logger));
377                }
378                
379                // Apply changes (synchronize action)
380                boolean ignoreRights = ignoreRights();
381                for (String contentId : _synchronizedContents)
382                {
383                    ModifiableDefaultContent content = _resolver.resolveById(contentId);
384                    _synchroComponent.applyChanges(content, SYNCHRONIZE_WORKFLOW_ACTION_ID, org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, ignoreRights, logger);
385                }
386                
387                // Validate contents
388                if (validateAfterImport())
389                {
390                    for (String contentId : _importedContents.keySet())
391                    {
392                        ModifiableDefaultContent content = _resolver.resolveById(contentId);
393                        Integer validationActionId = _importedContents.get(contentId);
394                        if (validationActionId > 0)
395                        {
396                            _synchroComponent.validateContent(content, validationActionId, ignoreRights, logger);
397                        }
398                    }
399                }
400            }
401        }
402        catch (IOException | ProcessingException e)
403        {
404            throw new ProcessingException("An error occured while transforming the stream.", e);
405        }
406        catch (SAXException e)
407        {
408            throw new ProcessingException("An error occured while parsing the stream.", e);
409        }
410        catch (RepositoryException e)
411        {
412            throw new ProcessingException("An error occured while applying changes on contents.", e);
413        }
414
415        resultMap.put("importedContents", _importedContents);
416        resultMap.put("synchronizedContents", _synchronizedContents);
417        resultMap.put("updatedContents", _updatedContents);
418        resultMap.put("nbCreatedContents", _nbCreatedContents);
419        resultMap.put("nbSynchronizedContents", _nbSynchronizedContents);
420        resultMap.put("nbNotChangedContents", _nbNotChangedContents);
421        resultMap.put("nbError", _nbError);
422        resultMap.put("importedPrograms", importedPrograms);
423
424        return resultMap;
425    }
426
427    /**
428     * True to validate the contents after import
429     * @return True to validate the contents after import
430     */
431    protected abstract boolean validateAfterImport();
432
433    /**
434     * When returns true, a content created by a previous synchro will be removed if it does not exist anymore during the current synchro.
435     * @return true if a content created by a previous synchro has to be removed if it does not exist anymore during the current synchro.
436     */
437    protected abstract boolean removalSync();
438    
439    /**
440     * Transform the document depending of it structure.
441     * @param document Document to transform.
442     * @param parameters Optional parameters for transformation
443     * @param logger The logger
444     * @return The transformed document.
445     * @throws IOException if an error occurs.
446     * @throws SAXException if an error occurs.
447     * @throws ProcessingException if an error occurs.
448     */
449    protected Document transformDocument(Document document, Map<String, Object> parameters, Logger logger) throws IOException, SAXException, ProcessingException
450    {
451        CDMFrSyncTransformer transformer = _cdmFrSyncExtensionPoint.getTransformer(document);
452        if (transformer == null)
453        {
454            logger.error("Cannot match a CDM-fr transformer to this file structure.");
455            return null;
456        }
457        
458        return transformer.transform(document, parameters);
459    }
460
461    /**
462     * Get the name of catalog to use for import
463     * @param contentNode The node of program
464     * @return The catalog to used
465     */
466    protected String getCatalogName(Node contentNode)
467    {
468        String defaultCatalog = _catalogsManager.getDefaultCatalogName();
469        
470        String contentCatalog = _getXPathString(contentNode, "catalog", defaultCatalog);
471        if (_catalogsManager.getCatalog(contentCatalog) == null)
472        {
473            // Catalog is empty or do not exist, use the default catalog
474            return defaultCatalog;
475        }
476        
477        return contentCatalog;
478    }
479    
480    /**
481     * Get or create the content from the synchronization code, the lang, the catalog and the content type.
482     * @param title The title
483     * @param lang The lang
484     * @param catalog The catalog
485     * @param syncCode The synchronization code
486     * @param wfDescription The workflow description
487     * @param logger The logger
488     * @return the retrieved or created content
489     * @throws RepositoryException if an error occurs
490     */
491    protected ModifiableDefaultContent _getOrCreateContent(String title, String lang, String catalog, String syncCode, ContentWorkflowDescription wfDescription, Logger logger) throws RepositoryException
492    {
493        ModifiableDefaultContent receivedContent = _getContent(lang, catalog, syncCode, wfDescription);
494        if (receivedContent != null)
495        {
496            return receivedContent;
497        }
498        
499        Map<String, Object> resultMap = _synchroComponent.createContentAction(wfDescription.getContentType(), wfDescription.getWorkflowName(), wfDescription.getInitialActionId(), lang, title, _contentPrefix, logger);
500        if ((boolean) resultMap.getOrDefault("error", false))
501        {
502            _nbError++;
503        }
504        
505        ModifiableDefaultContent content = (ModifiableDefaultContent) resultMap.get("content");
506        
507        if (content != null)
508        {
509            ExternalizableMetadataHelper.setMetadata(content.getMetadataHolder(), getIdField(), syncCode);
510            if (catalog != null && (content instanceof ProgramItem || content instanceof CoursePart))
511            {
512                ExternalizableMetadataHelper.setMetadata(content.getMetadataHolder(), ProgramItem.METADATA_CATALOG, catalog);
513            }
514            additionalOperationsBeforeSave(content, logger);
515            content.saveChanges();
516            _importedContents.put(content.getId(), wfDescription.getValidationActionId());
517            _nbCreatedContents++;
518        }
519        return content;
520    }
521
522    /**
523     * Get the content from the synchronization code, the lang, the catalog and the content type.
524     * @param lang The lang
525     * @param catalog The catalog
526     * @param syncCode The synchronization code
527     * @param wfDescription The workflow description
528     * @return the retrieved content
529     */
530    protected ModifiableDefaultContent _getContent(String lang, String catalog, String syncCode, ContentWorkflowDescription wfDescription)
531    {
532        List<Expression> expList = getExpressionsList(lang, syncCode, wfDescription.getContentType(), catalog);
533        AndExpression andExp = new AndExpression(expList.toArray(new Expression[expList.size()]));
534        String xPathQuery = ContentQueryHelper.getContentXPathQuery(andExp);
535
536        AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(xPathQuery);
537
538        if (contents.getSize() > 0)
539        {
540            return contents.iterator().next();
541        }
542        
543        return null;
544    }
545    
546    /**
547     * Additional parameters for specific treatments.
548     * @param parameters The parameters map to get
549     */
550    protected abstract void additionalParameters(Map<String, Object> parameters);
551    
552    /**
553     * Additional operation to do on the content before saving it.
554     * @param content The content
555     * @param logger The logger
556     * @throws RepositoryException if an error occurs
557     */
558    protected abstract void additionalOperationsBeforeSave(ModifiableDefaultContent content, Logger logger) throws RepositoryException;
559    
560    /**
561     * Import or synchronize the content.
562     * @param doc XML document
563     * @param contentNode Node of the content
564     * @param wfDescription The workflow description
565     * @param title The title
566     * @param lang The lang
567     * @param catalog The catalog
568     * @param syncCode The synchronization code
569     * @param logger The logger
570     * @return The imported or synchronized content
571     */
572    protected ModifiableDefaultContent _importOrSynchronizeContent(Document doc, Node contentNode, ContentWorkflowDescription wfDescription, String title, String lang, String catalog, String syncCode, Logger logger)
573    {
574        ModifiableDefaultContent content = null;
575        try
576        {
577            content = _getOrCreateContent(title, lang, catalog, syncCode, wfDescription, logger);
578            if (content != null)
579            {
580                _synchronizeContent(doc, contentNode, content, wfDescription.getContentType(), lang, catalog, syncCode, logger);
581            }
582        }
583        catch (RepositoryException e)
584        {
585            _nbError++;
586            logger.error("An error occurred while importing or synchronizing content", e);
587        }
588        return content;
589    }
590    
591    /**
592     * Synchronize content 
593     * @param doc The root document
594     * @param contentNode the DOM content node
595     * @param content The content to synchronize
596     * @param contentTypeId The content type ID
597     * @param lang Parent program language (to select the good courses in the CDM-FR file)
598     * @param catalog The catalog of parent program
599     * @param syncCode The synchronization code
600     * @param logger The logger
601     */
602    protected void _synchronizeContent(Document doc, Node contentNode, ModifiableDefaultContent content, String contentTypeId, String lang, String catalog, String syncCode, Logger logger)
603    {
604        if (_updatedContents.add(content.getId()))
605        {
606            logger.info("Synchronization of the content '{}' with the content type '{}'", content.getTitle(), contentTypeId);
607            
608            List<ModifiableDefaultContent> children = new LinkedList<>();
609            List<ModifiableDefaultContent> courseParts = new LinkedList<>();
610            boolean removeOldCourseParts = false;
611            
612            if (content.isLocked())
613            {
614                logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner());
615            }
616            else
617            {
618                boolean hasChanges = false;
619                ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
620                int courseListPosition = 0;
621                
622                NodeList metadataNodes = contentNode.getChildNodes();
623                for (int i = 0; i < metadataNodes.getLength(); i++)
624                {
625                    Node metadataNode = metadataNodes.item(i);
626                    String metadataName = metadataNode.getLocalName();
627                    
628                    if (metadataName.equals(_TAG_SUBPROGRAM))
629                    {
630                        String subContentSyncCode = _xPathProcessor.evaluateAsString(metadataNode, "@CDMid");
631                        String title = _xPathProcessor.evaluateAsString(metadataNode, "title");
632                        ModifiableDefaultContent subProgram = _importOrSynchronizeContent(doc, metadataNode, getSubProgramWfDescription(), title, lang, catalog, subContentSyncCode, logger);
633                        CollectionUtils.addIgnoreNull(children, subProgram);
634                    }
635                    else if (metadataName.equals(_TAG_CONTAINER))
636                    {
637                        String subContentSyncCode = _xPathProcessor.evaluateAsString(metadataNode, "@CDMid");
638                        String title = _xPathProcessor.evaluateAsString(metadataNode, "title");
639                        ModifiableDefaultContent container = _importOrSynchronizeContent(doc, metadataNode, getContainerWfDescription(), title, lang, catalog, subContentSyncCode, logger);
640                        CollectionUtils.addIgnoreNull(children, container);
641                    }
642                    else if (metadataName.equals(_TAG_COURSELIST))
643                    {
644                        courseListPosition++;
645                        // For courseList from another source than Ametys, there is no unique code, then a code is generated with the parent ID and the position in the parent : [parentId]-[position]
646                        String subContentSyncCode = _getXPathString(metadataNode, "@code", syncCode + "-" + courseListPosition);
647                        String title = _getXPathString(metadataNode, "@name", "Liste d'éléments pédagogiques");
648                        ModifiableDefaultContent courseList = _importOrSynchronizeContent(doc, metadataNode, getCourseListWfDescription(), title, lang, catalog, subContentSyncCode, logger);
649                        if (courseList != null)
650                        {
651                            _synchronizeCourseList(doc, metadataNode, courseList, lang, catalog, logger);
652                            children.add(courseList);
653                        }
654                    }
655                    else if (metadataName.equals(_TAG_COURSEPART))
656                    {
657                        String subContentSyncCode = _xPathProcessor.evaluateAsString(metadataNode, "code");
658                        if (StringUtils.isEmpty(subContentSyncCode))
659                        {
660                            subContentSyncCode = org.ametys.core.util.StringUtils.generateKey().toUpperCase();
661                            removeOldCourseParts = true;
662                        }
663                        String title = _xPathProcessor.evaluateAsString(metadataNode, "title");
664                        ModifiableDefaultContent coursePart = _importOrSynchronizeContent(doc, metadataNode, getCoursePartWfDescription(), title, lang, catalog, subContentSyncCode, logger);
665                        CollectionUtils.addIgnoreNull(courseParts, coursePart);
666                    }
667                    // Explicitely ignore catalog metadata which is forced at content creation
668                    else if (!metadataName.equals(ProgramItem.METADATA_CATALOG))
669                    {
670                        hasChanges = _synchronizeMetadata(doc, metadataNode, content, metadataName, metadataName, contentType, lang, catalog, logger) || hasChanges;
671                    }
672                }
673                
674                hasChanges = _setRelations(content, children, courseParts, removeOldCourseParts, logger) || hasChanges;
675                
676                // Create translation links
677                _linkTranslationsIfExist(content, contentTypeId);
678                
679                _saveContentChanges(content, contentTypeId, hasChanges, logger);
680            }
681        }
682    }
683
684    /**
685     * Synchronize a course list, it has attributes to synchronize.
686     * @param doc The XML document
687     * @param courseListNode The XML node of the course list
688     * @param courseList The course list content
689     * @param lang The lang
690     * @param catalog The catalog
691     * @param logger The logger
692     */
693    protected void _synchronizeCourseList(Document doc, Node courseListNode, ModifiableDefaultContent courseList, String lang, String catalog, Logger logger)
694    {
695        boolean hasChanges = false;
696        
697        ContentType contentType = _contentTypeEP.getExtension(getCourseListWfDescription().getContentType());
698        NamedNodeMap attributes = courseListNode.getAttributes();
699        for (int i = 0; i < attributes.getLength(); i++)
700        {
701            Node attributeNode = attributes.item(i);
702            String attributeName = attributeNode.getLocalName();
703            // Explicitely ignore catalog metadata which is forced at content creation
704            if (!attributeName.equals(ProgramItem.METADATA_CATALOG))
705            {
706                hasChanges = _synchronizeMetadata(doc, attributeNode, courseList, attributeName, attributeName, contentType, lang, catalog, logger) || hasChanges;
707            }
708        }
709
710        List<ModifiableDefaultContent> courses = new LinkedList<>();
711        NodeList itemNodes = courseListNode.getChildNodes();
712        for (int i = 0; i < itemNodes.getLength(); i++)
713        {
714            String syncCode = itemNodes.item(i).getTextContent().trim();
715            
716            Node courseNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "course[@CDMid = '" + syncCode + "' and @language = '" + lang + "']");
717            if (courseNode == null)
718            {
719                courseNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "course[@CDMid = '" + syncCode + "']");
720            }
721            
722            if (courseNode != null)
723            {
724                String elpLang = _getXPathString(courseNode, "@language", lang);
725
726                // Check the catalog is the same as the parent program
727                String courseCatalog = getCatalogName(courseNode);
728                if (courseCatalog != null && !courseCatalog.equals(catalog))
729                {
730                    String cdmCode = _xPathProcessor.evaluateAsString(courseNode, "cdmCode");
731                    logger.error("The course '{}' belongs to a different catalog than the one from the imported/synchronized program : '{}' vs '{}'. No synchronization will be done on this course.", cdmCode, courseCatalog, catalog);
732                }
733                else
734                {
735                    String title = _xPathProcessor.evaluateAsString(courseNode, "title");
736                    ModifiableDefaultContent course = _importOrSynchronizeContent(doc, courseNode, getCourseWfDescription(), title, elpLang, catalog, syncCode, logger);
737                    if (course != null)
738                    {
739                        courses.add(course);
740                        hasChanges = _shareableCourseHelper.initializeShareableFields((Course) course, (CourseList) courseList,  UserPopulationDAO.SYSTEM_USER_IDENTITY, true) || hasChanges;
741                    }
742                }
743            }
744        }
745        
746        if (!courses.isEmpty())
747        {
748            hasChanges = ExternalizableMetadataHelper.setMetadata(courseList.getMetadataHolder(), CourseList.METADATA_CHILD_COURSES, courses.toArray(new ModifiableDefaultContent[courses.size()])) || hasChanges;
749            for (ModifiableDefaultContent course : courses)
750            {
751                hasChanges = _synchroComponent.updateRelation(course.getMetadataHolder(), Course.METADATA_PARENT_COURSE_LISTS, courseList, false) || hasChanges;
752            }
753        }
754        
755        _saveContentChanges(courseList, getCourseListWfDescription().getContentType(), hasChanges, logger);
756    }
757
758    private void _fetchImages(Node docbookNode, ModifiableDefaultContent content, ModifiableRichText richText, String metadataPath, Logger logger)
759    {
760        NodeList unfetchUrls = _xPathProcessor.selectNodeList(docbookNode, ".//docbook:mediaobject/docbook:imageobject/docbook:imagedata", _PREFIX_RESOLVER);
761        
762        for (int i = 0; i < unfetchUrls.getLength(); i++)
763        {
764            Element href = (Element) unfetchUrls.item(i);
765            String imageUrl = href.getAttribute("fileref");
766            
767            if (DefaultCommentManagerExtensionPoint.URL_VALIDATOR.matcher(imageUrl).matches())
768            {
769                @SuppressWarnings("resource")
770                InputStream is = null;
771                try
772                {
773                    URL url = new URL(imageUrl);
774                    is = url.openStream();
775                    
776                    String path = url.getPath();
777                    String fileName = path.substring(path.lastIndexOf("/") + 1);
778                    
779                    ModifiableFile file = null;
780                    if (richText.getAdditionalDataFolder().hasFile(fileName))
781                    {
782                        file = richText.getAdditionalDataFolder().getFile(fileName);
783                    }
784                    else
785                    {
786                        file = richText.getAdditionalDataFolder().addFile(fileName); 
787                    }
788                    
789                    ModifiableResource resource = file.getResource();
790                    resource.setInputStream(is);
791                    resource.setLastModified(new Date());
792                    
793                    String mimeType = _cocoonContext.getMimeType(fileName);
794                    if (mimeType == null)
795                    {
796                        mimeType = "application/unknown";
797                    }
798                    resource.setMimeType(mimeType);
799                    
800                    String fetchUrl = content.getId() + "@" + metadataPath + ";" + fileName;
801                    href.setAttribute("fileref", fetchUrl);
802                }
803                catch (Exception e)
804                {
805                    logger.warn("Unable to retrieve remote image '{}'.", imageUrl, e);
806                }
807                finally
808                {
809                    IOUtils.closeQuietly(is);
810                }
811            }
812        }
813    }
814    
815    /**
816     * Synchronize a metadata (can be a composite or a repeater).
817     * @param doc The XML document
818     * @param metadataNode The metadata node
819     * @param content The content
820     * @param logicalMetadataPath The logical metadata path (to retrieve the definition)
821     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
822     * @param contentType The content type
823     * @param lang The lang
824     * @param catalog The catalog
825     * @param logger The logger
826     * @return <code>true</code> if changes occurs
827     */
828    protected boolean _synchronizeMetadata(Document doc, Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, ContentType contentType, String lang, String catalog, Logger logger)
829    {
830        List<String> metadataValues = null;
831        MetadataDefinition metadataDef = contentType.getMetadataDefinitionByPath(logicalMetadataPath);
832        Map<String, Object> params = ImmutableMap.of("contentType", contentType.getId());
833        boolean synchronize = getLocalAndExternalFields(params).contains(logicalMetadataPath);
834        if (metadataDef != null)
835        {
836            if (metadataDef instanceof RepeaterDefinition)
837            {
838                return _handleRepeaterMetadata(doc, metadataNode, content, logicalMetadataPath, completeMetadataPath, contentType, lang, catalog, logger);
839            }
840            else if (metadataDef.getType() == MetadataType.COMPOSITE)
841            {
842                return _handleCompositeMetadata(doc, metadataNode, content, logicalMetadataPath, completeMetadataPath, contentType, lang, catalog, logger);
843            }
844            else if (metadataDef.getType() == MetadataType.RICH_TEXT)
845            {
846                return _handleRichTextMetadata(content, logicalMetadataPath, completeMetadataPath, metadataNode, synchronize, logger);
847            }
848            else if (metadataDef.getType() == MetadataType.BINARY)
849            {
850                return _handleBinaryMetadata(metadataNode, content, completeMetadataPath, synchronize, logger);
851            }
852            else if (metadataDef.getType() == MetadataType.FILE)
853            {
854                return _handleFileMetadata(metadataNode, content, logicalMetadataPath, completeMetadataPath, synchronize, contentType, logger);
855            }
856            else if (metadataDef.getType() == MetadataType.GEOCODE)
857            {
858                return _handleGeocodeMetadata(metadataNode, content, completeMetadataPath, synchronize);
859            }
860            else if (metadataDef.isMultiple())
861            {
862                NodeList itemNodes = metadataNode.getChildNodes();
863                metadataValues = new ArrayList<>();
864                for (int j = 0; j < itemNodes.getLength(); j++)
865                {
866                    String metadataValue = itemNodes.item(j).getTextContent().trim();
867                    if (StringUtils.isNotEmpty(metadataValue))
868                    {
869                        metadataValues.add(metadataValue);
870                    }
871                }
872            }
873            else
874            {
875                String metadataValue = metadataNode.getTextContent().trim();
876                if (StringUtils.isNotEmpty(metadataValue))
877                {
878                    metadataValues = ImmutableList.of(metadataValue);
879                }
880            }
881        }
882
883        Map<String, Boolean> resultMap = _synchroComponent.synchronizeMetadata(content, contentType, logicalMetadataPath, completeMetadataPath, _handleMetadataValues(doc, content, metadataDef, metadataValues, lang, catalog, logger), synchronize, _importedContents.containsKey(content.getId()), logger);
884        if (resultMap.getOrDefault("error", Boolean.FALSE))
885        {
886            _nbError++;
887        }
888        return resultMap.getOrDefault("hasChanges", Boolean.FALSE).booleanValue();
889    }
890    
891    /**
892     * Synchronize geocode metadata
893     * @param metadataNode The metadata node
894     * @param content The content
895     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
896     * @param synchronize true if the data is external
897     * @return true if some changes has been made
898     */
899    protected boolean _handleGeocodeMetadata(Node metadataNode, ModifiableDefaultContent content, String completeMetadataPath, boolean synchronize)
900    {
901        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
902        String[] arrayPath = completeMetadataPath.split("/");
903        String metadataName = arrayPath[arrayPath.length - 1];
904        
905        return _handleGeocodeMetadata(metadataNode, content, metadataHolder, metadataName, synchronize);
906    }
907    
908    /**
909     * Synchronize geocode metadata
910     * @param metadataNode The metadata node
911     * @param content The content
912     * @param metadataHolder the metadata holder
913     * @param metadataName the metadata name
914     * @param synchronize true if the data is external
915     * @return true if some changes has been made
916     */
917    protected boolean _handleGeocodeMetadata(Node metadataNode, ModifiableDefaultContent content, ModifiableCompositeMetadata metadataHolder, String metadataName, boolean synchronize)
918    {
919        if (!metadataNode.hasChildNodes())
920        {
921            return _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize);
922        }
923
924        boolean hasChanges = false;
925        ExternalizableMetadataStatus status = synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL;
926        ModifiableCompositeMetadata geoCode = ExternalizableMetadataHelper.getCompositeMetadata(metadataHolder, metadataName, status, true);
927        if (synchronize && _importedContents.containsKey(content.getId()))
928        {
929            hasChanges = ExternalizableMetadataHelper.updateStatus(metadataHolder, metadataName, status) || hasChanges;
930        }
931        
932        NodeList coordinates = metadataNode.getChildNodes();
933        for (int j = 0; j < coordinates.getLength(); j++)
934        {
935            Node coordinate = coordinates.item(j);
936            hasChanges = ExternalizableMetadataHelper.setMetadata(geoCode, coordinate.getLocalName(), coordinate.getTextContent().trim()) || hasChanges;
937        }
938        return hasChanges;
939    }
940
941    /**
942     * Synchronize file metadata
943     * @param metadataNode The metadata node
944     * @param content The content
945     * @param logicalMetadataPath The logical metadata path (to retrieve the definition) 
946     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
947     * @param synchronize true if the data is external
948     * @param contentType the content type
949     * @param logger the logger
950     * @return true if some changes has been made
951     */
952    protected boolean _handleFileMetadata(Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, boolean synchronize, ContentType contentType, Logger logger)
953    {
954        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
955        String[] arrayPath = completeMetadataPath.split("/");
956        String metadataName = arrayPath[arrayPath.length - 1];
957        
958        return _handleFileMetadata(metadataNode, content, logicalMetadataPath, metadataHolder, metadataName, synchronize, contentType, logger);
959    }
960    
961    /**
962     * Synchronize file metadata
963     * @param metadataNode The metadata node
964     * @param content The content
965     * @param logicalMetadataPath The logical metadata path (to retrieve the definition) 
966     * @param metadataHolder the metadata holder
967     * @param metadataName the metadata name
968     * @param synchronize true if the data is external
969     * @param contentType the content type
970     * @param logger the logger
971     * @return true if some changes has been made
972     */
973    protected boolean _handleFileMetadata(Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, ModifiableCompositeMetadata metadataHolder, String metadataName, boolean synchronize, ContentType contentType, Logger logger)
974    {
975        boolean hasChanges = false;
976        String value = metadataNode.getTextContent().trim();
977        boolean metadataExists = metadataHolder.hasMetadata(metadataName);
978        CompositeMetadata.MetadataType metadataType = metadataExists ? metadataHolder.getType(metadataName) : null;
979        if (DefaultCommentManagerExtensionPoint.URL_VALIDATOR.matcher(value).matches())
980        {
981            if (metadataExists && metadataType != CompositeMetadata.MetadataType.BINARY)
982            {
983                hasChanges = _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize) || hasChanges;
984            }
985            hasChanges = _handleBinaryMetadata(metadataNode, content, metadataHolder, metadataName, synchronize, logger) || hasChanges;
986        }
987        else
988        {
989            if (metadataExists && metadataType != CompositeMetadata.MetadataType.STRING)
990            {
991                hasChanges = _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize) || hasChanges;
992            }
993            
994            Map<String, Boolean> resultMap = _synchroComponent.synchronizeMetadata(content, contentType, logicalMetadataPath, metadataHolder, metadataName, ImmutableList.of(value), synchronize, _importedContents.containsKey(content.getId()), logger);
995            if (resultMap.getOrDefault("error", Boolean.FALSE))
996            {
997                _nbError++;
998            }
999            hasChanges = resultMap.getOrDefault("hasChanges", Boolean.FALSE).booleanValue() || hasChanges;
1000        }
1001        return hasChanges;
1002    }
1003    
1004    /**
1005     * Synchronize binary metadata
1006     * @param metadataNode The metadata node
1007     * @param content The content
1008     * @param completeMetadataPath The complete metadata path (to retrieve the metadata holder)
1009     * @param synchronize true if the data is external
1010     * @param logger the logger
1011     * @return true if some changes has been made
1012     */
1013    protected boolean _handleBinaryMetadata(Node metadataNode, ModifiableDefaultContent content, String completeMetadataPath, boolean synchronize, Logger logger)
1014    {
1015        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
1016        String[] arrayPath = completeMetadataPath.split("/");
1017        String metadataName = arrayPath[arrayPath.length - 1];
1018        
1019        return _handleBinaryMetadata(metadataNode, content, metadataHolder, metadataName, synchronize, logger);
1020    }
1021    
1022    /**
1023     * Synchronize binary metadata
1024     * @param metadataNode The metadata node
1025     * @param content The content
1026     * @param metadataHolder the metadata holder
1027     * @param metadataName the metadata name
1028     * @param synchronize true if the data is external
1029     * @param logger the logger
1030     * @return true if some changes has been made
1031     */
1032    protected boolean _handleBinaryMetadata(Node metadataNode, ModifiableDefaultContent content, ModifiableCompositeMetadata metadataHolder, String metadataName, boolean synchronize, Logger logger)
1033    {
1034        boolean hasChanges = false;
1035        
1036        String value = metadataNode.getTextContent().trim();
1037        
1038        if (StringUtils.isEmpty(value))
1039        {
1040            return _synchroComponent.removeMetadataIfExists(metadataHolder, metadataName, synchronize);
1041        }
1042        
1043        try
1044        {
1045            URL url = new URL(value);
1046            
1047            byte[] bytes;
1048            try (InputStream is = url.openStream())
1049            {
1050                bytes = IOUtils.toByteArray(is);
1051            }
1052
1053            ExternalizableMetadataStatus status = synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL;
1054            ModifiableBinaryMetadata binaryMetadata = ExternalizableMetadataHelper.getBinaryMetadata(metadataHolder, metadataName, status, true);
1055
1056            byte[] oldValue = new byte[0];
1057            try (InputStream is = binaryMetadata.getInputStream())
1058            {
1059                oldValue = IOUtils.toByteArray(is);
1060            }
1061            catch (AmetysRepositoryException e)
1062            {
1063                logger.debug("The old value of '{}' should not be initialized on the content '{}'.", metadataName, content.getId(), e);
1064            }
1065
1066            if (!Objects.deepEquals(bytes, oldValue))
1067            {
1068                String path = url.getPath();
1069                String filename = path.substring(path.lastIndexOf("/") + 1);
1070                String mimeType = _cocoonContext.getMimeType(filename);
1071                if (mimeType == null)
1072                {
1073                    mimeType = "application/unknown";
1074                }
1075                binaryMetadata.setFilename(URLDecoder.decode(filename, "UTF-8"));
1076                binaryMetadata.setMimeType(mimeType);
1077                binaryMetadata.setLastModified(new Date());
1078                binaryMetadata.setInputStream(new ByteArrayInputStream(bytes));
1079    
1080                hasChanges = true;
1081            }
1082
1083            if (synchronize && _importedContents.containsKey(content.getId()))
1084            {
1085                hasChanges = ExternalizableMetadataHelper.updateStatus(metadataHolder, metadataName, status) || hasChanges;
1086            }
1087        }
1088        catch (IOException e)
1089        {
1090            logger.error("Unable to retrieve remote file input stream", e);
1091        }
1092
1093        return hasChanges;
1094    }
1095    
1096    private boolean _handleRepeaterMetadata(Document doc, Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, ContentType contentType, String lang, String catalog, Logger logger)
1097    {
1098        boolean hasChanges = false;
1099        
1100        ModifiableCompositeMetadata repeater = content.getMetadataHolder().getCompositeMetadata(completeMetadataPath, true);
1101        
1102        // Remove locale entries (unable to compare locales and remotes ...)
1103        String[] metadataNames = repeater.getMetadataNames();
1104        for (String entryName : metadataNames)
1105        {
1106            repeater.removeMetadata(entryName);
1107            hasChanges = true;
1108        }
1109        
1110        // Create new entries from remote data
1111        NodeList entryNodes = metadataNode.getChildNodes();
1112        for (int i = 0; i < entryNodes.getLength(); i++)
1113        {
1114            Node entryNode = entryNodes.item(i);
1115            String entryName = entryNode.getAttributes().getNamedItem("name").getTextContent().trim();
1116            
1117            NodeList childNodes = entryNode.getChildNodes();
1118            for (int j = 0; j < childNodes.getLength(); j++)
1119            {
1120                Node childNode = childNodes.item(j);
1121                String subMetadataName = childNode.getLocalName();
1122                
1123                hasChanges = _synchronizeMetadata(doc, childNode, content, logicalMetadataPath + "/" + subMetadataName, completeMetadataPath + "/" + entryName + "/" + subMetadataName, contentType, lang, catalog, logger) || hasChanges;
1124            }
1125        }
1126        
1127        return hasChanges;
1128    }
1129    
1130    private boolean _handleCompositeMetadata(Document doc, Node metadataNode, ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, ContentType contentType, String lang, String catalog, Logger logger)
1131    {
1132        boolean hasChanges = false;
1133        NodeList subMetadataNodes = metadataNode.getChildNodes();
1134        for (int i = 0; i < subMetadataNodes.getLength(); i++)
1135        {
1136            Node subMetadataNode = subMetadataNodes.item(i);
1137            hasChanges = _synchronizeMetadata(doc, subMetadataNode, content, logicalMetadataPath + "/" + subMetadataNode.getLocalName(), completeMetadataPath + "/" + subMetadataNode.getLocalName(), contentType, lang, catalog, logger) || hasChanges;
1138        }
1139        return hasChanges;
1140    }
1141    
1142    private boolean _handleRichTextMetadata(ModifiableDefaultContent content, String logicalMetadataPath, String completeMetadataPath, Node metadataNode, boolean synchronize, Logger logger)
1143    {
1144        ModifiableCompositeMetadata metadataHolder = _synchroComponent.getMetadataHolder(content.getMetadataHolder(), completeMetadataPath);
1145        String[] arrayPath = completeMetadataPath.split("/");
1146        String metadataName = arrayPath[arrayPath.length - 1];
1147
1148        if (metadataNode.hasChildNodes())
1149        {
1150            boolean hasChanges = false;
1151            try
1152            {
1153                String docbook = ContentImporterHelper.serializeNode(metadataNode.getFirstChild());
1154                try (ByteArrayInputStream is =  new ByteArrayInputStream(docbook.getBytes("UTF-8")))
1155                {
1156                    ExternalizableMetadataStatus status = synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL;
1157                    ModifiableRichText richText = ExternalizableMetadataHelper.getRichText(metadataHolder, metadataName, status, true);
1158                    
1159                    _fetchImages(metadataNode.getFirstChild(), content, richText, completeMetadataPath, logger);
1160                    richText.setInputStream(is);
1161                    richText.setMimeType("text/xml");
1162                    richText.setLastModified(new Date()); 
1163
1164                    if (synchronize && _importedContents.containsKey(content.getId()))
1165                    {
1166                        hasChanges = ExternalizableMetadataHelper.updateStatus(metadataHolder, metadataName, status) || hasChanges;
1167                    }
1168                    
1169                    hasChanges = true;
1170                }
1171                catch (IOException e)
1172                {
1173                    logger.error("An error occured while parsing the rich text '{}' of the content '{}'", logicalMetadataPath, content.getTitle(), e);
1174                }
1175            }
1176            catch (TransformerException e)
1177            {
1178                logger.error("Error serializing a docbook node.", e);
1179            }
1180            return hasChanges;
1181        }
1182        
1183        return ExternalizableMetadataHelper.removeMetadataIfExists(metadataHolder, metadataName);
1184    }
1185    
1186    /**
1187     * Handle metadata values
1188     * @param doc the document
1189     * @param content the content
1190     * @param metadataDef the metadata definition
1191     * @param metadataValues the metadata values
1192     * @param lang the language
1193     * @param catalog the catalog
1194     * @param logger the logger
1195     * @return the list of handle values
1196     */
1197    protected List<Object> _handleMetadataValues(Document doc, ModifiableDefaultContent content, MetadataDefinition metadataDef, List<String> metadataValues, String lang, String catalog, Logger logger)
1198    {
1199        if (metadataValues == null)
1200        {
1201            return null;
1202        }
1203        
1204        List<Object> finalMetadataValues = new ArrayList<>();
1205        for (String metadataValue : metadataValues)
1206        {
1207            switch (metadataDef.getType())
1208            {
1209                case DATE:
1210                    try
1211                    {
1212                        finalMetadataValues.add(CDMHelper.CDM_DATE_FORMAT.parse(metadataValue));
1213                    }
1214                    catch (ParseException e)
1215                    {
1216                        logger.warn("Unable to parse the metadata '{}' of the content '{}' with the value '{}', it should respect the following format '{}'.", metadataDef.getName(), content.getId(), metadataValue, CDMHelper.CDM_DATE_FORMAT);
1217                    }
1218                    break;
1219                case BOOLEAN:
1220                    finalMetadataValues.add(Boolean.valueOf(metadataValue));
1221                    break;
1222                case CONTENT:
1223                    Object value = _handleMetadataValuesAsContent(doc, metadataDef, metadataValue, lang, catalog, logger);
1224                    if (value != null)
1225                    {
1226                        finalMetadataValues.add(value);
1227                    }
1228                    break;
1229                default:
1230                    if (content instanceof CourseList && metadataDef.getName().equals(CourseList.METADATA_CHOICE_TYPE))
1231                    {
1232                        finalMetadataValues.add(metadataValue.toUpperCase());
1233                    }
1234                    else
1235                    {
1236                        finalMetadataValues.add(metadataValue);
1237                    }
1238                    break;
1239            }
1240        }
1241        
1242        return finalMetadataValues;
1243    }
1244    
1245    private Object _handleMetadataValuesAsContent(Document doc, MetadataDefinition metadataDef, String metadataValue, String lang, String catalog, Logger logger)
1246    {
1247        String refContentTypeId = metadataDef.getContentType();
1248        if (refContentTypeId != null)
1249        {
1250            ContentType refContentType = _contentTypeEP.getExtension(refContentTypeId);
1251            if (refContentTypeId.equals(PersonFactory.PERSON_CONTENT_TYPE))
1252            {
1253                Node personNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "person[@CDMid ='" + metadataValue + "']");
1254                ModifiableDefaultContent person = _importOrSynchronizeContent(doc, personNode, getPersonWfDescription(), metadataValue, lang, null, metadataValue, logger);
1255                if (person != null)
1256                {
1257                    return person;
1258                }
1259            }
1260            else if (refContentTypeId.equals(OrgUnitFactory.ORGUNIT_CONTENT_TYPE))
1261            {
1262                Node ouNode = _xPathProcessor.selectSingleNode(doc.getFirstChild(), "orgunit[@CDMid ='" + metadataValue + "']");
1263                ModifiableDefaultContent orgUnit = _importOrSynchronizeContent(doc, ouNode, getOrgUnitWfDescription(), metadataValue, lang, null, metadataValue, logger);
1264                if (orgUnit != null)
1265                {
1266                    return orgUnit;
1267                }
1268            }
1269            else if (metadataDef.getName().equals("courseHolder"))
1270            {
1271                List<Expression> expList = getExpressionsList(lang, metadataValue, getCourseWfDescription().getContentType(), catalog);
1272                AndExpression andExp = new AndExpression(expList.toArray(new Expression[expList.size()]));
1273                String xPathQuery = ContentQueryHelper.getContentXPathQuery(andExp);
1274                AmetysObjectIterable<CoursePart> contents = _resolver.query(xPathQuery);
1275                if (contents.getSize() > 0)
1276                {
1277                    return contents.iterator().next();
1278                }
1279                else
1280                {
1281                    logger.warn("There is no course part corresponding to the CDM ID '{}'.", metadataValue);
1282                }
1283            }
1284            else if (refContentType.isReferenceTable() && _odfRefTableHelper.isTableReference(refContentTypeId))
1285            {
1286                String entryId = _getIdFromCDMThenCode(refContentTypeId, metadataValue);
1287                if (StringUtils.isNotEmpty(entryId))
1288                {
1289                    return _resolver.resolveById(entryId);
1290                }
1291                else
1292                {
1293                    logger.warn("There is no entry corresponding to the CDM-fr or Ametys code '{}' in the reference table '{}'.", metadataValue, refContentTypeId);
1294                }
1295            }
1296        }
1297        else
1298        {
1299            logger.warn("Cannot match data '{}' of content type '{}' because it is not typed.", metadataDef.getName(), metadataDef.getReferenceContentType());
1300        }
1301        
1302        return null;
1303    }
1304
1305    /**
1306     * Search for translated contents
1307     * @param importedContent The imported content
1308     * @param contentType The content type
1309     */
1310    protected void _linkTranslationsIfExist(ModifiableContent importedContent, String contentType)
1311    {
1312        if (importedContent instanceof ProgramItem)
1313        {
1314            Map<String, String> translations = new HashMap<>();
1315    
1316            List<Expression> expList = getExpressionsList(importedContent.getLanguage(), importedContent.getMetadataHolder().getString(getIdField()), contentType, importedContent.getMetadataHolder().getString(ProgramItem.METADATA_CATALOG, null));
1317            AndExpression andExp = new AndExpression(expList.toArray(new Expression[expList.size()]));
1318            String xPathQuery = ContentQueryHelper.getContentXPathQuery(andExp);
1319    
1320            AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(xPathQuery);
1321            
1322            for (ModifiableDefaultContent content : contents)
1323            {
1324                translations.put(content.getLanguage(), content.getId());
1325                
1326                Map<String, String> translations2 = TranslationHelper.getTranslations(content);
1327                translations2.put(importedContent.getLanguage(), importedContent.getId());
1328                TranslationHelper.setTranslations(content, translations2);
1329    
1330                Map<String, Object> eventParams = new HashMap<>();
1331                eventParams.put(ObservationConstants.ARGS_CONTENT, content);
1332                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId());
1333                _observationManager.notify(new Event(OdfObservationConstants.ODF_CONTENT_TRANSLATED, _currentUserProvider.getUser(), eventParams));
1334            }
1335            
1336            TranslationHelper.setTranslations(importedContent, translations);
1337        }
1338    }
1339
1340    /**
1341     * Save the changes of the content.
1342     * @param content Content to save
1343     * @param contentTypeId The content type (for logs)
1344     * @param hasChanges If there are changes to save
1345     * @param logger The logger
1346     */
1347    protected void _saveContentChanges(ModifiableDefaultContent content, String contentTypeId, boolean hasChanges, Logger logger)
1348    {
1349        if (hasChanges)
1350        {
1351            content.saveChanges();
1352            _synchronizedContents.add(content.getId());
1353            if (!_importedContents.containsKey(content.getId()))
1354            {
1355                _nbSynchronizedContents++;
1356            }
1357        }
1358        else
1359        {
1360            logger.info("No changes detected for content '{}' with the content type '{}'", content.getTitle(), contentTypeId);
1361            if (!_importedContents.containsKey(content.getId()))
1362            {
1363                _nbNotChangedContents++;
1364            }
1365        }
1366    }
1367    
1368    /**
1369     * Set relations for the content.
1370     * @param content The content to update
1371     * @param children Children to set
1372     * @param courseParts {@link List} of {@link CoursePart} to set for {@link Course} content
1373     * @param removeOldCourseParts If the course parts doesn't have a code, we remove all the old {@link CoursePart}s before adding the new ones
1374     * @param logger The logger
1375     * @return <code>true</code> if changes occurs
1376     */
1377    protected boolean _setRelations(ModifiableDefaultContent content, List<ModifiableDefaultContent> children, List<ModifiableDefaultContent> courseParts, boolean removeOldCourseParts, Logger logger)
1378    {
1379        boolean hasChanges = false;
1380        
1381        if (!courseParts.isEmpty() && content instanceof Course)
1382        {
1383            if (removeOldCourseParts)
1384            {
1385                hasChanges = ExternalizableMetadataHelper.removeMetadataIfExists(content.getMetadataHolder(), Course.METADATA_CHILD_COURSE_PARTS) || hasChanges;
1386            }
1387            hasChanges = _updateDoubleRelation(content, courseParts, Course.METADATA_CHILD_COURSE_PARTS, CoursePart.METADATA_PARENT_COURSES, logger) || hasChanges;
1388        }
1389        
1390        if (!children.isEmpty())
1391        {
1392            hasChanges = _setChildren(content, children, logger) || hasChanges;
1393        }
1394        
1395        // Set the orgUnit parent (if no parent is set)
1396        if (content instanceof OrgUnit)
1397        {
1398            hasChanges = _setOrgUnitParent(content, logger) || hasChanges;
1399        }
1400        
1401        return hasChanges;
1402    }
1403    
1404    /**
1405     * Set children for the given content.
1406     * @param content Content to add the children
1407     * @param children Children to add
1408     * @param logger The logger
1409     * @return <code>true</code> if changes occurs
1410     */
1411    protected boolean _setChildren(ModifiableDefaultContent content, List<ModifiableDefaultContent> children, Logger logger)
1412    {
1413        boolean hasChanges = false;
1414        
1415        String metadataName = null;
1416        String invertMetadataName = null;
1417        if (content instanceof CourseList)
1418        {
1419            metadataName = CourseList.METADATA_CHILD_COURSES;
1420            invertMetadataName = Course.METADATA_PARENT_COURSE_LISTS;
1421        }
1422        else if (content instanceof TraversableProgramPart)
1423        {
1424            metadataName = TraversableProgramPart.METADATA_CHILD_PROGRAM_PARTS;
1425            invertMetadataName = ProgramPart.METADATA_PARENT_PROGRAM_PARTS;
1426        }
1427        else if (content instanceof Course)
1428        {
1429            metadataName = Course.METADATA_CHILD_COURSE_LISTS;
1430            invertMetadataName = CourseList.METADATA_PARENT_COURSES;
1431        }
1432        
1433        if (metadataName != null)
1434        {
1435            hasChanges = _updateDoubleRelation(content, children, metadataName, invertMetadataName, logger);
1436        }
1437        
1438        return hasChanges;
1439    }
1440    
1441    /**
1442     * Set the orgUnit parent to rootOrgUnit.
1443     * @param orgUnit The orgunit to link
1444     * @param logger The logger
1445     * @return <code>true</code> if changes occurs
1446     */
1447    protected boolean _setOrgUnitParent(ModifiableDefaultContent orgUnit, Logger logger)
1448    {
1449        boolean hasChanges = false;
1450        
1451        // Set the orgUnit parent (if no parent is set)
1452        ModifiableCompositeMetadata holder = orgUnit.getMetadataHolder();
1453        if (!holder.hasMetadata(OrgUnit.METADATA_PARENT_ORGUNIT))
1454        {
1455            OrgUnit rootOrgUnit = _rootOUProvider.getRoot();
1456            
1457            hasChanges = ExternalizableMetadataHelper.setMetadata(holder, OrgUnit.METADATA_PARENT_ORGUNIT, rootOrgUnit) || hasChanges;
1458            hasChanges = _synchroComponent.updateRelation(rootOrgUnit.getMetadataHolder(), OrgUnit.METADATA_CHILD_ORGUNITS, orgUnit, false) || hasChanges;
1459
1460            try
1461            {
1462                _synchroComponent.applyChanges(rootOrgUnit, 22, ObservationConstants.EVENT_CONTENT_MODIFIED, ignoreRights(), logger);
1463            }
1464            catch (RepositoryException e)
1465            {
1466                logger.error("An error occured during updating root org unit after synchronizing the content '{}'.", orgUnit.getId(), logger);
1467            }
1468        }
1469        
1470        return hasChanges;
1471    }
1472
1473    /**
1474     * Get the content ID from the CDM code, if there is no match with the CDM code, then we search with the code.
1475     * If nothing is found we return null. 
1476     * @param tableRefId The reference table ID
1477     * @param cdmCode The CDM code
1478     * @return A content ID or <code>null</code>
1479     */
1480    protected String _getIdFromCDMThenCode(String tableRefId, String cdmCode)
1481    {
1482        OdfReferenceTableEntry entry = _odfRefTableHelper.getItemFromCDM(tableRefId, cdmCode);
1483        if (entry == null)
1484        {
1485            entry = _odfRefTableHelper.getItemFromCode(tableRefId, cdmCode);
1486        }
1487        return entry != null ? entry.getId() : null;
1488    }
1489    
1490    private boolean _updateDoubleRelation(ModifiableDefaultContent content, List<ModifiableDefaultContent> children, String metadataName, String invertMetadataName, Logger logger)
1491    {
1492        boolean hasChanges = false;
1493        ModifiableCompositeMetadata holder = content.getMetadataHolder();
1494        
1495        // "Normal" relation
1496        if (removalSync() || !holder.hasMetadata(metadataName))
1497        {
1498            hasChanges = ExternalizableMetadataHelper.setMetadata(holder, metadataName, children.toArray(new ModifiableDefaultContent[children.size()]));
1499        }
1500        else
1501        {
1502            List<String> oldValues = new ArrayList<>(Arrays.asList(holder.getStringArray(metadataName)));
1503            
1504            for (ModifiableDefaultContent child : children)
1505            {
1506                hasChanges = _synchroComponent.updateRelation(holder, metadataName, child, false) || hasChanges;
1507                oldValues.remove(child.getId());
1508            }
1509            
1510            if (logger.isWarnEnabled())
1511            {
1512                for (String oldValue : oldValues)
1513                {
1514                    // Warn the old contents
1515                    ModifiableDefaultContent child = _resolver.resolveById(oldValue);
1516                    logger.warn("The ODF content '{}' ({}) is not linked anymore to the content '{}', it should be manually removed.", child.getTitle(), oldValue, content.getTitle());
1517                }
1518            }
1519        }
1520        
1521        // Invert relation
1522        for (ModifiableDefaultContent child : children)
1523        {
1524            hasChanges = _synchroComponent.updateRelation(child.getMetadataHolder(), invertMetadataName, content, false) || hasChanges;
1525        }
1526        
1527        return hasChanges;
1528    }
1529    
1530    private String _getXPathString(Node metadataNode, String xPath, String defaultValue)
1531    {
1532        String value = _xPathProcessor.evaluateAsString(metadataNode, xPath);
1533        if (StringUtils.isEmpty(value))
1534        {
1535            value = defaultValue;
1536        }
1537        return value;
1538    }
1539
1540    /**
1541     * If true, bypass the rights check during the import process
1542     * @return True if the rights check are bypassed during the import process
1543     */
1544    protected boolean ignoreRights()
1545    {
1546        return false;
1547    }
1548    
1549    /**
1550     * Get the program workflow description.
1551     * @return A {@link ContentWorkflowDescription} containing informations about the program workflow
1552     */
1553    protected ContentWorkflowDescription getProgramWfDescription()
1554    {
1555        return _PROGRAM_WF_DESCRIPTION;
1556    }
1557
1558    /**
1559     * Get the subprogram workflow description.
1560     * @return A {@link ContentWorkflowDescription} containing informations about the subprogram workflow
1561     */
1562    protected ContentWorkflowDescription getSubProgramWfDescription()
1563    {
1564        return _SUBPROGRAM_WF_DESCRIPTION;
1565    }
1566
1567    /**
1568     * Get the container workflow description.
1569     * @return A {@link ContentWorkflowDescription} containing informations about the container workflow
1570     */
1571    protected ContentWorkflowDescription getContainerWfDescription()
1572    {
1573        return _CONTAINER_WF_DESCRIPTION;
1574    }
1575
1576    /**
1577     * Get the course list workflow description.
1578     * @return A {@link ContentWorkflowDescription} containing informations about the course list workflow
1579     */
1580    protected ContentWorkflowDescription getCourseListWfDescription()
1581    {
1582        return _COURSELIST_WF_DESCRIPTION;
1583    }
1584
1585    /**
1586     * Get the course workflow description.
1587     * @return A {@link ContentWorkflowDescription} containing informations about the course workflow
1588     */
1589    protected ContentWorkflowDescription getCourseWfDescription()
1590    {
1591        return _COURSE_WF_DESCRIPTION;
1592    }
1593
1594    /**
1595     * Get the course part workflow description.
1596     * @return A {@link ContentWorkflowDescription} containing informations about the course part workflow
1597     */
1598    protected ContentWorkflowDescription getCoursePartWfDescription()
1599    {
1600        return _COURSEPART_WF_DESCRIPTION;
1601    }
1602
1603    /**
1604     * Get the orgunit workflow description.
1605     * @return A {@link ContentWorkflowDescription} containing informations about the orgunit workflow
1606     */
1607    protected ContentWorkflowDescription getOrgUnitWfDescription()
1608    {
1609        return _ORGUNIT_WF_DESCRIPTION;
1610    }
1611
1612    /**
1613     * Get the person workflow description.
1614     * @return A {@link ContentWorkflowDescription} containing informations about the person workflow
1615     */
1616    protected ContentWorkflowDescription getPersonWfDescription()
1617    {
1618        return _PERSON_WF_DESCRIPTION;
1619    }
1620    
1621    /**
1622     * Internal object to describe content workflow elements.
1623     */
1624    protected static class ContentWorkflowDescription
1625    {
1626        private String _contentType;
1627        private String _workflowName;
1628        private int _initialActionId;
1629        private int _validationActionId;
1630        
1631        ContentWorkflowDescription(String contentType, String workflowName, int initialActionId, int validationActionId)
1632        {
1633            _contentType = contentType;
1634            _workflowName = workflowName;
1635            _initialActionId = initialActionId;
1636            _validationActionId = validationActionId;
1637        }
1638        
1639        /**
1640         * Get the content type.
1641         * @return the content type ID
1642         */
1643        public String getContentType()
1644        {
1645            return _contentType;
1646        }
1647        
1648        /**
1649         * Get the workflow name.
1650         * @return the workflow name
1651         */
1652        public String getWorkflowName()
1653        {
1654            return _workflowName;
1655        }
1656        
1657        /**
1658         * Get the initial action ID.
1659         * @return the initial action ID
1660         */
1661        public int getInitialActionId()
1662        {
1663            return _initialActionId;
1664        }
1665        
1666        /**
1667         * Get the validation action ID.
1668         * @return the validation action ID
1669         */
1670        public int getValidationActionId()
1671        {
1672            return _validationActionId;
1673        }
1674    }
1675    
1676    private static class DocbookPrefixResolver implements PrefixResolver
1677    {
1678        private Map<String, String> _ns = new HashMap<>();
1679        
1680        public DocbookPrefixResolver()
1681        {
1682            _ns.put("docbook", "http://docbook.org/ns/docbook");
1683        }
1684        
1685        @Override
1686        public String prefixToNamespace(String prefix)
1687        {
1688            return _ns.get(prefix);
1689        }
1690    }
1691    
1692    @Override
1693    public List<Expression> getExpressionsList(String lang, String idValue, String contentType, String catalog)
1694    {
1695        List<Expression> expList = new ArrayList<>();
1696        
1697        if (StringUtils.isNotBlank(contentType))
1698        {
1699            expList.add(new ContentTypeExpression(Operator.EQ, contentType));
1700        }
1701        
1702        if (StringUtils.isNotBlank(idValue))
1703        {
1704            expList.add(new StringExpression(getIdField(), Operator.EQ, idValue));
1705        }
1706        
1707        if (StringUtils.isNotBlank(catalog))
1708        {
1709            expList.add(new StringExpression(ProgramItem.METADATA_CATALOG, Operator.EQ, catalog));
1710        }
1711        
1712        if (StringUtils.isNotBlank(lang))
1713        {
1714            expList.add(new LanguageExpression(Operator.EQ, lang));
1715        }
1716        
1717        return expList;
1718    }
1719
1720    @Override
1721    public Set<String> getLocalAndExternalFields(Map<String, Object> additionalParameters)
1722    {
1723        if (additionalParameters == null || !additionalParameters.containsKey("contentType"))
1724        {
1725            throw new IllegalArgumentException("Content type shouldn't be null.");
1726        }
1727
1728        String contentType = additionalParameters.get("contentType").toString();
1729        Set<String> syncFields = _syncFieldsByContentType.get(contentType);
1730        if (syncFields == null)
1731        {
1732            syncFields = new HashSet<>();
1733            _syncFieldsByContentType.put(contentType, syncFields);
1734        }
1735        return syncFields;
1736    }
1737}