Source code

001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.odfsync.apogee.scc;
017
018import java.io.ByteArrayInputStream;
019import java.io.File;
020import java.io.FileInputStream;
021import java.io.IOException;
022import java.io.InputStream;
023import java.math.BigDecimal;
024import java.sql.Clob;
025import java.sql.SQLException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Collection;
029import java.util.Date;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.LinkedHashMap;
033import java.util.LinkedHashSet;
034import java.util.LinkedList;
035import java.util.List;
036import java.util.Map;
037import java.util.Map.Entry;
038import java.util.Objects;
039import java.util.Optional;
040import java.util.Set;
041import java.util.stream.Collectors;
042import java.util.stream.Stream;
043
044import org.apache.avalon.framework.configuration.Configuration;
045import org.apache.avalon.framework.configuration.ConfigurationException;
046import org.apache.avalon.framework.configuration.DefaultConfigurationBuilder;
047import org.apache.avalon.framework.context.Context;
048import org.apache.avalon.framework.context.ContextException;
049import org.apache.avalon.framework.context.Contextualizable;
050import org.apache.avalon.framework.service.ServiceException;
051import org.apache.avalon.framework.service.ServiceManager;
052import org.apache.cocoon.Constants;
053import org.apache.cocoon.components.ContextHelper;
054import org.apache.cocoon.environment.Request;
055import org.apache.commons.collections4.ListUtils;
056import org.apache.commons.collections4.SetUtils;
057import org.apache.commons.collections4.SetUtils.SetView;
058import org.apache.commons.io.IOUtils;
059import org.apache.commons.lang.StringUtils;
060import org.slf4j.Logger;
061
062import org.ametys.cms.content.external.ExternalizableMetadataHelper;
063import org.ametys.cms.content.external.ExternalizableMetadataProvider.ExternalizableMetadataStatus;
064import org.ametys.cms.contenttype.ContentType;
065import org.ametys.cms.data.ContentValue;
066import org.ametys.cms.repository.ModifiableContent;
067import org.ametys.cms.repository.ModifiableDefaultContent;
068import org.ametys.core.util.JSONUtils;
069import org.ametys.odf.ODFHelper;
070import org.ametys.plugins.contentio.ContentImporterHelper;
071import org.ametys.plugins.contentio.synchronize.AbstractSimpleSynchronizableContentsCollection;
072import org.ametys.plugins.contentio.synchronize.SynchronizableContentsCollection;
073import org.ametys.plugins.contentio.synchronize.SynchronizableContentsCollectionDAO;
074import org.ametys.plugins.odfsync.apogee.ApogeeDAO;
075import org.ametys.plugins.odfsync.apogee.scc.impl.OrgUnitSynchronizableContentsCollection;
076import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
077import org.ametys.plugins.repository.metadata.ModifiableRichText;
078import org.ametys.runtime.config.Config;
079import org.ametys.runtime.i18n.I18nizableText;
080import org.ametys.runtime.model.ModelItem;
081import org.ametys.runtime.model.type.ModelItemTypeConstants;
082
083import com.google.common.base.CharMatcher;
084import com.google.common.collect.ImmutableList;
085import com.google.common.collect.ImmutableMap;
086
087/**
088 * Abstract class for Apogee synchronization
089 */
090public abstract class AbstractApogeeSynchronizableContentsCollection extends AbstractSimpleSynchronizableContentsCollection implements Contextualizable, ApogeeSynchronizableContentsCollection
091{
092    /** Name of parameter holding the data source id */
093    public static final String PARAM_DATASOURCE_ID = "datasourceId";
094    /** Name of parameter holding the administrative year */
095    public static final String PARAM_YEAR = "year";
096    /** Name of parameter holding the adding unexisting children parameter  */
097    public static final String PARAM_ADD_UNEXISTING_CHILDREN = "add-unexisting-children";
098    /** Name of parameter holding the field ID column */
099    protected static final String __PARAM_ID_COLUMN = "idColumn";
100    /** Name of parameter holding the fields mapping */
101    protected static final String __PARAM_MAPPING = "mapping";
102    /** Name of parameter into mapping holding the synchronized property */
103    protected static final String __PARAM_MAPPING_SYNCHRO = "synchro";
104    /** Name of parameter into mapping holding the path of metadata */
105    protected static final String __PARAM_MAPPING_METADATA_REF = "metadata-ref";
106    /** Name of parameter into mapping holding the remote attribute */
107    protected static final String __PARAM_MAPPING_ATTRIBUTE = "attribute";
108    /** Name of parameter holding the criteria */
109    protected static final String __PARAM_CRITERIA = "criteria";
110    /** Name of parameter into criteria holding a criterion */
111    protected static final String __PARAM_CRITERIA_CRITERION = "criterion";
112    /** Name of parameter into criterion holding the id */
113    protected static final String __PARAM_CRITERIA_CRITERION_ID = "id";
114    /** Name of parameter into criterion holding the label */
115    protected static final String __PARAM_CRITERIA_CRITERION_LABEL = "label";
116    /** Name of parameter into criterion holding the type */
117    protected static final String __PARAM_CRITERIA_CRITERION_TYPE = "type";
118    /** Name of paramter holding columns */
119    protected static final String __PARAM_COLUMNS = "columns";
120    /** Name of paramter into columns holding column */
121    protected static final String __PARAM_COLUMNS_COLUMN = "column";
122    
123    /** Default language configured for ODF */
124    protected String _odfLang;
125
126    /** Name of the Apogée column which contains the ID */
127    protected String _idColumn;
128    
129    /** Mapping between metadata and columns */
130    protected Map<String, List<String>> _mapping;
131    
132    /** External fields */
133    protected Set<String> _extFields;
134    
135    /** Synchronized fields */
136    protected Set<String> _syncFields;
137    
138    /** Synchronized fields */
139    protected Set<String> _columns;
140    
141    /** Synchronized fields */
142    protected Set<ApogeeCriterion> _criteria;
143    
144    /** Context */
145    protected Context _context;
146    
147    /** The DAO for remote DB Apogee */
148    protected ApogeeDAO _apogeeDAO;
149    
150    /** The JSON utils */
151    protected JSONUtils _jsonUtils;
152    
153    /** SCC DAO */
154    protected SynchronizableContentsCollectionDAO _sccDAO;
155    
156    /** The ODF helper */
157    protected ODFHelper _odfHelper;
158    
159    /** The Apogee SCC helper */
160    protected ApogeeSynchronizableContentsCollectionHelper _apogeeSCCHelper;
161    
162    @Override
163    public void service(ServiceManager manager) throws ServiceException
164    {
165        super.service(manager);
166        _apogeeDAO = (ApogeeDAO) manager.lookup(ApogeeDAO.ROLE);
167        _jsonUtils = (JSONUtils) manager.lookup(JSONUtils.ROLE);
168        _sccDAO = (SynchronizableContentsCollectionDAO) manager.lookup(SynchronizableContentsCollectionDAO.ROLE);
169        _odfHelper = (ODFHelper) manager.lookup(ODFHelper.ROLE);
170        _apogeeSCCHelper = (ApogeeSynchronizableContentsCollectionHelper) manager.lookup(ApogeeSynchronizableContentsCollectionHelper.ROLE);
171    }
172
173    @Override
174    public void contextualize(Context context) throws ContextException
175    {
176        _context = context;
177    }
178    
179    @Override
180    protected void configureDataSource(Configuration configuration) throws ConfigurationException
181    {
182        @SuppressWarnings("resource")
183        InputStream is = null;
184        _odfLang = Config.getInstance().getValue("odf.programs.lang");
185        try
186        {
187            org.apache.cocoon.environment.Context ctx = (org.apache.cocoon.environment.Context) _context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
188            File apogeeMapping = new File(ctx.getRealPath("/WEB-INF/param/odf/apogee-mapping.xml"));
189            if (!apogeeMapping.isFile())
190            {
191                is = getClass().getResourceAsStream("/org/ametys/plugins/odfsync/apogee/apogee-mapping.xml");
192            }
193            else
194            {
195                is = new FileInputStream(apogeeMapping);
196            }
197            Configuration cfg = new DefaultConfigurationBuilder().build(is);
198            Configuration child = cfg.getChild(getMappingName());
199            if (child != null)
200            {
201                _criteria = new LinkedHashSet<>();
202                _columns = new LinkedHashSet<>();
203                _idColumn = child.getChild(__PARAM_ID_COLUMN).getValue();
204                _mapping = new HashMap<>();
205                _extFields = new HashSet<>();
206                _syncFields = new HashSet<>();
207                String mappingAsString = child.getChild(__PARAM_MAPPING).getValue();
208                _mapping.put(getIdField(), ImmutableList.of(getIdColumn()));
209                if (StringUtils.isNotEmpty(mappingAsString))
210                {
211                    List<Object> mappingAsList = _jsonUtils.convertJsonToList(mappingAsString);
212                    for (Object object : mappingAsList)
213                    {
214                        @SuppressWarnings("unchecked")
215                        Map<String, Object> field = (Map<String, Object>) object;
216                        
217                        String metadataRef = (String) field.get(__PARAM_MAPPING_METADATA_REF);
218                        
219                        String[] attributes = ((String) field.get(__PARAM_MAPPING_ATTRIBUTE)).split(",");
220                        _mapping.put(metadataRef, Arrays.asList(attributes));
221    
222                        boolean isSynchronized = field.containsKey(__PARAM_MAPPING_SYNCHRO) ? (Boolean) field.get(__PARAM_MAPPING_SYNCHRO) : false;
223                        if (isSynchronized)
224                        {
225                            _syncFields.add(metadataRef);
226                        }
227                        else
228                        {
229                            _extFields.add(metadataRef);
230                        }
231                    }
232                }
233
234                Configuration[] criteria = child.getChild(__PARAM_CRITERIA).getChildren(__PARAM_CRITERIA_CRITERION);
235                for (Configuration criterion : criteria)
236                {
237                    String id = criterion.getChild(__PARAM_CRITERIA_CRITERION_ID).getValue();
238                    I18nizableText label = _getCriterionLabel(criterion.getChild(__PARAM_CRITERIA_CRITERION_LABEL), id);
239                    String type = criterion.getChild(__PARAM_CRITERIA_CRITERION_TYPE).getValue("STRING");
240                    
241                    _criteria.add(new ApogeeCriterion(id, label, type));
242                }
243
244                Configuration[] columns = child.getChild(__PARAM_COLUMNS).getChildren(__PARAM_COLUMNS_COLUMN);
245                for (Configuration column : columns)
246                {
247                    _columns.add(column.getValue());
248                }
249            }
250        }
251        catch (Exception e)
252        {
253            throw new ConfigurationException("Error while parsing apogee-mapping.xml", e);
254        }
255        finally
256        {
257            IOUtils.closeQuietly(is);
258        }
259    }
260    
261    private I18nizableText _getCriterionLabel(Configuration configuration, String defaultValue)
262    {
263        if (configuration.getAttributeAsBoolean("i18n", false))
264        {
265            return new I18nizableText("plugin.odf-sync", configuration.getValue(defaultValue));
266        }
267        else
268        {
269            return new I18nizableText(configuration.getValue(defaultValue));
270        }
271    }
272    
273    @Override
274    public List<ModifiableDefaultContent> populate(Logger logger)
275    {
276        boolean isRequestAttributeOwner = false;
277        
278        Request request = ContextHelper.getRequest(_context);
279        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) == null)
280        {
281            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS, new HashSet<String>());
282            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS, new HashSet<String>());
283            isRequestAttributeOwner = true;
284        }
285        
286        List<ModifiableDefaultContent> populatedContents = super.populate(logger);
287        
288        _applyChangesToParentContent(logger);
289        
290        if (isRequestAttributeOwner)
291        {
292            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
293            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS);
294        }
295
296        return populatedContents;
297    }
298    
299    @Override
300    protected List<ModifiableDefaultContent> _internalPopulate(Logger logger)
301    {
302        return _importOrSynchronizeContents(Map.of("isGlobalSync", true), false, logger);
303    }
304    
305    @Override
306    protected Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger)
307    {
308        Map<String, Object> searchParams = new HashMap<>(parameters);
309        if (offset > 0)
310        {
311            searchParams.put("__offset", offset);
312        }
313        if (limit < Integer.MAX_VALUE)
314        {
315            searchParams.put("__limit", offset + limit);
316        }
317        searchParams.put("__order", _getSort(sort));
318
319        // We don't use session.selectMap which reorder data
320        List<Map<String, Object>> requestValues = _search(searchParams, logger);
321
322        // Transform CLOBs to String
323        Set<String> clobColumns = getClobColumns();
324        if (!clobColumns.isEmpty())
325        {
326            for (Map<String, Object> contentValues : requestValues)
327            {
328                String idValue = contentValues.get(getIdColumn()).toString();
329                for (String clobKey : getClobColumns())
330                {
331                    // Get the old values for the CLOB
332                    @SuppressWarnings("unchecked")
333                    Optional<List<Object>> oldValues = Optional.of(clobKey)
334                        .map(contentValues::get)
335                        .map(obj -> (List<Object>) obj);
336                    
337                    if (oldValues.isPresent())
338                    {
339                        // Get the new values for the CLOB
340                        List<Object> newValues = oldValues.get()
341                            .stream()
342                            .map(value -> _transformClobToString(value, idValue, logger))
343                            .filter(Objects::nonNull)
344                            .collect(Collectors.toList());
345                        
346                        // Set the transformed CLOB values
347                        contentValues.put(clobKey, newValues);
348                    }
349                }
350            }
351        }
352        
353        // Reorganize results
354        String idColumn = getIdColumn();
355        Map<String, Map<String, Object>> results = new LinkedHashMap<>();
356        for (Map<String, Object> contentValues : requestValues)
357        {
358            results.put(contentValues.get(idColumn).toString(), contentValues);
359        }
360        
361        for (Map<String, Object> result : results.values())
362        {
363            result.put(SCC_UNIQUE_ID, result.get(getIdColumn()));
364        }
365        
366        return results;
367    }
368    
369    @Override
370    protected Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger)
371    {
372        Map<String, Map<String, List<Object>>> remoteValues = new HashMap<>();
373        
374        Map<String, Map<String, Object>> results = internalSearch(parameters, 0, Integer.MAX_VALUE, null, logger);
375        
376        if (results != null)
377        {
378            remoteValues = _sccHelper.organizeRemoteValuesByMetadata(results, _mapping);
379        }
380        
381        return remoteValues;
382    }
383    
384    @Override
385    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
386    {
387        return _importOrSynchronizeContent(idValue, _odfLang, remoteValues, forceImport, logger);
388    }
389
390    @Override
391    public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception
392    {
393        boolean isRequestAttributeOwner = false;
394        
395        Request request = ContextHelper.getRequest(_context);
396        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) == null)
397        {
398            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS, new HashSet<String>());
399            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS, new HashSet<String>());
400            isRequestAttributeOwner = true;
401        }
402
403        List<ModifiableDefaultContent> createdContents = new ArrayList<>();
404        
405        Map<String, Object> parameters = putIdParameter(idValue);
406        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
407        if (!results.isEmpty())
408        {
409            try
410            {
411                createdContents.add(_importContent(idValue, importParams, _odfLang, results.get(idValue), logger));
412            }
413            catch (Exception e)
414            {
415                _nbError++;
416                logger.error("An error occurred while importing or synchronizing content", e);
417            }
418        }
419        
420        _applyChangesToParentContent(logger);
421        
422        if (isRequestAttributeOwner)
423        {
424            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
425            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS);
426        }
427
428        return createdContents;
429    }
430    
431    @Override
432    public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception
433    {
434        boolean isRequestAttributeOwner = false;
435        
436        Request request = ContextHelper.getRequest(_context);
437        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) == null)
438        {
439            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS, new HashSet<String>());
440            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS, new HashSet<String>());
441            isRequestAttributeOwner = true;
442        }
443        
444        super.synchronizeContent(content, logger);
445        
446        _applyChangesToParentContent(logger);
447        
448        if (isRequestAttributeOwner)
449        {
450            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
451            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS);
452        }
453    }
454    
455    /**
456     * Apply changes to parent contents which are impacted by the synchronization
457     * @param logger the logger
458     */
459    protected void _applyChangesToParentContent(Logger logger)
460    {
461        Request request = ContextHelper.getRequest(_context);
462        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) != null)
463        {
464            @SuppressWarnings("unchecked")
465            Set<String> handleContents = (Set<String>) request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
466
467            @SuppressWarnings("unchecked")
468            Set<String> parentContents = (HashSet<String>) request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_PARENT_CONTENTS);
469
470            // Apply changes on parent content only if they are not handled by the synchronization
471            SetView<String> contentsToApplyChanges = SetUtils.difference(parentContents, handleContents);
472            for (String parentContentId : contentsToApplyChanges)
473            {
474                try
475                {
476                    ModifiableDefaultContent parentContent = _resolver.resolveById(parentContentId);
477                    applyChanges(parentContent, 22, org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, logger);
478                }
479                catch (Exception e) 
480                {
481                    logger.error("An error occurred applying changes to content '{}'", parentContentId, e);
482                }
483            }
484        }
485    }
486    
487    @Override
488    protected Map<String, Object> putIdParameter(String idValue)
489    {
490        Map<String, Object> parameters = new HashMap<>();
491        parameters.put(getIdField(), idValue);
492        return parameters;
493    }
494    
495    /**
496     * Search the contents with the search parameters. Use id parameter to search an unique content.
497     * @param searchParams Search parameters
498     * @param logger The logger
499     * @return A Map of mapped metadatas extract from Apogée database ordered by content unique Apogée ID
500     */
501    protected abstract List<Map<String, Object>> _search(Map<String, Object> searchParams, Logger logger);
502    
503    /**
504     * Convert the {@link BigDecimal} values retrieved from database into long values
505     * @param searchResults The initial search results from database
506     * @return The converted search results
507     */
508    protected List<Map<String, Object>> _convertBigDecimal(List<Map<String, Object>> searchResults)
509    {
510        List<Map<String, Object>> convertedSearchResults = new ArrayList<>();
511        
512        for (Map<String, Object> searchResult : searchResults)
513        {
514            for (String key : searchResult.keySet())
515            {
516                searchResult.put(key, _convertBigDecimal(getContentType(), key, searchResult.get(key))); 
517            }
518            
519            convertedSearchResults.add(searchResult);
520        }
521        
522        return convertedSearchResults;
523    }
524    
525    /**
526     * Convert the object in parameter to a long if it's a {@link BigDecimal}, otherwise return the object itself.
527     * @param contentTypeId The content type of the parent content
528     * @param attributeName The metadata name
529     * @param objectToConvert The object to convert if necessary
530     * @return The converted object
531     */
532    protected Object _convertBigDecimal(String contentTypeId, String attributeName, Object objectToConvert)
533    {
534        if (objectToConvert instanceof BigDecimal)
535        {
536            ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
537            if (contentType.hasModelItem(attributeName))
538            {
539                ModelItem definition = contentType.getModelItem(attributeName);    
540                String typeId = definition.getType().getId();
541                switch (typeId)
542                {
543                    case ModelItemTypeConstants.DOUBLE_TYPE_ID:
544                        return ((BigDecimal) objectToConvert).doubleValue();
545                    case ModelItemTypeConstants.LONG_TYPE_ID:
546                        return ((BigDecimal) objectToConvert).longValue();
547                    default:
548                        // Do nothing
549                        break;
550                }
551            }
552            return ((BigDecimal) objectToConvert).toString();
553        }
554        return objectToConvert;
555    }
556    
557    /**
558     * Transform CLOB value to String value.
559     * @param value The input value
560     * @param idValue The identifier of the program
561     * @param logger The logger
562     * @return the same value, with CLOB transformed to String.
563     */
564    protected Object _transformClobToString(Object value, String idValue, Logger logger)
565    {
566        if (value instanceof Clob)
567        {
568            Clob clob = (Clob) value;
569            try
570            {
571                String strValue = IOUtils.toString(clob.getCharacterStream());
572                return CharMatcher.javaIsoControl().and(CharMatcher.anyOf("\r\n\t").negate()).removeFrom(strValue);
573            }
574            catch (SQLException | IOException e)
575            {
576                logger.error("Unable to get education add elements from the program '{}'.", idValue, e);
577                return null;
578            }
579            finally
580            {
581                try
582                {
583                    clob.free();
584                }
585                catch (SQLException e)
586                {
587                    // Ignore the exception.
588                }
589            }
590        }
591        
592        return value;
593    }
594
595    /**
596     * Get the list of CLOB column's names.
597     * @return The list of the CLOB column's names to transform to {@link String}
598     */
599    protected Set<String> getClobColumns()
600    {
601        return Set.of();
602    }
603    
604    /**
605     * Transform a {@link List} of {@link Map} to a {@link Map} of {@link List} computed by keys (lines to columns).
606     * @param lines {@link List} to reorganize
607     * @return {@link Map} of {@link List}
608     */
609    protected Map<String, List<Object>> _transformListOfMap2MapOfList(List<Map<String, Object>> lines)
610    {
611        return lines.stream()
612            .filter(Objects::nonNull)
613            .map(Map::entrySet)
614            .flatMap(Collection::stream)
615            .filter(e -> Objects.nonNull(e.getValue()))
616            .collect(
617                Collectors.toMap(
618                    Entry::getKey,
619                    e -> List.of(e.getValue()),
620                    (l1, l2) -> ListUtils.union(l1, l2)
621                )
622            );
623    }
624    
625    /**
626     * Get the name of the mapping.
627     * @return the mapping name
628     */
629    protected abstract String getMappingName();
630    
631    /**
632     * Get the id of data source
633     * @return The id of data source
634     */
635    protected String getDataSourceId()
636    {
637        return (String) getParameterValues().get(PARAM_DATASOURCE_ID);
638    }
639    
640    /**
641     * Get the administrative year
642     * @return The administrative year
643     */
644    protected String getYear()
645    {
646        return (String) getParameterValues().get(PARAM_YEAR);
647    }
648    
649    /**
650     * Check if unexisting contents in Ametys should be added from data source
651     * @return <code>true</code> if unexisting contents in Ametys should be added from data source, default value is <code>false</code>
652     */
653    protected boolean addUnexistingChildren()
654    {
655        return Boolean.valueOf((String) getParameterValues().getOrDefault(PARAM_ADD_UNEXISTING_CHILDREN, Boolean.TRUE.toString()));
656    }
657    
658    /**
659     * Get the identifier column (can be a concatened column).
660     * @return the column id
661     */
662    protected String getIdColumn()
663    {
664        return _idColumn;
665    }
666    
667    @Override
668    public String getIdField()
669    {
670        return "apogeeSyncCode";
671    }
672
673    @Override
674    public Set<String> getLocalAndExternalFields(Map<String, Object> additionalParameters)
675    {
676        return _syncFields;
677    }
678
679    @Override
680    public Set<String> getExternalOnlyFields(Map<String, Object> additionalParameters)
681    {
682        return _extFields;
683    }
684    
685    @Override
686    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
687    {
688        ModifiableCompositeMetadata holder = content.getMetadataHolder();
689        
690        boolean hasChanges = _fillRichTexts(remoteValues, content, create, logger);
691        hasChanges = super._fillContent(remoteValues, content, create, logger) || hasChanges;
692        hasChanges = _handleAdditionalMetadata(holder, create) || hasChanges;
693        
694        if (!holder.hasMetadata(getIdField()))
695        {
696            holder.setMetadata(getIdField(), remoteValues.get(getIdField()).get(0).toString());
697            hasChanges |= create;
698        }
699        
700        return hasChanges;
701    }
702    
703    /**
704     * Fill the richt texts of the content with the remote values.
705     * @param remoteValues The remote values
706     * @param content The content to synchronize
707     * @param create <code>true</code> if content is creating, false if it is updated
708     * @param logger The logger
709     * @return <code>true</code> if changes were made
710     */
711    protected boolean _fillRichTexts(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
712    {
713        boolean hasChanges = false;
714
715        ModifiableCompositeMetadata holder = content.getMetadataHolder();
716        Map<String, Object> params = Map.of("contentTypes", List.of(getContentType()));
717        
718        for (String metadataName : getRichTextFields())
719        {
720            if (remoteValues.containsKey(metadataName))
721            {
722                boolean synchronize = getLocalAndExternalFields(params).contains(metadataName);
723                
724                ModifiableRichText richText = ExternalizableMetadataHelper.getRichText(holder, metadataName, synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL, true);
725                richText.setMimeType("text/xml");
726                richText.setLastModified(new Date());
727                try
728                {
729                    _setRichTextValue(richText, remoteValues.get(metadataName));
730                }
731                catch (IOException e)
732                {
733                    logger.error("An error occured while parsing the rich text '{}' of the content '{}'", metadataName, content.getTitle(), e);
734                }
735                
736                if (synchronize && create)
737                {
738                    // Force external status
739                    ExternalizableMetadataHelper.updateStatus(holder,  metadataName, ExternalizableMetadataStatus.EXTERNAL);
740                }
741                
742                remoteValues.remove(metadataName);
743                hasChanges = true;
744            }
745        }
746        
747        return hasChanges;
748    }
749    
750    /**
751     * Set the value on the rich text.
752     * @param richText The rich text to update
753     * @param remoteValue The remote value of the rich text (can be composed of several texts to concatenate)
754     * @throws IOException if an exception occurs
755     */
756    protected void _setRichTextValue(ModifiableRichText richText, List<Object> remoteValue) throws IOException
757    {
758        List<String> lines = new LinkedList<>();
759        for (Object remoteText : remoteValue)
760        {
761            lines.add(remoteText.toString());
762        }
763        
764        String docbook = ContentImporterHelper.textToDocbook(lines.toArray(new String[lines.size()]));
765        
766        try (ByteArrayInputStream is =  new ByteArrayInputStream(docbook.getBytes("UTF-8")))
767        {
768            richText.setInputStream(is);
769        }
770    }
771
772    /**
773     * Method to add additional metadata on import or synchronize.
774     * @param holder The holder of the content to update
775     * @param create If we are on creation mode
776     * @return <code>true</code> if changes has been made
777     */
778    protected boolean _handleAdditionalMetadata(ModifiableCompositeMetadata holder, boolean create)
779    {
780        // Do nothing
781        return false;
782    }
783
784    /**
785     * Get the list of rich text fields of the imported content.
786     * @return The list of the rich text fields metadata name
787     */
788    protected Set<String> getRichTextFields()
789    {
790        return new HashSet<>();
791    }
792    
793    @Override
794    protected void configureSearchModel()
795    {
796        for (ApogeeCriterion criterion : _criteria)
797        {
798            _searchModelConfiguration.addCriterion(criterion.getId(), criterion.getLabel(), criterion.getType());
799        }
800        for (String columnName : _columns)
801        {
802            _searchModelConfiguration.addColumn(columnName);
803        }
804    }
805    
806    @Override
807    protected boolean additionalCommonOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, boolean create, Logger logger)
808    {
809        boolean hasChanges = super.additionalCommonOperations(content, remoteValues, importParams, create, logger);
810
811        Object parentId = importParams != null && importParams.containsKey("parentId") ? importParams.get("parentId") : null;
812        ModifiableDefaultContent parentContent = parentId != null ? _resolver.resolveById(parentId.toString()) : null;
813        
814        hasChanges = handleParent(content, parentContent, logger) || hasChanges;
815        hasChanges = handleChildren(content, create, logger) || hasChanges;
816        hasChanges = setAdditionalMetadata(content, remoteValues, create, logger) || hasChanges;
817        
818        return hasChanges;
819    }
820    
821    /**
822     * Set the parent metadata and invert relation.
823     * @param currentContent Current content
824     * @param parentContent Parent content to set
825     * @param logger The logger
826     * @return <code>true</code> if there are changes
827     */
828    protected boolean handleParent(ModifiableDefaultContent currentContent, ModifiableDefaultContent parentContent, Logger logger)
829    {
830        // Nothing to do by default
831        return false;
832    }
833    
834    /**
835     * Set the children metadata and invert relation, import and synchronize the children too.
836     * @param content Current content
837     * @param create true if the content has been newly created
838     * @param logger The logger
839     * @return <code>true</code> if there are changes
840     */
841    protected boolean handleChildren(ModifiableDefaultContent content, boolean create, Logger logger)
842    {
843        return importOrSynchronizeChildren(content, getChildrenSCCModelId(), getChildrenMetadataName(), getChildrenInvertMetadataName(), create, logger);
844    }
845    
846    /**
847     * Get the children SCC model id. Can be null if no implementation is defined.
848     * @return the children SCC model id
849     */
850    protected String getChildrenSCCModelId()
851    {
852        // Default implementation
853        return null;
854    }
855    
856    /**
857     * Get the metadata name to get children
858     * @return the metadata name to get children
859     */
860    protected abstract String getChildrenMetadataName();
861
862    /**
863     * Get the metadata name to get parent from children
864     * @return the metadata name to get parent from children
865     */
866    protected abstract String getChildrenInvertMetadataName();
867
868    /**
869     * Set the additional metadata.
870     * @param content Current content
871     * @param remoteValues Values of the content
872     * @param create true if the content has been newly created
873     * @param logger The logger
874     * @return <code>true</code> if there are changes
875     */
876    protected boolean setAdditionalMetadata(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, boolean create, Logger logger)
877    {
878        // Nothing to do by default
879        return false;
880    }
881    
882    /**
883     * Transform the given {@link List} of {@link Object} to a {@link String} representing the ordered fields for SQL.
884     * @param sortList The sort list object to transform to the list of ordered fields compatible with SQL.
885     * @return A string representing the list of ordered fields
886     */
887    @SuppressWarnings("unchecked")
888    protected String _getSort(List<Object> sortList)
889    {
890        if (sortList != null)
891        {
892            StringBuilder sort = new StringBuilder();
893            
894            for (Object sortValueObj : sortList)
895            {
896                Map<String, Object> sortValue = (Map<String, Object>) sortValueObj;
897                
898                sort.append(sortValue.get("property"));
899                if (sortValue.containsKey("direction"))
900                {
901                    sort.append(" ");
902                    sort.append(sortValue.get("direction"));
903                    sort.append(",");
904                }
905                else
906                {
907                    sort.append(" ASC,");
908                }
909            }
910            
911            sort.deleteCharAt(sort.length() - 1);
912            
913            return sort.toString();
914        }
915        
916        return null;
917    }
918    
919    @Override
920    public int getTotalCount(Map<String, Object> parameters, Logger logger)
921    {
922        // Remove empty parameters
923        Map<String, Object> searchParams = new HashMap<>();
924        for (String parameterName : parameters.keySet())
925        {
926            Object parameterValue = parameters.get(parameterName);
927            if (parameterValue != null && !parameterValue.toString().isEmpty())
928            {
929                searchParams.put(parameterName, parameterValue);
930            }
931        }
932        
933        searchParams.put("__count", true);
934        
935        List<Map<String, Object>> results = _search(searchParams, logger);
936        if (results != null && !results.isEmpty())
937        {
938            return Integer.valueOf(results.get(0).get("COUNT(*)").toString()).intValue();
939        }
940        
941        return 0;
942    }
943    
944    @Override
945    protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
946    {
947        ModifiableDefaultContent content = super._importContent(idValue, importParams, lang, _transformOrgUnitMetadata(remoteValues, logger), logger);
948        if (content != null)
949        {
950            _apogeeSCCHelper.addToHandleContents(content.getId());
951        }
952        return content;
953    }
954    
955    @Override
956    protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
957    {
958        if (!_apogeeSCCHelper.addToHandleContents(content.getId()))
959        {
960            return content;
961        }
962        return super._synchronizeContent(content, _transformOrgUnitMetadata(remoteValues, logger), logger);
963    }
964    
965    /**
966     * Import and synchronize children of the given content, then edit the structure of the content and its children.
967     * In the usual case, if we are in import mode (create to true) or if we trust the source (removalSync to true) we just synchronize structure but we don't synchronize child contents
968     * @param content Parent content
969     * @param sccModelId SCC model ID
970     * @param metadataName Metadata name to set
971     * @param invertMetadataName Metadata name of the invert relation
972     * @param create true if the content has been newly created
973     * @param logger The logger
974     * @return <code>true</code> if there are changes
975     */
976    protected boolean importOrSynchronizeChildren(ModifiableDefaultContent content, String sccModelId, String metadataName, String invertMetadataName, boolean create, Logger logger)
977    {
978        // Get the SCC for children
979        SynchronizableContentsCollection scc = _sccHelper.getSCCFromModelId(sccModelId);
980        
981        return create
982            // Import mode
983            ? _importChildren(content, scc, metadataName, invertMetadataName, logger)
984            // Synchronization mode
985            : _synchronizeChildren(content, scc, metadataName, invertMetadataName, logger);
986    }
987    
988    
989
990    /**
991     * Import children
992     * @param content Parent content
993     * @param scc the scc
994     * @param metadataName Metadata name to set
995     * @param invertMetadataName Metadata name of the invert relation
996     * @param logger The logger
997     * @return <code>true</code> if there are changes
998     */
999    protected boolean _importChildren(ModifiableDefaultContent content, SynchronizableContentsCollection scc, String metadataName, String invertMetadataName, Logger logger)
1000    {
1001        // If SCC exists, search for children
1002        if (scc != null && scc instanceof ApogeeSynchronizableContentsCollection)
1003        {
1004            // Synchronize or import children content
1005            ModifiableCompositeMetadata cm = content.getMetadataHolder();
1006            String syncCode = cm.getString(getIdField());
1007            List<ModifiableDefaultContent> importedOrSynchronizedChildren = ((ApogeeSynchronizableContentsCollection) scc).importOrSynchronizeContents(ImmutableMap.of("parentCode", syncCode), logger);
1008
1009            return _resetChildren(content, importedOrSynchronizedChildren, metadataName, invertMetadataName, logger);
1010        }
1011        
1012        return false;
1013    }
1014    
1015    /**
1016     * Synchronize children
1017     * @param content Parent content
1018     * @param scc the scc
1019     * @param metadataName Metadata name to set
1020     * @param invertMetadataName Metadata name of the invert relation
1021     * @param logger The logger
1022     * @return <code>true</code> if there are changes
1023     */
1024    protected boolean _synchronizeChildren(ModifiableDefaultContent content, SynchronizableContentsCollection scc, String metadataName, String invertMetadataName, Logger logger)
1025    {
1026        // Get the children remote sync codes if needed
1027        // These remote sync codes are needed if we want to add unexisting contents in Ametys from Apogee or delete obsolete contents 
1028        Set<String> childrenRemoteSyncCode = (removalSync() || addUnexistingChildren())
1029            ? _getChildrenRemoteSyncCode(content, scc, logger)
1030            : null;
1031        
1032        // First we get the existing children in Ametys
1033        List<ModifiableContent> ametysChildren = Stream.of(content.getValue(metadataName, false, new ContentValue[0]))
1034            .map(ContentValue::getContentIfExists)
1035            .filter(Optional::isPresent)
1036            .map(Optional::get)
1037            .collect(Collectors.toList());
1038        
1039        // Remove obsolete children if removeSync is active
1040        if (childrenRemoteSyncCode != null && removalSync())
1041        {
1042            ametysChildren = ametysChildren.stream()
1043                .filter(c -> !_isChildWillBeRemoved(c, scc, childrenRemoteSyncCode, logger))
1044                .collect(Collectors.toList());
1045        }
1046        
1047        // Then we synchronize the existing children in Ametys
1048        for (ModifiableContent childContent : ametysChildren)
1049        {
1050            _apogeeSCCHelper.synchronizeContent(childContent, logger);
1051        }
1052        
1053        // Then we add unexisting children if needed
1054        if (childrenRemoteSyncCode != null && addUnexistingChildren())
1055        {
1056            ametysChildren.addAll(_addUnexistingChildren(scc, childrenRemoteSyncCode, logger));
1057        }
1058        
1059        // Reset children if needed
1060        if (childrenRemoteSyncCode != null && (removalSync() || addUnexistingChildren()))
1061        {
1062            return _resetChildren(content, ametysChildren, metadataName, invertMetadataName, logger);
1063        }
1064        
1065        return false;
1066    }
1067
1068    /**
1069     * Get the remote sync codes
1070     * @param content Parent content
1071     * @param scc the scc
1072     * @param logger The logger
1073     * @return the remote sync codes or null if the scc is not from Apogee
1074     */
1075    protected Set<String> _getChildrenRemoteSyncCode(ModifiableDefaultContent content, SynchronizableContentsCollection scc, Logger logger)
1076    {
1077        if (scc != null && scc instanceof AbstractApogeeSynchronizableContentsCollection)
1078        {
1079            String syncCode = content.getMetadataHolder().getString(getIdField());
1080            return ((AbstractApogeeSynchronizableContentsCollection) scc)
1081                        .search(ImmutableMap.of("parentCode", syncCode), 0, Integer.MAX_VALUE, null, logger)
1082                        .keySet();
1083        }
1084        
1085        return null;
1086    }
1087
1088    /**
1089     * Add unexisting children in Ametys from Apogee
1090     * @param scc the scc to import contents
1091     * @param childrenRemoteSyncCode the remote sync codes
1092     * @param logger the logger
1093     * @return the list of unexisting contents
1094     */
1095    protected List<ModifiableContent> _addUnexistingChildren(SynchronizableContentsCollection scc, Set<String> childrenRemoteSyncCode, Logger logger)
1096    {
1097        return childrenRemoteSyncCode
1098            .stream()
1099            .filter(code -> scc.getContent(_odfLang, code) == null) // get only remote content values which doesn't exist in Ametys
1100            .flatMap(code -> _importUnexistingChildren(scc, code, null, logger))
1101            .collect(Collectors.toList());
1102    }
1103
1104    /**
1105     * Import an unexisting child in Ametys
1106     * @param scc the scc to import the content
1107     * @param syncCode the sync code of the content
1108     * @param params the additional params
1109     * @param logger the logger
1110     * @return the list of created children
1111     */
1112    @SuppressWarnings("unchecked")
1113    protected Stream<ModifiableDefaultContent> _importUnexistingChildren(SynchronizableContentsCollection scc, String syncCode, Map<String, List<Object>> params, Logger logger)
1114    {
1115        try
1116        {
1117            return scc.importContent(syncCode, (Map<String, Object>) (Object) params, logger).stream();
1118        }
1119        catch (Exception e)
1120        {
1121            logger.error("An error occured while importing a new children content with syncCode '{}' from SCC '{}'", syncCode, scc.getId(), e);
1122        }
1123        
1124        return Stream.empty();
1125    }
1126    
1127    /**
1128     * True if the content will be removed from the structure
1129     * @param content the content
1130     * @param scc the scc
1131     * @param childrenRemoteSyncCode the remote sync codes
1132     * @param logger the logger
1133     * @return <code>true</code> if the content will be removed from the structure
1134     */
1135    protected boolean _isChildWillBeRemoved(ModifiableContent content, SynchronizableContentsCollection scc, Set<String> childrenRemoteSyncCode, Logger logger)
1136    {
1137        String syncCode = content.getMetadataHolder().getString(scc.getIdField(), null);
1138        return !childrenRemoteSyncCode.contains(syncCode);
1139    }
1140    
1141    /**
1142     * Reset the children of the structure to force with the children from Apogée.
1143     * @param content Parent content
1144     * @param importedOrSynchronizedChildren the imported or synchronized children
1145     * @param metadataName Metadata name to set
1146     * @param invertMetadataName Metadata name of the invert relation
1147     * @param logger The logger
1148     * @return <code>true</code> if there are changes
1149     */
1150    protected boolean _resetChildren(ModifiableDefaultContent content, List<? extends ModifiableContent> importedOrSynchronizedChildren , String metadataName, String invertMetadataName, Logger logger)
1151    {
1152        boolean hasChanges = false;
1153
1154        ModifiableCompositeMetadata cm = content.getMetadataHolder();
1155        
1156        // Remove all children relations
1157        ContentValue[] oldChildren = content.getValue(metadataName, false, new ContentValue[0]);
1158        if (oldChildren.length > 0)
1159        {
1160            hasChanges = ExternalizableMetadataHelper.removeMetadataIfExists(cm, metadataName) || hasChanges;
1161            
1162            for (ContentValue child : oldChildren)
1163            {
1164                Optional<ModifiableContent> childContent = child.getContentIfExists();
1165                if (childContent.isPresent())
1166                {
1167                    hasChanges = _updateRelation(childContent.get().getMetadataHolder(), invertMetadataName, content, true) || hasChanges;
1168                }
1169            }
1170        }
1171        
1172        if (!importedOrSynchronizedChildren.isEmpty())
1173        {
1174            // Update child relation to link to the parent
1175            ModifiableDefaultContent[] newChildren = importedOrSynchronizedChildren.toArray(new ModifiableDefaultContent[importedOrSynchronizedChildren.size()]);
1176            hasChanges = ExternalizableMetadataHelper.setMetadata(cm, metadataName, newChildren) || hasChanges;
1177
1178            // Update parent relation on each children
1179            for (ModifiableDefaultContent child : newChildren)
1180            {
1181                hasChanges = _updateRelation(child.getMetadataHolder(), invertMetadataName, content) || hasChanges;
1182            }
1183        }
1184        
1185        return hasChanges;
1186    }
1187
1188    @Override
1189    public List<ModifiableDefaultContent> importOrSynchronizeContents(Map<String, Object> searchParams, Logger logger)
1190    {
1191        return _importOrSynchronizeContents(searchParams, true, logger);
1192    }
1193    
1194    @SuppressWarnings("unchecked")
1195    private Map<String, List<Object>> _transformOrgUnitMetadata(Map<String, List<Object>> remoteValues, Logger logger)
1196    {
1197        // Transform orgUnit values and import content if necessary (useful for Course and SubProgram)
1198        SynchronizableContentsCollection scc = _sccHelper.getSCCFromModelId(OrgUnitSynchronizableContentsCollection.MODEL_ID);
1199        
1200        List<Object> orgUnitCodes = remoteValues.get("orgUnit");
1201        if (orgUnitCodes != null && !orgUnitCodes.isEmpty())
1202        {
1203            List<?> orgUnitContents = null;
1204            String orgUnitCode = orgUnitCodes.get(0).toString();
1205            
1206            if (scc != null)
1207            {
1208                try
1209                {
1210                    ModifiableDefaultContent orgUnitContent = scc.getContent(_odfLang, orgUnitCode);
1211                    if (orgUnitContent == null)
1212                    {
1213                        orgUnitContents = scc.importContent(orgUnitCode, null, logger);
1214                    }
1215                    else
1216                    {
1217                        orgUnitContents = ImmutableList.of(orgUnitContent);
1218                    }
1219                }
1220                catch (Exception e)
1221                {
1222                    logger.error("An error occured during the import of the OrgUnit identified by the synchronization code '{}'", orgUnitCode, e);
1223                }
1224            }
1225            
1226            if (orgUnitContents == null)
1227            {
1228                // Impossible link to orgUnit
1229                remoteValues.remove("orgUnit");
1230                logger.warn("Impossible to import the OrgUnit with the synchronization code '{}', check if you set the OrgUnit SCC with the following model ID: '{}'", orgUnitCode, OrgUnitSynchronizableContentsCollection.MODEL_ID);
1231            }
1232            else
1233            {
1234                remoteValues.put("orgUnit", (List<Object>) orgUnitContents);
1235            }
1236        }
1237        
1238        return remoteValues;
1239    }
1240
1241    @Override
1242    public boolean handleRightAssignmentContext()
1243    {
1244        // Rights on ODF contents are handled by ODFRightAssignmentContext
1245        return false;
1246    }
1247}