Source code

001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.odfsync.apogee.scc;
017
018import java.io.ByteArrayInputStream;
019import java.io.File;
020import java.io.FileInputStream;
021import java.io.IOException;
022import java.io.InputStream;
023import java.math.BigDecimal;
024import java.sql.Clob;
025import java.sql.SQLException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Collection;
029import java.util.Date;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.LinkedHashMap;
033import java.util.LinkedHashSet;
034import java.util.LinkedList;
035import java.util.List;
036import java.util.Map;
037import java.util.Map.Entry;
038import java.util.Objects;
039import java.util.Optional;
040import java.util.Set;
041import java.util.stream.Collectors;
042import java.util.stream.Stream;
043
044import org.apache.avalon.framework.configuration.Configuration;
045import org.apache.avalon.framework.configuration.ConfigurationException;
046import org.apache.avalon.framework.configuration.DefaultConfigurationBuilder;
047import org.apache.avalon.framework.context.Context;
048import org.apache.avalon.framework.context.ContextException;
049import org.apache.avalon.framework.context.Contextualizable;
050import org.apache.avalon.framework.service.ServiceException;
051import org.apache.avalon.framework.service.ServiceManager;
052import org.apache.cocoon.Constants;
053import org.apache.cocoon.components.ContextHelper;
054import org.apache.cocoon.environment.Request;
055import org.apache.commons.collections4.ListUtils;
056import org.apache.commons.io.IOUtils;
057import org.apache.commons.lang.StringUtils;
058import org.slf4j.Logger;
059
060import org.ametys.cms.content.external.ExternalizableMetadataHelper;
061import org.ametys.cms.content.external.ExternalizableMetadataProvider.ExternalizableMetadataStatus;
062import org.ametys.cms.contenttype.ContentType;
063import org.ametys.cms.data.ContentValue;
064import org.ametys.cms.repository.ModifiableContent;
065import org.ametys.cms.repository.ModifiableDefaultContent;
066import org.ametys.core.util.JSONUtils;
067import org.ametys.odf.ODFHelper;
068import org.ametys.plugins.contentio.ContentImporterHelper;
069import org.ametys.plugins.contentio.synchronize.AbstractSimpleSynchronizableContentsCollection;
070import org.ametys.plugins.contentio.synchronize.SynchronizableContentsCollection;
071import org.ametys.plugins.contentio.synchronize.SynchronizableContentsCollectionDAO;
072import org.ametys.plugins.odfsync.apogee.ApogeeDAO;
073import org.ametys.plugins.odfsync.apogee.scc.impl.OrgUnitSynchronizableContentsCollection;
074import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
075import org.ametys.plugins.repository.metadata.ModifiableRichText;
076import org.ametys.runtime.config.Config;
077import org.ametys.runtime.i18n.I18nizableText;
078import org.ametys.runtime.model.ModelItem;
079import org.ametys.runtime.model.type.ModelItemTypeConstants;
080
081import com.google.common.base.CharMatcher;
082import com.google.common.collect.ImmutableList;
083import com.google.common.collect.ImmutableMap;
084
085/**
086 * Abstract class for Apogee synchronization
087 */
088public abstract class AbstractApogeeSynchronizableContentsCollection extends AbstractSimpleSynchronizableContentsCollection implements Contextualizable, ApogeeSynchronizableContentsCollection
089{
090    /** Name of parameter holding the data source id */
091    public static final String PARAM_DATASOURCE_ID = "datasourceId";
092    /** Name of parameter holding the administrative year */
093    public static final String PARAM_YEAR = "year";
094    /** Name of parameter holding the adding unexisting children parameter  */
095    public static final String PARAM_ADD_UNEXISTING_CHILDREN = "add-unexisting-children";
096    /** Name of parameter holding the field ID column */
097    protected static final String __PARAM_ID_COLUMN = "idColumn";
098    /** Name of parameter holding the fields mapping */
099    protected static final String __PARAM_MAPPING = "mapping";
100    /** Name of parameter into mapping holding the synchronized property */
101    protected static final String __PARAM_MAPPING_SYNCHRO = "synchro";
102    /** Name of parameter into mapping holding the path of metadata */
103    protected static final String __PARAM_MAPPING_METADATA_REF = "metadata-ref";
104    /** Name of parameter into mapping holding the remote attribute */
105    protected static final String __PARAM_MAPPING_ATTRIBUTE = "attribute";
106    /** Name of parameter holding the criteria */
107    protected static final String __PARAM_CRITERIA = "criteria";
108    /** Name of parameter into criteria holding a criterion */
109    protected static final String __PARAM_CRITERIA_CRITERION = "criterion";
110    /** Name of parameter into criterion holding the id */
111    protected static final String __PARAM_CRITERIA_CRITERION_ID = "id";
112    /** Name of parameter into criterion holding the label */
113    protected static final String __PARAM_CRITERIA_CRITERION_LABEL = "label";
114    /** Name of parameter into criterion holding the type */
115    protected static final String __PARAM_CRITERIA_CRITERION_TYPE = "type";
116    /** Name of paramter holding columns */
117    protected static final String __PARAM_COLUMNS = "columns";
118    /** Name of paramter into columns holding column */
119    protected static final String __PARAM_COLUMNS_COLUMN = "column";
120    
121    /** Default language configured for ODF */
122    protected String _odfLang;
123
124    /** Name of the Apogée column which contains the ID */
125    protected String _idColumn;
126    
127    /** Mapping between metadata and columns */
128    protected Map<String, List<String>> _mapping;
129    
130    /** External fields */
131    protected Set<String> _extFields;
132    
133    /** Synchronized fields */
134    protected Set<String> _syncFields;
135    
136    /** Synchronized fields */
137    protected Set<String> _columns;
138    
139    /** Synchronized fields */
140    protected Set<ApogeeCriterion> _criteria;
141    
142    /** Context */
143    protected Context _context;
144    
145    /** The DAO for remote DB Apogee */
146    protected ApogeeDAO _apogeeDAO;
147    
148    /** The JSON utils */
149    protected JSONUtils _jsonUtils;
150    
151    /** SCC DAO */
152    protected SynchronizableContentsCollectionDAO _sccDAO;
153    
154    /** The ODF helper */
155    protected ODFHelper _odfHelper;
156    
157    /** The Apogee SCC helper */
158    protected ApogeeSynchronizableContentsCollectionHelper _apogeeSCCHelper;
159    
160    @Override
161    public void service(ServiceManager manager) throws ServiceException
162    {
163        super.service(manager);
164        _apogeeDAO = (ApogeeDAO) manager.lookup(ApogeeDAO.ROLE);
165        _jsonUtils = (JSONUtils) manager.lookup(JSONUtils.ROLE);
166        _sccDAO = (SynchronizableContentsCollectionDAO) manager.lookup(SynchronizableContentsCollectionDAO.ROLE);
167        _odfHelper = (ODFHelper) manager.lookup(ODFHelper.ROLE);
168        _apogeeSCCHelper = (ApogeeSynchronizableContentsCollectionHelper) manager.lookup(ApogeeSynchronizableContentsCollectionHelper.ROLE);
169    }
170
171    @Override
172    public void contextualize(Context context) throws ContextException
173    {
174        _context = context;
175    }
176    
177    @Override
178    protected void configureDataSource(Configuration configuration) throws ConfigurationException
179    {
180        @SuppressWarnings("resource")
181        InputStream is = null;
182        _odfLang = Config.getInstance().getValue("odf.programs.lang");
183        try
184        {
185            org.apache.cocoon.environment.Context ctx = (org.apache.cocoon.environment.Context) _context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
186            File apogeeMapping = new File(ctx.getRealPath("/WEB-INF/param/odf/apogee-mapping.xml"));
187            if (!apogeeMapping.isFile())
188            {
189                is = getClass().getResourceAsStream("/org/ametys/plugins/odfsync/apogee/apogee-mapping.xml");
190            }
191            else
192            {
193                is = new FileInputStream(apogeeMapping);
194            }
195            Configuration cfg = new DefaultConfigurationBuilder().build(is);
196            Configuration child = cfg.getChild(getMappingName());
197            if (child != null)
198            {
199                _criteria = new LinkedHashSet<>();
200                _columns = new LinkedHashSet<>();
201                _idColumn = child.getChild(__PARAM_ID_COLUMN).getValue();
202                _mapping = new HashMap<>();
203                _extFields = new HashSet<>();
204                _syncFields = new HashSet<>();
205                String mappingAsString = child.getChild(__PARAM_MAPPING).getValue();
206                _mapping.put(getIdField(), ImmutableList.of(getIdColumn()));
207                if (StringUtils.isNotEmpty(mappingAsString))
208                {
209                    List<Object> mappingAsList = _jsonUtils.convertJsonToList(mappingAsString);
210                    for (Object object : mappingAsList)
211                    {
212                        @SuppressWarnings("unchecked")
213                        Map<String, Object> field = (Map<String, Object>) object;
214                        
215                        String metadataRef = (String) field.get(__PARAM_MAPPING_METADATA_REF);
216                        
217                        String[] attributes = ((String) field.get(__PARAM_MAPPING_ATTRIBUTE)).split(",");
218                        _mapping.put(metadataRef, Arrays.asList(attributes));
219    
220                        boolean isSynchronized = field.containsKey(__PARAM_MAPPING_SYNCHRO) ? (Boolean) field.get(__PARAM_MAPPING_SYNCHRO) : false;
221                        if (isSynchronized)
222                        {
223                            _syncFields.add(metadataRef);
224                        }
225                        else
226                        {
227                            _extFields.add(metadataRef);
228                        }
229                    }
230                }
231
232                Configuration[] criteria = child.getChild(__PARAM_CRITERIA).getChildren(__PARAM_CRITERIA_CRITERION);
233                for (Configuration criterion : criteria)
234                {
235                    String id = criterion.getChild(__PARAM_CRITERIA_CRITERION_ID).getValue();
236                    I18nizableText label = _getCriterionLabel(criterion.getChild(__PARAM_CRITERIA_CRITERION_LABEL), id);
237                    String type = criterion.getChild(__PARAM_CRITERIA_CRITERION_TYPE).getValue("STRING");
238                    
239                    _criteria.add(new ApogeeCriterion(id, label, type));
240                }
241
242                Configuration[] columns = child.getChild(__PARAM_COLUMNS).getChildren(__PARAM_COLUMNS_COLUMN);
243                for (Configuration column : columns)
244                {
245                    _columns.add(column.getValue());
246                }
247            }
248        }
249        catch (Exception e)
250        {
251            throw new ConfigurationException("Error while parsing apogee-mapping.xml", e);
252        }
253        finally
254        {
255            IOUtils.closeQuietly(is);
256        }
257    }
258    
259    private I18nizableText _getCriterionLabel(Configuration configuration, String defaultValue)
260    {
261        if (configuration.getAttributeAsBoolean("i18n", false))
262        {
263            return new I18nizableText("plugin.odf-sync", configuration.getValue(defaultValue));
264        }
265        else
266        {
267            return new I18nizableText(configuration.getValue(defaultValue));
268        }
269    }
270    
271    @Override
272    public List<ModifiableDefaultContent> populate(Logger logger)
273    {
274        boolean isRequestAttributeOwner = false;
275        
276        Request request = ContextHelper.getRequest(_context);
277        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) == null)
278        {
279            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS, new HashSet<String>());
280            isRequestAttributeOwner = true;
281        }
282        
283        List<ModifiableDefaultContent> populatedContents = super.populate(logger);
284        
285        if (isRequestAttributeOwner)
286        {
287            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
288        }
289
290        return populatedContents;
291    }
292    
293    @Override
294    protected List<ModifiableDefaultContent> _internalPopulate(Logger logger)
295    {
296        return _importOrSynchronizeContents(Map.of("isGlobalSync", true), false, logger);
297    }
298    
299    @Override
300    protected Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger)
301    {
302        Map<String, Object> searchParams = new HashMap<>(parameters);
303        if (offset > 0)
304        {
305            searchParams.put("__offset", offset);
306        }
307        if (limit < Integer.MAX_VALUE)
308        {
309            searchParams.put("__limit", offset + limit);
310        }
311        searchParams.put("__order", _getSort(sort));
312
313        // We don't use session.selectMap which reorder data
314        List<Map<String, Object>> requestValues = _search(searchParams, logger);
315
316        // Transform CLOBs to String
317        Set<String> clobColumns = getClobColumns();
318        if (!clobColumns.isEmpty())
319        {
320            for (Map<String, Object> contentValues : requestValues)
321            {
322                String idValue = contentValues.get(getIdColumn()).toString();
323                for (String clobKey : getClobColumns())
324                {
325                    // Get the old values for the CLOB
326                    @SuppressWarnings("unchecked")
327                    Optional<List<Object>> oldValues = Optional.of(clobKey)
328                        .map(contentValues::get)
329                        .map(obj -> (List<Object>) obj);
330                    
331                    if (oldValues.isPresent())
332                    {
333                        // Get the new values for the CLOB
334                        List<Object> newValues = oldValues.get()
335                            .stream()
336                            .map(value -> _transformClobToString(value, idValue, logger))
337                            .filter(Objects::nonNull)
338                            .collect(Collectors.toList());
339                        
340                        // Set the transformed CLOB values
341                        contentValues.put(clobKey, newValues);
342                    }
343                }
344            }
345        }
346        
347        // Reorganize results
348        String idColumn = getIdColumn();
349        Map<String, Map<String, Object>> results = new LinkedHashMap<>();
350        for (Map<String, Object> contentValues : requestValues)
351        {
352            results.put(contentValues.get(idColumn).toString(), contentValues);
353        }
354        
355        for (Map<String, Object> result : results.values())
356        {
357            result.put(SCC_UNIQUE_ID, result.get(getIdColumn()));
358        }
359        
360        return results;
361    }
362    
363    @Override
364    protected Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger)
365    {
366        Map<String, Map<String, List<Object>>> remoteValues = new HashMap<>();
367        
368        Map<String, Map<String, Object>> results = internalSearch(parameters, 0, Integer.MAX_VALUE, null, logger);
369        
370        if (results != null)
371        {
372            remoteValues = _sccHelper.organizeRemoteValuesByMetadata(results, _mapping);
373        }
374        
375        return remoteValues;
376    }
377    
378    @Override
379    protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger)
380    {
381        return _importOrSynchronizeContent(idValue, _odfLang, remoteValues, forceImport, logger);
382    }
383
384    @Override
385    public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception
386    {
387        boolean isRequestAttributeOwner = false;
388        
389        Request request = ContextHelper.getRequest(_context);
390        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) == null)
391        {
392            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS, new HashSet<String>());
393            isRequestAttributeOwner = true;
394        }
395
396        List<ModifiableDefaultContent> createdContents = new ArrayList<>();
397        
398        Map<String, Object> parameters = putIdParameter(idValue);
399        Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger);
400        if (!results.isEmpty())
401        {
402            try
403            {
404                createdContents.add(_importContent(idValue, importParams, _odfLang, results.get(idValue), logger));
405            }
406            catch (Exception e)
407            {
408                _nbError++;
409                logger.error("An error occurred while importing or synchronizing content", e);
410            }
411        }
412        
413        if (isRequestAttributeOwner)
414        {
415            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
416        }
417
418        return createdContents;
419    }
420    
421    @Override
422    public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception
423    {
424        boolean isRequestAttributeOwner = false;
425        
426        Request request = ContextHelper.getRequest(_context);
427        if (request.getAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS) == null)
428        {
429            request.setAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS, new HashSet<String>());
430            isRequestAttributeOwner = true;
431        }
432        
433        super.synchronizeContent(content, logger);
434        
435        if (isRequestAttributeOwner)
436        {
437            request.removeAttribute(ApogeeSynchronizableContentsCollectionHelper.HANDLE_CONTENTS);
438        }
439    }
440    
441    @Override
442    protected Map<String, Object> putIdParameter(String idValue)
443    {
444        Map<String, Object> parameters = new HashMap<>();
445        parameters.put(getIdField(), idValue);
446        return parameters;
447    }
448    
449    /**
450     * Search the contents with the search parameters. Use id parameter to search an unique content.
451     * @param searchParams Search parameters
452     * @param logger The logger
453     * @return A Map of mapped metadatas extract from Apogée database ordered by content unique Apogée ID
454     */
455    protected abstract List<Map<String, Object>> _search(Map<String, Object> searchParams, Logger logger);
456    
457    /**
458     * Convert the {@link BigDecimal} values retrieved from database into long values
459     * @param searchResults The initial search results from database
460     * @return The converted search results
461     */
462    protected List<Map<String, Object>> _convertBigDecimal(List<Map<String, Object>> searchResults)
463    {
464        List<Map<String, Object>> convertedSearchResults = new ArrayList<>();
465        
466        for (Map<String, Object> searchResult : searchResults)
467        {
468            for (String key : searchResult.keySet())
469            {
470                searchResult.put(key, _convertBigDecimal(getContentType(), key, searchResult.get(key))); 
471            }
472            
473            convertedSearchResults.add(searchResult);
474        }
475        
476        return convertedSearchResults;
477    }
478    
479    /**
480     * Convert the object in parameter to a long if it's a {@link BigDecimal}, otherwise return the object itself.
481     * @param contentTypeId The content type of the parent content
482     * @param attributeName The metadata name
483     * @param objectToConvert The object to convert if necessary
484     * @return The converted object
485     */
486    protected Object _convertBigDecimal(String contentTypeId, String attributeName, Object objectToConvert)
487    {
488        if (objectToConvert instanceof BigDecimal)
489        {
490            ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
491            if (contentType.hasModelItem(attributeName))
492            {
493                ModelItem definition = contentType.getModelItem(attributeName);    
494                String typeId = definition.getType().getId();
495                switch (typeId)
496                {
497                    case ModelItemTypeConstants.DOUBLE_TYPE_ID:
498                        return ((BigDecimal) objectToConvert).doubleValue();
499                    case ModelItemTypeConstants.LONG_TYPE_ID:
500                        return ((BigDecimal) objectToConvert).longValue();
501                    default:
502                        // Do nothing
503                        break;
504                }
505            }
506            return ((BigDecimal) objectToConvert).toString();
507        }
508        return objectToConvert;
509    }
510    
511    /**
512     * Transform CLOB value to String value.
513     * @param value The input value
514     * @param idValue The identifier of the program
515     * @param logger The logger
516     * @return the same value, with CLOB transformed to String.
517     */
518    protected Object _transformClobToString(Object value, String idValue, Logger logger)
519    {
520        if (value instanceof Clob)
521        {
522            Clob clob = (Clob) value;
523            try
524            {
525                String strValue = IOUtils.toString(clob.getCharacterStream());
526                return CharMatcher.javaIsoControl().and(CharMatcher.anyOf("\r\n\t").negate()).removeFrom(strValue);
527            }
528            catch (SQLException | IOException e)
529            {
530                logger.error("Unable to get education add elements from the program '{}'.", idValue, e);
531                return null;
532            }
533            finally
534            {
535                try
536                {
537                    clob.free();
538                }
539                catch (SQLException e)
540                {
541                    // Ignore the exception.
542                }
543            }
544        }
545        
546        return value;
547    }
548
549    /**
550     * Get the list of CLOB column's names.
551     * @return The list of the CLOB column's names to transform to {@link String}
552     */
553    protected Set<String> getClobColumns()
554    {
555        return Set.of();
556    }
557    
558    /**
559     * Transform a {@link List} of {@link Map} to a {@link Map} of {@link List} computed by keys (lines to columns).
560     * @param lines {@link List} to reorganize
561     * @return {@link Map} of {@link List}
562     */
563    protected Map<String, List<Object>> _transformListOfMap2MapOfList(List<Map<String, Object>> lines)
564    {
565        return lines.stream()
566            .filter(Objects::nonNull)
567            .map(Map::entrySet)
568            .flatMap(Collection::stream)
569            .filter(e -> Objects.nonNull(e.getValue()))
570            .collect(
571                Collectors.toMap(
572                    Entry::getKey,
573                    e -> List.of(e.getValue()),
574                    (l1, l2) -> ListUtils.union(l1, l2)
575                )
576            );
577    }
578    
579    /**
580     * Get the name of the mapping.
581     * @return the mapping name
582     */
583    protected abstract String getMappingName();
584    
585    /**
586     * Get the id of data source
587     * @return The id of data source
588     */
589    protected String getDataSourceId()
590    {
591        return (String) getParameterValues().get(PARAM_DATASOURCE_ID);
592    }
593    
594    /**
595     * Get the administrative year
596     * @return The administrative year
597     */
598    protected String getYear()
599    {
600        return (String) getParameterValues().get(PARAM_YEAR);
601    }
602    
603    /**
604     * Check if unexisting contents in Ametys should be added from data source
605     * @return <code>true</code> if unexisting contents in Ametys should be added from data source, default value is <code>false</code>
606     */
607    protected boolean addUnexistingChildren()
608    {
609        return Boolean.valueOf((String) getParameterValues().getOrDefault(PARAM_ADD_UNEXISTING_CHILDREN, Boolean.TRUE.toString()));
610    }
611    
612    /**
613     * Get the identifier column (can be a concatened column).
614     * @return the column id
615     */
616    protected String getIdColumn()
617    {
618        return _idColumn;
619    }
620    
621    @Override
622    public String getIdField()
623    {
624        return "apogeeSyncCode";
625    }
626
627    @Override
628    public Set<String> getLocalAndExternalFields(Map<String, Object> additionalParameters)
629    {
630        return _syncFields;
631    }
632
633    @Override
634    public Set<String> getExternalOnlyFields(Map<String, Object> additionalParameters)
635    {
636        return _extFields;
637    }
638    
639    @Override
640    protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
641    {
642        ModifiableCompositeMetadata holder = content.getMetadataHolder();
643        
644        boolean hasChanges = _fillRichTexts(remoteValues, content, create, logger);
645        hasChanges = super._fillContent(remoteValues, content, create, logger) || hasChanges;
646        hasChanges = _handleAdditionalMetadata(holder, create) || hasChanges;
647        
648        if (!holder.hasMetadata(getIdField()))
649        {
650            holder.setMetadata(getIdField(), remoteValues.get(getIdField()).get(0).toString());
651            hasChanges |= create;
652        }
653        
654        return hasChanges;
655    }
656    
657    /**
658     * Fill the richt texts of the content with the remote values.
659     * @param remoteValues The remote values
660     * @param content The content to synchronize
661     * @param create <code>true</code> if content is creating, false if it is updated
662     * @param logger The logger
663     * @return <code>true</code> if changes were made
664     */
665    protected boolean _fillRichTexts(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger)
666    {
667        boolean hasChanges = false;
668
669        ModifiableCompositeMetadata holder = content.getMetadataHolder();
670        Map<String, Object> params = Map.of("contentTypes", List.of(getContentType()));
671        
672        for (String metadataName : getRichTextFields())
673        {
674            if (remoteValues.containsKey(metadataName))
675            {
676                boolean synchronize = getLocalAndExternalFields(params).contains(metadataName);
677                
678                ModifiableRichText richText = ExternalizableMetadataHelper.getRichText(holder, metadataName, synchronize ? ExternalizableMetadataStatus.EXTERNAL : ExternalizableMetadataStatus.LOCAL, true);
679                richText.setMimeType("text/xml");
680                richText.setLastModified(new Date());
681                try
682                {
683                    _setRichTextValue(richText, remoteValues.get(metadataName));
684                }
685                catch (IOException e)
686                {
687                    logger.error("An error occured while parsing the rich text '{}' of the content '{}'", metadataName, content.getTitle(), e);
688                }
689                
690                if (synchronize && create)
691                {
692                    // Force external status
693                    ExternalizableMetadataHelper.updateStatus(holder,  metadataName, ExternalizableMetadataStatus.EXTERNAL);
694                }
695                
696                remoteValues.remove(metadataName);
697                hasChanges = true;
698            }
699        }
700        
701        return hasChanges;
702    }
703    
704    /**
705     * Set the value on the rich text.
706     * @param richText The rich text to update
707     * @param remoteValue The remote value of the rich text (can be composed of several texts to concatenate)
708     * @throws IOException if an exception occurs
709     */
710    protected void _setRichTextValue(ModifiableRichText richText, List<Object> remoteValue) throws IOException
711    {
712        List<String> lines = new LinkedList<>();
713        for (Object remoteText : remoteValue)
714        {
715            lines.add(remoteText.toString());
716        }
717        
718        String docbook = ContentImporterHelper.textToDocbook(lines.toArray(new String[lines.size()]));
719        
720        try (ByteArrayInputStream is =  new ByteArrayInputStream(docbook.getBytes("UTF-8")))
721        {
722            richText.setInputStream(is);
723        }
724    }
725
726    /**
727     * Method to add additional metadata on import or synchronize.
728     * @param holder The holder of the content to update
729     * @param create If we are on creation mode
730     * @return <code>true</code> if changes has been made
731     */
732    protected boolean _handleAdditionalMetadata(ModifiableCompositeMetadata holder, boolean create)
733    {
734        // Do nothing
735        return false;
736    }
737
738    /**
739     * Get the list of rich text fields of the imported content.
740     * @return The list of the rich text fields metadata name
741     */
742    protected Set<String> getRichTextFields()
743    {
744        return new HashSet<>();
745    }
746    
747    @Override
748    protected void configureSearchModel()
749    {
750        for (ApogeeCriterion criterion : _criteria)
751        {
752            _searchModelConfiguration.addCriterion(criterion.getId(), criterion.getLabel(), criterion.getType());
753        }
754        for (String columnName : _columns)
755        {
756            _searchModelConfiguration.addColumn(columnName);
757        }
758    }
759    
760    @Override
761    protected boolean additionalCommonOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, boolean create, Logger logger)
762    {
763        boolean hasChanges = super.additionalCommonOperations(content, remoteValues, importParams, create, logger);
764
765        Object parentId = importParams != null && importParams.containsKey("parentId") ? importParams.get("parentId") : null;
766        ModifiableDefaultContent parentContent = parentId != null ? _resolver.resolveById(parentId.toString()) : null;
767        
768        hasChanges = handleParent(content, parentContent, logger) || hasChanges;
769        hasChanges = handleChildren(content, create, logger) || hasChanges;
770        hasChanges = setAdditionalMetadata(content, remoteValues, create, logger) || hasChanges;
771        
772        return hasChanges;
773    }
774    
775    /**
776     * Set the parent metadata and invert relation.
777     * @param currentContent Current content
778     * @param parentContent Parent content to set
779     * @param logger The logger
780     * @return <code>true</code> if there are changes
781     */
782    protected boolean handleParent(ModifiableDefaultContent currentContent, ModifiableDefaultContent parentContent, Logger logger)
783    {
784        // Nothing to do by default
785        return false;
786    }
787    
788    /**
789     * Set the children metadata and invert relation, import and synchronize the children too.
790     * @param content Current content
791     * @param create true if the content has been newly created
792     * @param logger The logger
793     * @return <code>true</code> if there are changes
794     */
795    protected boolean handleChildren(ModifiableDefaultContent content, boolean create, Logger logger)
796    {
797        return importOrSynchronizeChildren(content, getChildrenSCCModelId(), getChildrenMetadataName(), getChildrenInvertMetadataName(), create, logger);
798    }
799    
800    /**
801     * Get the children SCC model id. Can be null if no implementation is defined.
802     * @return the children SCC model id
803     */
804    protected String getChildrenSCCModelId()
805    {
806        // Default implementation
807        return null;
808    }
809    
810    /**
811     * Get the metadata name to get children
812     * @return the metadata name to get children
813     */
814    protected abstract String getChildrenMetadataName();
815
816    /**
817     * Get the metadata name to get parent from children
818     * @return the metadata name to get parent from children
819     */
820    protected abstract String getChildrenInvertMetadataName();
821
822    /**
823     * Set the additional metadata.
824     * @param content Current content
825     * @param remoteValues Values of the content
826     * @param create true if the content has been newly created
827     * @param logger The logger
828     * @return <code>true</code> if there are changes
829     */
830    protected boolean setAdditionalMetadata(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, boolean create, Logger logger)
831    {
832        // Nothing to do by default
833        return false;
834    }
835    
836    /**
837     * Transform the given {@link List} of {@link Object} to a {@link String} representing the ordered fields for SQL.
838     * @param sortList The sort list object to transform to the list of ordered fields compatible with SQL.
839     * @return A string representing the list of ordered fields
840     */
841    @SuppressWarnings("unchecked")
842    protected String _getSort(List<Object> sortList)
843    {
844        if (sortList != null)
845        {
846            StringBuilder sort = new StringBuilder();
847            
848            for (Object sortValueObj : sortList)
849            {
850                Map<String, Object> sortValue = (Map<String, Object>) sortValueObj;
851                
852                sort.append(sortValue.get("property"));
853                if (sortValue.containsKey("direction"))
854                {
855                    sort.append(" ");
856                    sort.append(sortValue.get("direction"));
857                    sort.append(",");
858                }
859                else
860                {
861                    sort.append(" ASC,");
862                }
863            }
864            
865            sort.deleteCharAt(sort.length() - 1);
866            
867            return sort.toString();
868        }
869        
870        return null;
871    }
872    
873    @Override
874    public int getTotalCount(Map<String, Object> parameters, Logger logger)
875    {
876        // Remove empty parameters
877        Map<String, Object> searchParams = new HashMap<>();
878        for (String parameterName : parameters.keySet())
879        {
880            Object parameterValue = parameters.get(parameterName);
881            if (parameterValue != null && !parameterValue.toString().isEmpty())
882            {
883                searchParams.put(parameterName, parameterValue);
884            }
885        }
886        
887        searchParams.put("__count", true);
888        
889        List<Map<String, Object>> results = _search(searchParams, logger);
890        if (results != null && !results.isEmpty())
891        {
892            return Integer.valueOf(results.get(0).get("COUNT(*)").toString()).intValue();
893        }
894        
895        return 0;
896    }
897    
898    @Override
899    protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
900    {
901        ModifiableDefaultContent content = super._importContent(idValue, importParams, lang, _transformOrgUnitMetadata(remoteValues, logger), logger);
902        if (content != null)
903        {
904            _apogeeSCCHelper.addToHandleContents(content.getId());
905        }
906        return content;
907    }
908    
909    @Override
910    protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception
911    {
912        if (!_apogeeSCCHelper.addToHandleContents(content.getId()))
913        {
914            return content;
915        }
916        return super._synchronizeContent(content, _transformOrgUnitMetadata(remoteValues, logger), logger);
917    }
918    
919    /**
920     * Import and synchronize children of the given content, then edit the structure of the content and its children.
921     * In the usual case, if we are in import mode (create to true) or if we trust the source (removalSync to true) we just synchronize structure but we don't synchronize child contents
922     * @param content Parent content
923     * @param sccModelId SCC model ID
924     * @param metadataName Metadata name to set
925     * @param invertMetadataName Metadata name of the invert relation
926     * @param create true if the content has been newly created
927     * @param logger The logger
928     * @return <code>true</code> if there are changes
929     */
930    protected boolean importOrSynchronizeChildren(ModifiableDefaultContent content, String sccModelId, String metadataName, String invertMetadataName, boolean create, Logger logger)
931    {
932        // Get the SCC for children
933        SynchronizableContentsCollection scc = _sccHelper.getSCCFromModelId(sccModelId);
934        
935        return create
936            // Import mode
937            ? _importChildren(content, scc, metadataName, invertMetadataName, logger)
938            // Synchronization mode
939            : _synchronizeChildren(content, scc, metadataName, invertMetadataName, logger);
940    }
941    
942    
943
944    /**
945     * Import children
946     * @param content Parent content
947     * @param scc the scc
948     * @param metadataName Metadata name to set
949     * @param invertMetadataName Metadata name of the invert relation
950     * @param logger The logger
951     * @return <code>true</code> if there are changes
952     */
953    protected boolean _importChildren(ModifiableDefaultContent content, SynchronizableContentsCollection scc, String metadataName, String invertMetadataName, Logger logger)
954    {
955        // If SCC exists, search for children
956        if (scc != null && scc instanceof ApogeeSynchronizableContentsCollection)
957        {
958            // Synchronize or import children content
959            ModifiableCompositeMetadata cm = content.getMetadataHolder();
960            String syncCode = cm.getString(getIdField());
961            List<ModifiableDefaultContent> importedOrSynchronizedChildren = ((ApogeeSynchronizableContentsCollection) scc).importOrSynchronizeContents(ImmutableMap.of("parentCode", syncCode), logger);
962
963            return _resetChildren(content, importedOrSynchronizedChildren, metadataName, invertMetadataName, logger);
964        }
965        
966        return false;
967    }
968    
969    /**
970     * Synchronize children
971     * @param content Parent content
972     * @param scc the scc
973     * @param metadataName Metadata name to set
974     * @param invertMetadataName Metadata name of the invert relation
975     * @param logger The logger
976     * @return <code>true</code> if there are changes
977     */
978    protected boolean _synchronizeChildren(ModifiableDefaultContent content, SynchronizableContentsCollection scc, String metadataName, String invertMetadataName, Logger logger)
979    {
980        // Get the children remote sync codes if needed
981        // These remote sync codes are needed if we want to add unexisting contents in Ametys from Apogee or delete obsolete contents 
982        Set<String> childrenRemoteSyncCode = (removalSync() || addUnexistingChildren())
983            ? _getChildrenRemoteSyncCode(content, scc, logger)
984            : null;
985        
986        // First we get the existing children in Ametys
987        List<ModifiableContent> ametysChildren = Stream.of(content.getValue(metadataName, false, new ContentValue[0]))
988            .map(ContentValue::getContentIfExists)
989            .filter(Optional::isPresent)
990            .map(Optional::get)
991            .collect(Collectors.toList());
992        
993        // Remove obsolete children if removeSync is active
994        if (childrenRemoteSyncCode != null && removalSync())
995        {
996            ametysChildren = ametysChildren.stream()
997                .filter(c -> !_isChildWillBeRemoved(c, scc, childrenRemoteSyncCode, logger))
998                .collect(Collectors.toList());
999        }
1000        
1001        // Then we synchronize the existing children in Ametys
1002        for (ModifiableContent childContent : ametysChildren)
1003        {
1004            _apogeeSCCHelper.synchronizeContent(childContent, logger);
1005        }
1006        
1007        // Then we add unexisting children if needed
1008        if (childrenRemoteSyncCode != null && addUnexistingChildren())
1009        {
1010            ametysChildren.addAll(_addUnexistingChildren(scc, childrenRemoteSyncCode, logger));
1011        }
1012        
1013        // Reset children if needed
1014        if (childrenRemoteSyncCode != null && (removalSync() || addUnexistingChildren()))
1015        {
1016            return _resetChildren(content, ametysChildren, metadataName, invertMetadataName, logger);
1017        }
1018        
1019        return false;
1020    }
1021
1022    /**
1023     * Get the remote sync codes
1024     * @param content Parent content
1025     * @param scc the scc
1026     * @param logger The logger
1027     * @return the remote sync codes or null if the scc is not from Apogee
1028     */
1029    protected Set<String> _getChildrenRemoteSyncCode(ModifiableDefaultContent content, SynchronizableContentsCollection scc, Logger logger)
1030    {
1031        if (scc != null && scc instanceof AbstractApogeeSynchronizableContentsCollection)
1032        {
1033            String syncCode = content.getMetadataHolder().getString(getIdField());
1034            return ((AbstractApogeeSynchronizableContentsCollection) scc)
1035                        .search(ImmutableMap.of("parentCode", syncCode), 0, Integer.MAX_VALUE, null, logger)
1036                        .keySet();
1037        }
1038        
1039        return null;
1040    }
1041
1042    /**
1043     * Add unexisting children in Ametys from Apogee
1044     * @param scc the scc to import contents
1045     * @param childrenRemoteSyncCode the remote sync codes
1046     * @param logger the logger
1047     * @return the list of unexisting contents
1048     */
1049    protected List<ModifiableContent> _addUnexistingChildren(SynchronizableContentsCollection scc, Set<String> childrenRemoteSyncCode, Logger logger)
1050    {
1051        return childrenRemoteSyncCode
1052            .stream()
1053            .filter(code -> scc.getContent(_odfLang, code) == null) // get only remote content values which doesn't exist in Ametys
1054            .flatMap(code -> _importUnexistingChildren(scc, code, null, logger))
1055            .collect(Collectors.toList());
1056    }
1057
1058    /**
1059     * Import an unexisting child in Ametys
1060     * @param scc the scc to import the content
1061     * @param syncCode the sync code of the content
1062     * @param params the additional params
1063     * @param logger the logger
1064     * @return the list of created children
1065     */
1066    @SuppressWarnings("unchecked")
1067    protected Stream<ModifiableDefaultContent> _importUnexistingChildren(SynchronizableContentsCollection scc, String syncCode, Map<String, List<Object>> params, Logger logger)
1068    {
1069        try
1070        {
1071            return scc.importContent(syncCode, (Map<String, Object>) (Object) params, logger).stream();
1072        }
1073        catch (Exception e)
1074        {
1075            logger.error("An error occured while importing a new children content with syncCode '{}' from SCC '{}'", syncCode, scc.getId(), e);
1076        }
1077        
1078        return Stream.empty();
1079    }
1080    
1081    /**
1082     * True if the content will be removed from the structure
1083     * @param content the content
1084     * @param scc the scc
1085     * @param childrenRemoteSyncCode the remote sync codes
1086     * @param logger the logger
1087     * @return <code>true</code> if the content will be removed from the structure
1088     */
1089    protected boolean _isChildWillBeRemoved(ModifiableContent content, SynchronizableContentsCollection scc, Set<String> childrenRemoteSyncCode, Logger logger)
1090    {
1091        String syncCode = content.getMetadataHolder().getString(scc.getIdField(), null);
1092        return !childrenRemoteSyncCode.contains(syncCode);
1093    }
1094    
1095    /**
1096     * Reset the children of the structure to force with the children from Apogée.
1097     * @param content Parent content
1098     * @param importedOrSynchronizedChildren the imported or synchronized children
1099     * @param metadataName Metadata name to set
1100     * @param invertMetadataName Metadata name of the invert relation
1101     * @param logger The logger
1102     * @return <code>true</code> if there are changes
1103     */
1104    protected boolean _resetChildren(ModifiableDefaultContent content, List<? extends ModifiableContent> importedOrSynchronizedChildren , String metadataName, String invertMetadataName, Logger logger)
1105    {
1106        boolean hasChanges = false;
1107
1108        ModifiableCompositeMetadata cm = content.getMetadataHolder();
1109        
1110        // Remove all children relations
1111        ContentValue[] oldChildren = content.getValue(metadataName, false, new ContentValue[0]);
1112        if (oldChildren.length > 0)
1113        {
1114            hasChanges = ExternalizableMetadataHelper.removeMetadataIfExists(cm, metadataName) || hasChanges;
1115            
1116            for (ContentValue child : oldChildren)
1117            {
1118                Optional<ModifiableContent> childContent = child.getContentIfExists();
1119                if (childContent.isPresent())
1120                {
1121                    hasChanges = _updateRelation(childContent.get().getMetadataHolder(), invertMetadataName, content, true) || hasChanges;
1122                }
1123            }
1124        }
1125        
1126        if (!importedOrSynchronizedChildren.isEmpty())
1127        {
1128            // Update child relation to link to the parent
1129            ModifiableDefaultContent[] newChildren = importedOrSynchronizedChildren.toArray(new ModifiableDefaultContent[importedOrSynchronizedChildren.size()]);
1130            hasChanges = ExternalizableMetadataHelper.setMetadata(cm, metadataName, newChildren) || hasChanges;
1131
1132            // Update parent relation on each children
1133            for (ModifiableDefaultContent child : newChildren)
1134            {
1135                hasChanges = _updateRelation(child.getMetadataHolder(), invertMetadataName, content) || hasChanges;
1136            }
1137        }
1138        
1139        return hasChanges;
1140    }
1141
1142    @Override
1143    public List<ModifiableDefaultContent> importOrSynchronizeContents(Map<String, Object> searchParams, Logger logger)
1144    {
1145        return _importOrSynchronizeContents(searchParams, true, logger);
1146    }
1147    
1148    @SuppressWarnings("unchecked")
1149    private Map<String, List<Object>> _transformOrgUnitMetadata(Map<String, List<Object>> remoteValues, Logger logger)
1150    {
1151        // Transform orgUnit values and import content if necessary (useful for Course and SubProgram)
1152        SynchronizableContentsCollection scc = _sccHelper.getSCCFromModelId(OrgUnitSynchronizableContentsCollection.MODEL_ID);
1153        
1154        List<Object> orgUnitCodes = remoteValues.get("orgUnit");
1155        if (orgUnitCodes != null && !orgUnitCodes.isEmpty())
1156        {
1157            List<?> orgUnitContents = null;
1158            String orgUnitCode = orgUnitCodes.get(0).toString();
1159            
1160            if (scc != null)
1161            {
1162                try
1163                {
1164                    ModifiableDefaultContent orgUnitContent = scc.getContent(_odfLang, orgUnitCode);
1165                    if (orgUnitContent == null)
1166                    {
1167                        orgUnitContents = scc.importContent(orgUnitCode, null, logger);
1168                    }
1169                    else
1170                    {
1171                        orgUnitContents = ImmutableList.of(orgUnitContent);
1172                    }
1173                }
1174                catch (Exception e)
1175                {
1176                    logger.error("An error occured during the import of the OrgUnit identified by the synchronization code '{}'", orgUnitCode, e);
1177                }
1178            }
1179            
1180            if (orgUnitContents == null)
1181            {
1182                // Impossible link to orgUnit
1183                remoteValues.remove("orgUnit");
1184                logger.warn("Impossible to import the OrgUnit with the synchronization code '{}', check if you set the OrgUnit SCC with the following model ID: '{}'", orgUnitCode, OrgUnitSynchronizableContentsCollection.MODEL_ID);
1185            }
1186            else
1187            {
1188                remoteValues.put("orgUnit", (List<Object>) orgUnitContents);
1189            }
1190        }
1191        
1192        return remoteValues;
1193    }
1194
1195    @Override
1196    public boolean handleRightAssignmentContext()
1197    {
1198        // Rights on ODF contents are handled by ODFRightAssignmentContext
1199        return false;
1200    }
1201}