001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.cms.content.indexing.solr;
017
018import java.io.IOException;
019import java.nio.ByteBuffer;
020import java.nio.CharBuffer;
021import java.nio.charset.CharsetDecoder;
022import java.nio.charset.CodingErrorAction;
023import java.nio.charset.StandardCharsets;
024import java.text.SimpleDateFormat;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collection;
028import java.util.Comparator;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.List;
032import java.util.Map;
033import java.util.Objects;
034import java.util.Set;
035import java.util.function.Function;
036import java.util.stream.Collectors;
037import java.util.stream.StreamSupport;
038
039import javax.jcr.RepositoryException;
040
041import org.apache.avalon.framework.activity.Initializable;
042import org.apache.avalon.framework.component.Component;
043import org.apache.avalon.framework.context.Context;
044import org.apache.avalon.framework.context.ContextException;
045import org.apache.avalon.framework.context.Contextualizable;
046import org.apache.avalon.framework.service.ServiceException;
047import org.apache.avalon.framework.service.ServiceManager;
048import org.apache.avalon.framework.service.Serviceable;
049import org.apache.cocoon.components.ContextHelper;
050import org.apache.cocoon.environment.Request;
051import org.apache.commons.collections4.IterableUtils;
052import org.apache.commons.lang3.ObjectUtils;
053import org.apache.commons.lang3.StringUtils;
054import org.apache.solr.client.solrj.SolrClient;
055import org.apache.solr.client.solrj.SolrQuery;
056import org.apache.solr.client.solrj.SolrResponse;
057import org.apache.solr.client.solrj.SolrServerException;
058import org.apache.solr.client.solrj.request.CoreAdminRequest;
059import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
060import org.apache.solr.client.solrj.request.schema.SchemaRequest;
061import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update;
062import org.apache.solr.client.solrj.response.CoreAdminResponse;
063import org.apache.solr.client.solrj.response.QueryResponse;
064import org.apache.solr.client.solrj.response.SolrResponseBase;
065import org.apache.solr.client.solrj.response.UpdateResponse;
066import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
067import org.apache.solr.client.solrj.response.schema.SchemaResponse;
068import org.apache.solr.client.solrj.util.ClientUtils;
069import org.apache.solr.common.SolrInputDocument;
070import org.apache.solr.common.util.NamedList;
071import org.slf4j.Logger;
072
073import org.ametys.cms.indexing.IndexingException;
074import org.ametys.cms.indexing.solr.ReloadAclCacheRequest;
075import org.ametys.cms.indexing.solr.UpdateAclCacheRequest;
076import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest;
077import org.ametys.cms.repository.Content;
078import org.ametys.cms.repository.ContentQueryHelper;
079import org.ametys.cms.repository.WorkflowAwareContent;
080import org.ametys.cms.rights.solrchecking.ReadAccessHelper;
081import org.ametys.cms.search.query.ContentAttachmentQuery;
082import org.ametys.cms.search.query.DocumentTypeQuery;
083import org.ametys.cms.search.query.OrQuery;
084import org.ametys.cms.search.query.Query;
085import org.ametys.cms.search.query.ResourceLocationQuery;
086import org.ametys.cms.search.solr.NoAutoCommitUpdateClient;
087import org.ametys.cms.search.solr.SolrClientProvider;
088import org.ametys.cms.search.solr.schema.SchemaDefinition;
089import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider;
090import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint;
091import org.ametys.cms.search.solr.schema.SchemaFields;
092import org.ametys.cms.search.solr.schema.SchemaHelper;
093import org.ametys.core.group.GroupIdentity;
094import org.ametys.core.right.AllowedUsers;
095import org.ametys.core.schedule.progression.ContainerProgressionTracker;
096import org.ametys.core.schedule.progression.ProgressionTrackerFactory;
097import org.ametys.core.schedule.progression.SimpleProgressionTracker;
098import org.ametys.core.user.UserIdentity;
099import org.ametys.core.util.DateUtils;
100import org.ametys.plugins.explorer.resources.Resource;
101import org.ametys.plugins.explorer.resources.ResourceCollection;
102import org.ametys.plugins.repository.AmetysObject;
103import org.ametys.plugins.repository.AmetysObjectIterable;
104import org.ametys.plugins.repository.AmetysObjectResolver;
105import org.ametys.plugins.repository.RepositoryConstants;
106import org.ametys.plugins.repository.TraversableAmetysObject;
107import org.ametys.plugins.repository.UnknownAmetysObjectException;
108import org.ametys.plugins.repository.collection.AmetysObjectCollection;
109import org.ametys.plugins.repository.provider.AbstractRepository;
110import org.ametys.plugins.repository.provider.JackrabbitRepository;
111import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector;
112import org.ametys.plugins.repository.provider.WorkspaceSelector;
113import org.ametys.runtime.config.Config;
114import org.ametys.runtime.i18n.I18nizableText;
115import org.ametys.runtime.plugin.component.AbstractLogEnabled;
116
117/**
118 * Solr indexer.
119 */
120public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable
121{
122    /** The component role. */
123    public static final String ROLE = SolrIndexer.class.getName();
124    
125    private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>();
126    
127    private static final String _CONFIGSET_NAME_PREFIX = "configset-";
128    
129    private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_");
130    private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general");
131    
132    private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766;
133    
134    /** The ametys object resolver. */
135    protected AmetysObjectResolver _resolver;
136    /** The schema definition provider extension point. */
137    protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP;
138    /** The schema helper. */
139    protected SchemaHelper _schemaHelper;
140    /** Solr Ametys contents indexer */
141    protected SolrContentIndexer _solrContentIndexer;
142    /** Solr workflow indexer. */
143    protected SolrWorkflowIndexer _solrWorkflowIndexer;
144    /** Solr resource indexer. */
145    protected SolrResourceIndexer _solrResourceIndexer;
146    
147    /** The Solr client provider */
148    protected SolrClientProvider _solrClientProvider;
149    
150    /** The solr core prefix. */
151    protected String _solrCorePrefix;
152    /** The Ametys internal URL used by Solr to query Ametys */
153    protected String _ametysInternalUrl;
154    
155    /** The workspace selector. */
156    protected WorkspaceSelector _workspaceSelector;
157    /** The JCR repository */
158    protected JackrabbitRepository _repository;
159    /** The helper for read access */
160    protected ReadAccessHelper _readAccessHelper;
161    
162    /** The avalon context */
163    protected Context _context;
164
165    /**
166     * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone
167     * @return The date format for indexing dates
168     * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead
169     */
170    @Deprecated
171    public static SimpleDateFormat dateFormat()
172    {
173        if (__DATE_FORMAT.get() == null)
174        {
175            __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
176        }
177        return __DATE_FORMAT.get();
178    }
179    
180    /**
181     * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr.
182     * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept.
183     * @param value The string value to index
184     * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log.
185     * @param documentId The id of the document being indexed. Can be null if you do not want to log.
186     * @param fieldName The name of the field being indexed. Can be null if you do not want to log.
187     * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes)
188     */
189    public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName)
190    {
191        if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT)
192        {
193            // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length())
194            // As a result, value.getBytes().length <= limit
195            return value;
196        }
197        
198        // There is a doubt, the string may need to be truncated (or not)
199        byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
200        int bytesLength = valueBytes.length;
201        if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT)
202        {
203            return value;
204        }
205        
206        if (ObjectUtils.allNotNull(logger, documentId, fieldName))
207        {
208            logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT);
209        }
210        
211        // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974)
212        CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT);
213        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
214                                                       .onMalformedInput(CodingErrorAction.IGNORE);
215        decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true);
216        decoder.flush(charBuffer);
217        return new String(charBuffer.array(), 0, charBuffer.position());
218    }
219    
220    @Override
221    public void service(ServiceManager serviceManager) throws ServiceException
222    {
223        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
224        _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE);
225        _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE);
226        _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE);
227        _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE);
228        _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE);
229        _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE);
230        _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE);
231        _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE);
232        _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE);
233    }
234    
235    @Override
236    public void initialize() throws Exception
237    {
238        Config config = Config.getInstance();
239        _solrCorePrefix = config.getValue("cms.solr.core.prefix");
240        
241        _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url");
242        if (StringUtils.isBlank(_ametysInternalUrl))
243        {
244            // fallback to CMS URL as internal URL is an optional parameter
245            _ametysInternalUrl = config.getValue("cms.url");
246        }
247    }
248    
249    @Override
250    public void contextualize(Context context) throws ContextException
251    {
252        _context = context;
253    }
254    
255    /**
256     * Gets the 'autocommit' Solr client
257     * @param workspaceName The name of the workspace
258     * @return the Solr client
259     */
260    protected SolrClient _getAutoCommitSolrClient(String workspaceName)
261    {
262        return _solrClientProvider.getUpdateClient(workspaceName, true);
263    }
264    
265    /**
266     * Gets the 'no autocommit' Solr client
267     * @param workspaceName The name of the workspace
268     * @return the Solr client
269     */
270    protected SolrClient _getNoAutoCommitSolrClient(String workspaceName)
271    {
272        return _solrClientProvider.getUpdateClient(workspaceName, false);
273    }
274    
275    // for admin operations
276    private SolrClient _defaultSolrClient()
277    {
278        return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE);
279    }
280    
281    /**
282     * Get the names of the Solr cores.
283     * @return The names of the Solr cores.
284     * @throws IOException If an I/O error occurs.
285     * @throws SolrServerException If a Solr error occurs.
286     */
287    @SuppressWarnings("unchecked")
288    public Set<String> getCoreNames() throws IOException, SolrServerException
289    {
290        Set<String> coreNames = new HashSet<>();
291        
292        getLogger().debug("Getting core list.");
293        
294        SolrQuery query = new SolrQuery();
295        query.setRequestHandler("/admin/cores");
296        query.setParam("action", "STATUS");
297        
298        QueryResponse response = _defaultSolrClient().query(query);
299        
300        NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status");
301        for (Map.Entry<String, NamedList<?>> core : status)
302        {
303            String fullName = (String) core.getValue().get("name");
304            if (fullName.startsWith(_solrCorePrefix))
305            {
306                coreNames.add(fullName.substring(_solrCorePrefix.length()));
307            }
308        }
309        
310        return coreNames;
311    }
312    
313    /**
314     * Get the names of the Solr cores.
315     * @return The names of the Solr cores.
316     * @throws IOException If an I/O error occurs.
317     * @throws SolrServerException If a Solr error occurs.
318     */
319    protected Set<String> getRealCoreNames() throws IOException, SolrServerException
320    {
321        Set<String> coreNames = new HashSet<>();
322        
323        getLogger().debug("Getting core list.");
324        
325        CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient());
326        
327        NamedList<NamedList<Object>> status = response.getCoreStatus();
328        for (Map.Entry<String, NamedList<Object>> core : status)
329        {
330            String fullName = (String) core.getValue().get("name");
331            if (fullName.startsWith(_solrCorePrefix))
332            {
333                coreNames.add(fullName.substring(_solrCorePrefix.length()));
334            }
335        }
336        
337        return coreNames;
338    }
339    
340    /**
341     * Create a Solr core.
342     * @param name The name of the core to create.
343     * @throws IOException If an I/O error occurs.
344     * @throws SolrServerException If a Solr error occurs.
345     */
346    public void createCore(String name) throws IOException, SolrServerException
347    {
348        String fullName = _solrCorePrefix + name;
349        String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix);
350        _createConfigset(configsetName);
351        
352        getLogger().info("Creating core '{}' (full name: '{}').", name, fullName);
353        
354        SolrQuery query = new SolrQuery();
355        query.setRequestHandler("/admin/cores");
356        query.setParam("action", "CREATE");
357        query.setParam("name", fullName);
358        query.setParam("configSet", configsetName);
359        query.setParam("property.ametys.url", _ametysInternalUrl);
360        
361        QueryResponse response = _defaultSolrClient().query(query);
362        NamedList<?> results = response.getResponse();
363        
364        NamedList<?> error = (NamedList<?>) results.get("error");
365        if (error != null)
366        {
367            throw new IOException("Error creating the core: " + error.get("msg"));
368        }
369    }
370    
371    /**
372     * Updates the ametys.url property of the Solr cores.
373     */
374    public void updateAmetysUrlCoreProperty()
375    {
376        Set<String> coreNames;
377        try
378        {
379            coreNames = getCoreNames();
380        }
381        catch (SolrServerException | IOException e)
382        {
383            getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e);
384            return;
385        }
386        
387        for (String coreName : coreNames)
388        {
389            String collection = _solrClientProvider.getCollectionName(coreName);
390            SolrResponseBase response;
391            try
392            {
393                response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection);
394            }
395            catch (SolrServerException | IOException e)
396            {
397                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e);
398                continue;
399            }
400            
401            NamedList<Object> responseParams = response.getResponse();
402            if ("ok".equals(responseParams.get("result")))
403            {
404                Boolean valueChanged = responseParams.getBooleanArg("valueChanged");
405                if (valueChanged)
406                {
407                    getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName);
408                }
409                else
410                {
411                    getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName);
412                }
413            }
414            else
415            {
416                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName);
417            }
418        }
419    }
420    
421    private void _createConfigset(String name) throws IOException, SolrServerException
422    {
423        getLogger().info("Creating (if necessary) configset '{}'", name);
424        
425        // This request handler will check if configset exists. If not it will be created.
426        SolrQuery query = new SolrQuery();
427        query.setRequestHandler("/admin/cores");
428        query.setParam("action", "createConfigset");
429        query.setParam("name", name);
430        
431        QueryResponse response = _defaultSolrClient().query(query);
432        NamedList<?> results = response.getResponse();
433        
434        NamedList<?> error = (NamedList<?>) results.get("error");
435        if (error != null)
436        {
437            throw new IOException("Error creating the core: " + error.get("msg"));
438        }
439    }
440    
441    /**
442     * Delete a Solr core.
443     * @param name The name of the core to delete.
444     * @throws IOException If an I/O error occurs.
445     * @throws SolrServerException If a Solr error occurs.
446     */
447    public void deleteCore(String name) throws IOException, SolrServerException
448    {
449        String fullName = _solrCorePrefix + name;
450        
451        getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName);
452        
453        SolrQuery query = new SolrQuery();
454        query.setRequestHandler("/admin/cores");
455        query.setParam("action", "UNLOAD");
456        query.setParam("core", fullName);
457        query.setParam("deleteInstanceDir", "true");
458        
459        QueryResponse response = _defaultSolrClient().query(query);
460        NamedList<?> results = response.getResponse();
461        
462        NamedList<?> error = (NamedList<?>) results.get("error");
463        if (error != null)
464        {
465            throw new IOException("Error deleting core" + name + ": " + error.get("msg"));
466        }
467    }
468    
469    /**
470     * Send the schema.
471     * @throws IOException If a communication error occurs.
472     * @throws SolrServerException If a solr error occurs.
473     */
474    public void sendSchema() throws IOException, SolrServerException
475    {
476        getLogger().info("Computing and sending the schema to the solr server.");
477        
478        String workspaceName = _workspaceSelector.getWorkspace();
479        String collection = _solrClientProvider.getCollectionName(workspaceName);
480        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
481        
482//        SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema();
483        SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml");
484        
485        // TODO Clear the schema except fields marked ametysReadOnly="true".
486        
487        // Clear the current schema.
488        clearSchema(solrClient, collection);
489        
490        SchemaRequest schemaRequest = new SchemaRequest();
491        SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
492        
493        // The cleared schema contains only the basic fields which can't be deleted.
494        SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation();
495        SchemaFields schemaFields = new SchemaFields(clearedSchema);
496        
497        getLogger().debug("Schema after clear: \n{}", schemaFields.toString());
498        
499        // Add the static schema types and fields.
500        List<SchemaRequest.Update> updates = new ArrayList<>();
501        
502        // Set "add field" definitions from the static schema to the update list.
503        addStaticSchemaUpdates(updates, staticSchema, schemaFields);
504        
505        getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString());
506        
507        // Set "add field" definitions from the static schema to the update list.
508        addCustomUpdates(updates, schemaFields);
509        
510        updates.sort(new SchemaRequestComparator());
511        
512        SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates);
513        SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
514        
515        getLogger().debug("Send schema response: {}", updateResponse.toString());
516        Object errors = updateResponse.getResponse().get("errors");
517        if (errors != null && errors instanceof List && !((List) errors).isEmpty())
518        {
519            String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString();
520            throw new SolrServerException(msg);
521        }
522    
523        getLogger().info("Schema sent to the solr server.");
524        
525        reloadCores();
526    }
527    
528    /**
529     * Compute the list of {@link Update} directives from the static schema.
530     * @param updates The list of {@link Update} directives to fill.
531     * @param staticSchema The static schema representation.
532     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
533     */
534    protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields)
535    {
536        List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes();
537        for (FieldTypeDefinition fieldType : fieldTypes)
538        {
539            String name = (String) fieldType.getAttributes().get("name");
540            if (!schemaFields.hasFieldType(name))
541            {
542                updates.add(new SchemaRequest.AddFieldType(fieldType));
543                schemaFields.addFieldType(name);
544            }
545        }
546        for (Map<String, Object> field : staticSchema.getFields())
547        {
548            String name = (String) field.get("name");
549            if (!schemaFields.hasField(name))
550            {
551                updates.add(new SchemaRequest.AddField(field));
552                schemaFields.addField(name);
553            }
554        }
555        for (Map<String, Object> field : staticSchema.getDynamicFields())
556        {
557            String name = (String) field.get("name");
558            if (!schemaFields.hasDynamicField(name))
559            {
560                updates.add(new SchemaRequest.AddDynamicField(field));
561                schemaFields.addDynamicField(name);
562            }
563        }
564        for (Map<String, Object> field : staticSchema.getCopyFields())
565        {
566            String source = (String) field.get("source");
567            String dest = (String) field.get("dest");
568            if (!schemaFields.hasCopyField(source, dest))
569            {
570                updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest)));
571                schemaFields.addCopyField(source, dest);
572            }
573        }
574    }
575    
576    /**
577     * Compute the list of custom {@link Update} directives.
578     * @param updates The list of {@link Update} directives to fill.
579     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
580     */
581    protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields)
582    {
583        // Add all our property-managed fields.
584        for (String providerId : _schemaDefProviderEP.getExtensionsIds())
585        {
586            SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId);
587            
588            for (SchemaDefinition definition : definitionProvider.getDefinitions())
589            {
590                if (!definition.exists(schemaFields))
591                {
592                    SchemaRequest.Update update = definition.getSchemaUpdate();
593                    if (update != null)
594                    {
595                        updates.add(update);
596                    }
597                }
598            }
599        }
600    }
601    
602    /**
603     * Delete all the fields of the existing schema in the given collection.
604     * @param solrClient The Solr client
605     * @param collection The collection.
606     * @throws IOException If a communication error occurs.
607     * @throws SolrServerException If a solr error occurs.
608     */
609    protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException
610    {
611        try
612        {
613            getLogger().info("Clearing the existing schema on the solr server.");
614            
615            SchemaRequest schemaRequest = new SchemaRequest();
616            SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
617            
618            SchemaRepresentation schema = schemaResponse.getSchemaRepresentation();
619            
620            List<SchemaRequest.Update> deletions = new ArrayList<>();
621            
622            // First the copy fields, then dynamic and simple fields, and field types in the end.
623            for (Map<String, Object> field : schema.getCopyFields())
624            {
625                String source = (String) field.get("source");
626                String dest = (String) field.get("dest");
627                deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest)));
628            }
629            for (Map<String, Object> field : schema.getDynamicFields())
630            {
631                String name = (String) field.get("name");
632                deletions.add(new SchemaRequest.DeleteDynamicField(name));
633            }
634            for (Map<String, Object> field : schema.getFields())
635            {
636                String name = (String) field.get("name");
637                if (!_READ_ONLY_FIELDS.contains(name))
638                {
639                    deletions.add(new SchemaRequest.DeleteField(name));
640                }
641            }
642            for (FieldTypeDefinition fieldType : schema.getFieldTypes())
643            {
644                String name = (String) fieldType.getAttributes().get("name");
645                if (!_READ_ONLY_FIELDTYPES.contains(name))
646                {
647                    deletions.add(new SchemaRequest.DeleteFieldType(name));
648                }
649            }
650            
651            SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions);
652            SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
653            
654            Object errors = updateResponse.getResponse().get("errors");
655            if (errors != null && errors instanceof List && !((List) errors).isEmpty())
656            {
657                String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString();
658                throw new SolrServerException(msg);
659            }
660            else
661            {
662                getLogger().debug("Clear schema response: {}", updateResponse.toString());
663            }
664            
665            getLogger().info("Solr schema cleared.");
666        }
667        catch (SolrServerException | IOException e)
668        {
669            getLogger().error("Error clearing schema in collection " + collection, e);
670            throw e;
671        }
672    }
673    
674    /**
675     * Reload the solr cores.
676     * @throws IOException If a communication error occurs.
677     * @throws SolrServerException If a solr error occurs.
678     */
679    protected void reloadCores() throws IOException, SolrServerException
680    {
681        getLogger().info("Reloading solr cores.");
682        
683        for (String coreName : getCoreNames())
684        {
685            String fullName = _solrCorePrefix + coreName;
686            
687            CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient());
688            
689            getLogger().debug("Reload core response: {}", reloadResponse.toString());
690        }
691        
692        getLogger().info("All cores reloaded.");
693    }
694    
695    /**
696     * Reloads the ACL Solr cache for all users
697     * @throws IOException If an I/O error occurs.
698     * @throws SolrServerException If a Solr error occurs.
699     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
700     */
701    public void reloadAclCache() throws IOException, SolrServerException, RepositoryException
702    {
703        String[] workspaceNames = _repository.getWorkspaces();
704        for (String workspaceName : workspaceNames)
705        {
706            reloadAclCache(workspaceName);
707        }
708    }
709    
710    /**
711     * Reloads the ACL Solr cache for all users
712     * @param workspaceName The workspace name
713     * @throws IOException If an I/O error occurs.
714     * @throws SolrServerException If a Solr error occurs.
715     */
716    public void reloadAclCache(String workspaceName) throws IOException, SolrServerException
717    {
718        reloadAclCache(workspaceName, false);
719    }
720    
721    /**
722     * Reloads the ACL Solr cache for all users
723     * @param workspaceName The workspace name
724     * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache)
725     * @throws IOException If an I/O error occurs.
726     * @throws SolrServerException If a Solr error occurs.
727     */
728    public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException
729    {
730        getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
731        
732        String collection = _solrClientProvider.getCollectionName(workspaceName);
733        SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection);
734        NamedList<Object> responseObj = responseBase.getResponse();
735        
736        if ("ok".equals(responseObj.get("result")))
737        {
738            getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
739        }
740        else
741        {
742            Object error = responseObj.get("error");
743            getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error);
744        }
745    }
746    
747    /**
748     * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces.
749     * @param objects the {@link AmetysObject}s to update.
750     * @throws IOException If an I/O error occurs.
751     * @throws SolrServerException If a Solr error occurs.
752     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
753     */
754    public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException
755    {
756        String[] workspaceNames = _repository.getWorkspaces();
757        for (String workspaceName : workspaceNames)
758        {
759            updateAclCache(objects, workspaceName);
760        }
761    }
762    
763    /**
764     * Updates the ACL Solr cache for some {@link AmetysObject}s.
765     * @param objects the {@link AmetysObject}s to update.
766     * @param workspaceName The workspace name
767     * @throws IOException If an I/O error occurs.
768     * @throws SolrServerException If a Solr error occurs.
769     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
770     */
771    public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException
772    {
773        Map<String, Map<String, Object>> solrParams = new HashMap<>();
774        
775        for (AmetysObject object : objects)
776        {
777            AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object);
778            
779            solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(),
780                                                  "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(),
781                                                  "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
782                                                  "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
783                                                  "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()),
784                                                  "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList())));
785        }
786        
787        String collection = _solrClientProvider.getCollectionName(workspaceName);
788        SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection);
789        NamedList<Object> responseObj = response.getResponse();
790        
791        if ("ok".equals(responseObj.get("result")))
792        {
793            if (getLogger().isInfoEnabled())
794            {
795                getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet());
796            }
797        }
798        else
799        {
800            @SuppressWarnings("unchecked")
801            List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects");
802            getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects);
803        }
804    }
805    
806    /**
807     * Index all the contents in a given workspace.
808     * @param workspaceName the workspace where to index
809     * @param indexAttachments to index content attachments
810     * @param solrClient The solr client to use
811     * @return The indexation result as a Map.
812     * @throws Exception if an error occurs while indexing.
813     */
814    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
815    {
816        return indexAllContents(workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index all contents", getLogger()));
817    }
818    
819    /**
820     * Index all the contents in a given workspace.
821     * @param workspaceName the workspace where to index
822     * @param indexAttachments to index content attachments
823     * @param solrClient The solr client to use
824     * @param progressionTracker The progression of the indexation
825     * @return The indexation result as a Map.
826     * @throws Exception if an error occurs while indexing.
827     */
828    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
829    {
830        Request request = ContextHelper.getRequest(_context);
831        
832        // Retrieve the current workspace.
833        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
834        
835        try
836        {
837            // Force the workspace.
838            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
839            
840            getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName);
841            
842            long start = System.currentTimeMillis();
843            
844            // Delete all contents
845            unindexAllContents(workspaceName, indexAttachments, solrClient);
846
847            String query = ContentQueryHelper.getContentXPathQuery(null);
848            AmetysObjectIterable<Content> contents = _resolver.query(query);
849            
850            IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient, progressionTracker);
851            
852            long end = System.currentTimeMillis();
853            
854            if (!result.hasErrors())
855            {
856                getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
857            }
858            else
859            {
860                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
861            }
862            
863            Map<String, Object> results = new HashMap<>();
864            results.put("successCount", result.getSuccessCount());
865            if (result.hasErrors())
866            {
867                results.put("errorCount", result.getErrorCount());
868            }
869            
870            return results;
871        }
872        catch (Exception e)
873        {
874            String error = String.format("Failed to index all contents in workspace %s", workspaceName);
875            getLogger().error(error, e);
876            throw new IndexingException(error, e);
877        }
878        finally
879        {
880            // Restore context
881            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
882        }
883    }
884    /**
885     * Unindex all content documents.
886     * @param workspaceName The workspace name
887     * @param unindexAttachments also unindex content attachments
888     * @param solrClient The solr client to use
889     * @throws Exception if an error occurs while unindexing.
890     */
891    protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
892    {
893        String collection = _solrClientProvider.getCollectionName(workspaceName);
894        
895        Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT);
896        Query query;
897        if (unindexAttachments)
898        {
899            Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE);
900            Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE);
901            query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents);
902        }
903        else
904        {
905            query = contents;
906        }
907        solrClient.deleteByQuery(collection, query.build());
908    }
909    
910    /**
911     * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit
912     * @param collectionId The id of collection 
913     * @param indexAttachments to index content attachments
914     * @throws Exception if an error occurs while indexing.
915     */
916    public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception
917    {
918        String[] workspaceNames = _repository.getWorkspaces();
919        for (String workspaceName : workspaceNames)
920        {
921            indexSubcontents(collectionId, workspaceName, indexAttachments);
922        }
923    }
924    
925    /**
926     * Index the child contents of a {@link AmetysObjectCollection}
927     * @param collectionId The id of collection 
928     * @param workspaceName the workspace where to index
929     * @param indexAttachments to index content attachments
930     * @throws Exception if an error occurs while unindexing.
931     */
932    public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception
933    {
934        Request request = ContextHelper.getRequest(_context);
935        
936        // Retrieve the current workspace.
937        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
938        
939        try
940        {
941            // Force the workspace.
942            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
943            
944            if (_resolver.hasAmetysObjectForId(collectionId))
945            {
946                SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
947                AmetysObjectCollection collection = _resolver.resolveById(collectionId);
948                AmetysObjectIterable<AmetysObject> children = collection.getChildren();
949                
950                for (AmetysObject child : children)
951                {
952                    if (child instanceof Content)
953                    {
954                        Content content = (Content) child;
955                        
956                        _doIndexContent(content, workspaceName, indexAttachments, solrClient);
957                    }
958                }
959            }
960        }
961        finally
962        {
963            // Restore context
964            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
965        }
966    }
967    
968    /**
969     * Add or update a content into Solr index on all workspaces and commit
970     * @param contentId The id of the content to index
971     * @param indexAttachments to index content attachments
972     * @throws Exception if an error occurs while indexing.
973     */
974    public void indexContent(String contentId, boolean indexAttachments) throws Exception
975    {
976        String[] workspaceNames = _repository.getWorkspaces();
977        for (String workspaceName : workspaceNames)
978        {
979            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
980            indexContent(contentId, workspaceName, indexAttachments, solrClient);
981        }
982    }
983    
984    /**
985     * Add or update a content into Solr index
986     * @param contentId The id of the content to index
987     * @param workspaceName the workspace where to index
988     * @param indexAttachments to index content attachments
989     * @throws Exception if an error occurs while indexing.
990     */
991    public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception
992    {
993        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
994        indexContent(contentId, workspaceName, indexAttachments, solrClient);
995    }
996    
997    /**
998     * Add or update a content into Solr index
999     * @param contentId The id of the content to index
1000     * @param workspaceName the workspace where to index
1001     * @param indexAttachments to index content attachments
1002     * @param solrClient The solr client to use
1003     * @throws Exception if an error occurs while indexing.
1004     */
1005    public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1006    {
1007        Request request = ContextHelper.getRequest(_context);
1008        
1009        // Retrieve the current workspace.
1010        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1011        
1012        try
1013        {
1014            // Force the workspace.
1015            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1016            
1017            if (_resolver.hasAmetysObjectForId(contentId))
1018            {
1019                Content content = _resolver.resolveById(contentId);
1020                _doIndexContent(content, workspaceName, indexAttachments, solrClient);
1021            }
1022        }
1023        finally
1024        {
1025            // Restore context
1026            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1027        }
1028    }
1029    
1030    private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException
1031    {
1032        try
1033        {
1034            long time_0 = System.currentTimeMillis();
1035            
1036            getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName);
1037            
1038            deleteRepeaterDocs(content.getId(), workspaceName, solrClient);
1039            doIndexContent(content, workspaceName, solrClient);
1040            doIndexContentWorkflow(content, workspaceName, solrClient);
1041            if (indexAttachments)
1042            {
1043                indexContentAttachments(content.getRootAttachments(), content, solrClient);
1044            }
1045            
1046            getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0);
1047        }
1048        catch (Exception e)
1049        {
1050            String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName);
1051            getLogger().error(error, e);
1052            throw new IndexingException(error, e);
1053        }
1054    }
1055    
1056    /**
1057     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1058     * @param contents the collection of contents to index.
1059     * @return the indexation result.
1060     * @throws Exception if an error occurs while indexing.
1061     */
1062    public IndexationResult indexContents(Iterable<Content> contents) throws Exception
1063    {
1064        return indexContents(contents, ProgressionTrackerFactory.createContainerProgressionTracker("Index contents", getLogger()));
1065    }
1066    
1067    
1068    /**
1069     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1070     * @param contents the collection of contents to index.
1071     * @param progressionTracker The progression of the indexation 
1072     * @return the indexation result.
1073     * @throws Exception if an error occurs while indexing.
1074     */
1075    public IndexationResult indexContents(Iterable<Content> contents, ContainerProgressionTracker progressionTracker) throws Exception
1076    {
1077        
1078        IndexationResult result = new IndexationResult();
1079        String[] workspaceNames = _repository.getWorkspaces();
1080        
1081        for (String workspaceName : workspaceNames)
1082        {
1083            progressionTracker.addSimpleStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_WORKSPACE_STEP_LABEL", List.of(workspaceName)));
1084        }
1085        
1086        for (String workspaceName : workspaceNames)
1087        {
1088            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1089            IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient, progressionTracker.getStep(workspaceName));
1090            
1091            result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount());
1092        }
1093        
1094        return result;
1095    }
1096    
1097    /**
1098     * Send a collection of contents for indexation in the solr server.
1099     * @param contents the collection of contents to index.
1100     * @param workspaceName the workspace where to index
1101     * @param indexAttachments to index content attachments
1102     * @param solrClient The solr client to use
1103     * @return the indexation result.
1104     * @throws Exception if an error occurs while indexing.
1105     */
1106    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1107    {
1108        return indexContents(contents, workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index contents for workspace " + workspaceName, getLogger()));
1109    }
1110    
1111    /**
1112     * Send a collection of contents for indexation in the solr server.
1113     * @param contents the collection of contents to index.
1114     * @param workspaceName the workspace where to index
1115     * @param indexAttachments to index content attachments
1116     * @param solrClient The solr client to use
1117     * @param progressionTracker The progression of the indexation
1118     * @return the indexation result.
1119     * @throws Exception if an error occurs while indexing.
1120     */
1121    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1122    {
1123        Request request = ContextHelper.getRequest(_context);
1124        
1125        // Retrieve the current workspace.
1126        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1127        
1128        try
1129        {
1130            // Force the workspace.
1131            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1132            
1133            getLogger().info("Starting indexation of several contents for workspace {}", workspaceName);
1134            
1135            long start = System.currentTimeMillis();
1136            
1137            List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false)
1138                .map(Content::getId)
1139                .map(this::_resolveSilently)
1140                .filter(Objects::nonNull)
1141                .collect(Collectors.toList());
1142            
1143            IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient, progressionTracker);
1144            
1145            long end = System.currentTimeMillis();
1146            
1147            if (!result.hasErrors())
1148            {
1149                getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1150            }
1151            else
1152            {
1153                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1154            }
1155            
1156            return result;
1157        }
1158        catch (Exception e)
1159        {
1160            String error = String.format("Failed to index several contents in workspace %s", workspaceName);
1161            getLogger().error(error, e);
1162            throw new IndexingException(error, e);
1163        }
1164        finally
1165        {
1166            // Restore context
1167            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1168        }
1169    }
1170    
1171    private Content _resolveSilently(String contentId)
1172    {
1173        try
1174        {
1175            return _resolver.resolveById(contentId);
1176        }
1177        catch (UnknownAmetysObjectException e) 
1178        {
1179            return null;
1180        }
1181    }
1182    
1183    /**
1184     * Send some contents for indexation in the solr server.
1185     * @param contents the contents to index.
1186     * @param workspaceName The workspace name
1187     * @param indexAttachments to index content attachments
1188     * @param solrClient The solr client to use
1189     * @param progressionTracker The progression of the indexation
1190     * @return the indexation result.
1191     * @throws Exception if an error occurs committing the results.
1192     */
1193    protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1194    {
1195        int successCount = 0;
1196        int errorCount = 0;
1197        int numberOfContents = IterableUtils.size(contents);
1198        
1199        progressionTracker.setSize(numberOfContents);
1200        
1201        for (Content content : contents)
1202        {
1203            try
1204            {
1205                doIndexContent(content, workspaceName, solrClient);
1206                doIndexContentWorkflow(content, workspaceName, solrClient);
1207                if (indexAttachments)
1208                {
1209                    indexContentAttachments(content.getRootAttachments(), content, solrClient);
1210                }
1211                successCount++;
1212            }
1213            catch (Exception e)
1214            {
1215                getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e);
1216                errorCount++;
1217            }
1218            
1219            progressionTracker.increment();
1220        }
1221        
1222        return new IndexationResult(successCount, errorCount);
1223    }
1224    
1225    /**
1226     * Update the value of a specific system property in a content document.
1227     * @param content The content to update.
1228     * @param propertyId The system property ID.
1229     * @param workspaceName The workspace name
1230     * @throws Exception if an error occurs while indexing.
1231     */
1232    public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception
1233    {
1234        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1235        updateSystemProperty(content, propertyId, workspaceName, solrClient);
1236    }
1237    
1238    /**
1239     * Update the value of a specific system property in a content document.
1240     * @param content The content to update.
1241     * @param propertyId The system property ID.
1242     * @param workspaceName The workspace name
1243     * @param solrClient The solr client to use
1244     * @throws Exception if an error occurs while indexing.
1245     */
1246    public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1247    {
1248        getLogger().debug("Updating the system property '{}' for content {} into Solr.", propertyId, content);
1249        
1250        SolrInputDocument document = new SolrInputDocument();
1251        boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document);
1252        
1253        if (!hasUpdate)
1254        {
1255            getLogger().debug("Did not index '{}' system property for content {} in Solr because no update to apply.", propertyId, content);
1256            return;
1257        }
1258        
1259        int status = _pushSolrDocument(document, workspaceName, solrClient);
1260        if (status != 0)
1261        {
1262            throw new IOException("Indexing of system property '" + propertyId + "': got status code '" + status + "'.");
1263        }
1264        
1265        getLogger().debug("Succesfully indexed '{}' system property for content {} in Solr.", propertyId, content);
1266    }
1267    
1268    /**
1269     * Update the value of a specific property in a content document.
1270     * @param content The content to update.
1271     * @param propertyId The system property ID.
1272     * @param workspaceName The workspace name
1273     * @throws Exception if an error occurs while indexing.
1274     */
1275    public void updateProperty(Content content, String propertyId, String workspaceName) throws Exception
1276    {
1277        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1278        updateProperty(content, propertyId, workspaceName, solrClient);
1279    }
1280    
1281    /**
1282     * Update the value of a specific property in a content document.
1283     * @param content The content to update.
1284     * @param propertyId The system property ID.
1285     * @param workspaceName The workspace name
1286     * @param solrClient The solr client to use
1287     * @throws Exception if an error occurs while indexing.
1288     */
1289    public void updateProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1290    {
1291        getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content);
1292        
1293        SolrInputDocument document = new SolrInputDocument();
1294        boolean hasUpdate = _solrContentIndexer.indexPartialProperty(content, propertyId, document);
1295        
1296        if (!hasUpdate)
1297        {
1298            getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content);
1299            return;
1300        }
1301        
1302        int status = _pushSolrDocument(document, workspaceName, solrClient);
1303        if (status != 0)
1304        {
1305            throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'.");
1306        }
1307        
1308        getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content);
1309    }
1310    
1311    private int _pushSolrDocument(SolrInputDocument document, String workspaceName, SolrClient solrClient) throws Exception
1312    {
1313        String collection = _solrClientProvider.getCollectionName(workspaceName);
1314        UpdateResponse solrResponse = solrClient.add(collection, document);
1315        return solrResponse.getStatus();
1316    }
1317    
1318    /**
1319     * Remove a content from Solr index for all workspaces and commit
1320     * @param contentId The id of content to unindex
1321     * @param unindexAttachments also unindex content attachments
1322     * @throws Exception if an error occurs while indexing.
1323     */
1324    public void unindexContent(String contentId, boolean unindexAttachments) throws Exception
1325    {
1326        String[] workspaceNames = _repository.getWorkspaces();
1327        for (String workspaceName : workspaceNames)
1328        {
1329            unindexContent(contentId, workspaceName, unindexAttachments);
1330        }
1331    }
1332    
1333    /**
1334     * Remove a content from Solr index
1335     * @param contentId The id of content to unindex
1336     * @param workspaceName The workspace where to work in 
1337     * @param unindexAttachments also unindex content attachments
1338     * @throws Exception if an error occurs while indexing.
1339     */
1340    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception
1341    {
1342        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1343        unindexContent(contentId, workspaceName, unindexAttachments, solrClient);
1344    }
1345    
1346    /**
1347     * Remove a content from Solr index
1348     * @param contentId The id of content to unindex
1349     * @param workspaceName The workspace where to work in 
1350     * @param unindexAttachments also unindex content attachments
1351     * @param solrClient The solr client to use
1352     * @throws Exception if an error occurs while indexing.
1353     */
1354    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
1355    {
1356        Request request = ContextHelper.getRequest(_context);
1357        
1358        // Retrieve the current workspace.
1359        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1360        
1361        try
1362        {
1363            // Force the workspace.
1364            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1365            
1366            getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName);
1367            
1368            deleteRepeaterDocs(contentId, workspaceName, solrClient);
1369            doUnindexDocument(contentId, workspaceName, solrClient);
1370            if (unindexAttachments)
1371            {
1372                doUnindexContentAttachments(contentId, workspaceName, solrClient);
1373            }
1374            _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1375            
1376            getLogger().debug("Succesfully deleted content {} from Solr.", contentId);
1377        }
1378        catch (Exception e)
1379        {
1380            String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName);
1381            getLogger().error(error, e);
1382            throw new IndexingException(error, e);
1383        }
1384        finally
1385        {
1386            // Restore workspace
1387            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1388        }
1389    }
1390    
1391    /**
1392     * Remove a content from Solr index for all workspaces and commit
1393     * @param contentIds The id of content to unindex
1394     * @throws Exception if an error occurs while indexing.
1395     */
1396    public void unindexContents(Collection<String> contentIds) throws Exception
1397    {
1398        String[] workspaceNames = _repository.getWorkspaces();
1399        for (String workspaceName : workspaceNames)
1400        {
1401            unindexContents(contentIds, workspaceName);
1402        }
1403    }
1404    
1405    /**
1406     * Remove a content from Solr index
1407     * @param contentIds The id of content to unindex
1408     * @param workspaceName The workspace where to work in 
1409     * @throws Exception if an error occurs while indexing.
1410     */
1411    public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception
1412    {
1413        Request request = ContextHelper.getRequest(_context);
1414        
1415        // Retrieve the current workspace.
1416        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1417        
1418        try
1419        {
1420            // Force the workspace.
1421            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1422            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1423            
1424            getLogger().debug("Unindexing several contents from Solr.");
1425            
1426            for (String contentId : contentIds)
1427            {
1428                deleteRepeaterDocs(contentId, workspaceName, solrClient);
1429                doUnindexDocument(contentId, workspaceName, solrClient);
1430                _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1431            }
1432            
1433            getLogger().debug("Succesfully unindexed content from Solr.");
1434        }
1435        catch (Exception e)
1436        {
1437            String error = String.format("Failed to unindex several contents in workspace %s", workspaceName);
1438            getLogger().error(error, e);
1439            throw new IndexingException(error, e);
1440        }
1441        finally
1442        {
1443            // Restore workspace
1444            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1445        }
1446    }
1447    
1448    /**
1449     * Add or update a content into Solr index
1450     * @param content The content to index
1451     * @param workspaceName The workspace where to index
1452     * @param solrClient The solr client to use
1453     * @throws Exception if an error occurs while indexing.
1454     */
1455    protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception
1456    {
1457        long time_0 = System.currentTimeMillis();
1458        
1459        SolrInputDocument document = new SolrInputDocument();
1460        List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document);
1461        
1462        long time_1 = System.currentTimeMillis();
1463        getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0);
1464        
1465        indexAclInitValues(content, document);
1466        
1467        long time_2 = System.currentTimeMillis();
1468        getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1);
1469        
1470        List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments);
1471        documents.add(document);
1472        
1473        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents);
1474        int status = solrResponse.getStatus();
1475        
1476        long time_3 = System.currentTimeMillis();
1477        getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2);
1478        
1479        if (status != 0)
1480        {
1481            throw new IOException("Content indexation: got status code '" + status + "'.");
1482        }
1483    }
1484    
1485    /**
1486     * Indexes read-ACl initial values for object
1487     * @param ametysObject The object
1488     * @param document The Solr document
1489     */
1490    public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document)
1491    {
1492        // Indexation of AmetysObject property
1493        document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true);
1494        
1495        // Indexation of initValues for AllowedUsers
1496        AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject);
1497        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed());
1498        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed());
1499        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString);
1500        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString);
1501        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString);
1502        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString);
1503    }
1504    
1505    private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier)
1506    {
1507        if (values == null)
1508        {
1509            return;
1510        }
1511        
1512        for (T value : values)
1513        {
1514            document.addField(fieldName, stringifier.apply(value));
1515        }
1516    }
1517    
1518    /**
1519     * Index the whole workflow of a content.
1520     * @param content The content.
1521     * @param workspaceName The workspace name
1522     * @param solrClient The solr client to use
1523     * @throws Exception if an error occurs while indexing.
1524     */
1525    protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception
1526    {
1527        if (content instanceof WorkflowAwareContent)
1528        {
1529            _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient);
1530        }
1531    }
1532    
1533    /**
1534     * Index content attachments as new entries in the idnex
1535     * @param collection the collection of attachments
1536     * @param content the content whose attachments will be indexed
1537     * @throws Exception if something goes wrong when indexing the attachments of the content
1538     */
1539    public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception
1540    {
1541        Request request = ContextHelper.getRequest(_context);
1542        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1543        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1544        indexContentAttachments(collection, content, solrClient);
1545    }
1546    
1547    /**
1548     * Index content attachments as new entries in the idnex
1549     * @param collection the collection of attachments
1550     * @param content the content whose attachments will be indexed
1551     * @param solrClient The solr client to use
1552     * @throws Exception if something goes wrong when indexing the attachments of the content
1553     */
1554    public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception
1555    {
1556        if (collection == null)
1557        {
1558            return;
1559        }
1560        
1561        try (AmetysObjectIterable<AmetysObject> children = collection.getChildren())
1562        {
1563            for (AmetysObject object : children)
1564            {
1565                if (object instanceof ResourceCollection)
1566                {
1567                    indexContentAttachments((ResourceCollection) object, content, solrClient);
1568                }
1569                else if (object instanceof Resource)
1570                {
1571                    Resource resource = (Resource) object;
1572                    indexContentAttachment(resource, content, solrClient);
1573                }
1574            }
1575        }
1576    }
1577    
1578    /**
1579     * Index a content attachment
1580     * @param resource the content attachment as a {@link Resource}
1581     * @param content the content whose attachment is going to be indexed
1582     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1583     */
1584    public void indexContentAttachment(Resource resource, Content content) throws Exception
1585    {
1586        Request request = ContextHelper.getRequest(_context);
1587        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1588        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1589        indexContentAttachment(resource, content, solrClient);
1590    }
1591    
1592    /**
1593     * Index a content attachment
1594     * @param resource the content attachment as a {@link Resource}
1595     * @param content the content whose attachment is going to be indexed
1596     * @param solrClient The solr client to use
1597     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1598     */
1599    public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception
1600    {
1601        SolrInputDocument document = new SolrInputDocument();
1602        
1603        // Prepare resource doc
1604        _indexContentAttachment(resource, document, content);
1605        
1606        // Indexation of the document
1607        _indexResourceDocument(resource, document, solrClient);
1608    }
1609    
1610    private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception
1611    {
1612        String language = content.getLanguage();
1613        
1614        _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language);
1615        
1616        // Need the id of the content for unindexing attachment during the unindexing of the content
1617        document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId());
1618    }
1619    
1620    /**
1621     * Index a populated solr input document of type Resource.
1622     * @param resource the resource from which the input document is created
1623     * @param document the input document
1624     * @param solrClient The solr client to use
1625     * @throws SolrServerException if there is an error on the server
1626     * @throws IOException if there is a communication error with the server
1627     */
1628    protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException
1629    {
1630        String resourceId = resource.getId();
1631        
1632        // Retrieve appropriate collection name
1633        Request request = ContextHelper.getRequest(_context);
1634        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1635        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
1636        
1637        // Add document
1638        UpdateResponse solrResponse = solrClient.add(collectionName, document);
1639        int status = solrResponse.getStatus();
1640        
1641        if (status != 0)
1642        {
1643            throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId);
1644        }
1645        
1646        getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId);
1647    }
1648    
1649    /**
1650     * Delete repeater documents of a specified content.
1651     * @param workspaceName The workspace name
1652     * @param contentId the content ID.
1653     * @param solrClient The solr client to use
1654     * @throws Exception if an error occurs while indexing.
1655     */
1656    protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception
1657    {
1658        long time_0 = System.currentTimeMillis();
1659        
1660        // _documentType:repeater AND id:content\://xxx/*
1661        StringBuilder query = new StringBuilder();
1662        query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER)
1663             .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*");
1664        
1665        solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString());
1666        
1667        getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0);
1668    }
1669    
1670    /**
1671     * Index all the resources in a given workspace.
1672     * @param workspaceName The workspace where to index
1673     * @param solrClient The solr client to use
1674     * @param progressionTracker The progression of the indexation
1675     * @return The indexation result as a Map.
1676     * @throws Exception if an error occurs while indexing.
1677     */
1678    public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1679    {
1680        Request request = ContextHelper.getRequest(_context);
1681        
1682        // Retrieve the current workspace.
1683        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1684        
1685        try
1686        {
1687            // Force the workspace.
1688            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1689            
1690            getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName);
1691            
1692            long start = System.currentTimeMillis();
1693            
1694            // Delete all resources
1695            _unindexAllResources(workspaceName, solrClient);
1696            
1697            Map<String, Object> results = new HashMap<>();
1698            
1699            try
1700            {
1701                TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources");
1702                AmetysObjectIterable<Resource> resources = resourceRoot.getChildren();
1703                
1704                IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient, progressionTracker);
1705                
1706                long end = System.currentTimeMillis();
1707                
1708                if (!result.hasErrors())
1709                {
1710                    getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1711                }
1712                else
1713                {
1714                    getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1715                }
1716                
1717                results.put("successCount", result.getSuccessCount());
1718                if (result.hasErrors())
1719                {
1720                    results.put("errorCount", result.getErrorCount());
1721                }
1722            }
1723            catch (UnknownAmetysObjectException e)
1724            {
1725                getLogger().info("There is no root for resources in current workspace.");
1726                progressionTracker.setSize(0);
1727            }
1728            
1729            return results;
1730        }
1731        catch (Exception e)
1732        {
1733            String error = String.format("Failed to index all resources in workspace %s", workspaceName);
1734            getLogger().error(error, e);
1735            throw new IndexingException(error, e);
1736        }
1737        finally
1738        {
1739            // Restore context
1740            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1741        }
1742    }
1743    
1744    /**
1745     * Send a collection of contents for indexation in the solr server.
1746     * @param resources the collection of contents to index.
1747     * @param documentType The document type of the resource
1748     * @param workspaceName The workspace where to index
1749     * @return the indexation result.
1750     * @throws Exception if an error occurs while indexing.
1751     */
1752    public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception
1753    {
1754        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1755        return indexResources(resources, documentType, null, workspaceName, solrClient);
1756    }
1757    
1758    /**
1759     * Send a collection of contents for indexation in the solr server.
1760     * @param resources the collection of contents to index.
1761     * @param documentType The document type of the resource
1762     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1763     * @param workspaceName The workspace where to index
1764     * @param solrClient The solr client to use
1765     * @return the indexation result.
1766     * @throws Exception if an error occurs while indexing.
1767     */
1768    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception
1769    {
1770        return indexResources(resources, documentType, resourceRoot, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resources", getLogger()));
1771    }
1772    
1773    /**
1774     * Send a collection of contents for indexation in the solr server.
1775     * @param resources the collection of contents to index.
1776     * @param documentType The document type of the resource
1777     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1778     * @param workspaceName The workspace where to index
1779     * @param solrClient The solr client to use
1780     * @param progressionTracker The progression of the indexation
1781     * @return the indexation result.
1782     * @throws Exception if an error occurs while indexing.
1783     */
1784    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1785    {
1786        Request request = ContextHelper.getRequest(_context);
1787        
1788        // Retrieve the current workspace.
1789        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1790        
1791        try
1792        {
1793            // Force the workspace.
1794            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1795            
1796            getLogger().info("Starting indexation of several resources for workspace {}", workspaceName);
1797            
1798            long start = System.currentTimeMillis();
1799            
1800            IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient, progressionTracker);
1801            
1802            long end = System.currentTimeMillis();
1803            
1804            if (!result.hasErrors())
1805            {
1806                getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1807            }
1808            else
1809            {
1810                getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1811            }
1812            
1813            return result;
1814        }
1815        catch (Exception e)
1816        {
1817            String error = String.format("Failed to index several resources in workspace %s", workspaceName);
1818            getLogger().error(error, e);
1819            throw new IndexingException(error, e);
1820        }
1821        finally
1822        {
1823            // Restore context
1824            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1825        }
1826    }
1827    
1828    /**
1829     * Send some resources for indexation in the solr server.
1830     * @param resources the resources to index.
1831     * @param documentType The document type of the resource
1832     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 
1833     * @param workspaceName The workspace where to index
1834     * @param solrClient The solr client to use
1835     * @param progressionTracker The progression of the indexation
1836     * @return the indexation result.
1837     * @throws Exception if an error occurs committing the results.
1838     */
1839    protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1840    {
1841        int successCount = 0;
1842        int errorCount = 0;
1843        int numberOfResources = IterableUtils.size(resources);
1844
1845        progressionTracker.setSize(numberOfResources);
1846
1847        for (AmetysObject resource : resources)
1848        {
1849            try
1850            {
1851                doIndexExplorerItem(resource, documentType, resourceRoot, solrClient);
1852                successCount++;
1853            }
1854            catch (Exception e)
1855            {
1856                getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e);
1857                errorCount++;
1858            }
1859            
1860            progressionTracker.increment();
1861        }
1862        
1863        return new IndexationResult(successCount, errorCount);
1864    }
1865    
1866    /**
1867     * Add or update a resource into Solr index
1868     * @param resource The resource to index
1869     * @param documentType The document type of the resource
1870     * @param workspaceName The workspace where to index
1871     * @throws Exception if an error occurs while indexing.
1872     */
1873    public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception
1874    {
1875        Request request = ContextHelper.getRequest(_context);
1876        
1877        // Retrieve the current workspace.
1878        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1879        
1880        try
1881        {
1882            // Force the workspace.
1883            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1884            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1885            
1886            getLogger().debug("Indexing resource {} into Solr.", resource.getId());
1887            
1888            doIndexResource(resource, documentType, null, solrClient);
1889            
1890            getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId());
1891        }
1892        catch (Exception e)
1893        {
1894            String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName);
1895            getLogger().error(error, e);
1896            throw new IndexingException(error, e);
1897        }
1898        finally
1899        {
1900            // Restore context
1901            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1902        }
1903    }
1904    
1905    private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception
1906    {
1907        getLogger().debug("Unindexing all resources from Solr.");
1908        
1909        String collection = _solrClientProvider.getCollectionName(workspaceName);
1910        solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE);
1911        
1912        getLogger().debug("Succesfully deleted all resource documents from Solr.");
1913    }
1914    
1915    /**
1916     * Delete all resource documents at a given path for all workspaces and commit
1917     * @param rootId The resource root ID, must not be null.
1918     * @param path The resource path relative to the given root, must start with a slash.
1919     * @throws Exception If an error occurs while unindexing.
1920     */
1921    public void unindexResourcesByPath(String rootId, String path) throws Exception
1922    {
1923        String[] workspaceNames = _repository.getWorkspaces();
1924        for (String workspaceName : workspaceNames)
1925        {
1926            unindexResourcesByPath(rootId, path, workspaceName);
1927        }
1928    }
1929    
1930    /**
1931     * Delete all resource documents at a given path.
1932     * @param rootId The resource root ID, must not be null.
1933     * @param path The resource path relative to the given root, must start with a slash.
1934     * @param workspaceName The workspace where to work in 
1935     * @throws Exception If an error occurs while unindexing.
1936     */
1937    public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception
1938    {
1939        Request request = ContextHelper.getRequest(_context);
1940        
1941        // Retrieve the current workspace.
1942        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1943        
1944        try
1945        {
1946            // Force the workspace.
1947            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1948            
1949            getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId);
1950            
1951            Query query = new ResourceLocationQuery(rootId, path);
1952            
1953            String collection = _solrClientProvider.getCollectionName(workspaceName);
1954            _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build());
1955            
1956            getLogger().debug("Succesfully deleted resource document from Solr.");
1957        }
1958        catch (Exception e)
1959        {
1960            String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName);
1961            getLogger().error(error, e);
1962            throw new IndexingException(error, e);
1963        }
1964        finally
1965        {
1966            // Restore workspace
1967            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1968        }
1969    }
1970    
1971    /**
1972     * Remove a resource from Solr index for all workspaces and commit
1973     * @param resourceId The id of resource to unindex
1974     * @throws Exception if an error occurs while indexing.
1975     */
1976    public void unindexResource(String resourceId) throws Exception
1977    {
1978        String[] workspaceNames = _repository.getWorkspaces();
1979        for (String workspaceName : workspaceNames)
1980        {
1981            unindexResource(resourceId, workspaceName);
1982        }
1983    }
1984    
1985    /**
1986     * Remove a resource from the Solr index.
1987     * @param resourceId The id of resource to unindex
1988     * @param workspaceName The workspace where to work in 
1989     * @throws Exception if an error occurs while unindexing.
1990     */
1991    public void unindexResource(String resourceId, String workspaceName) throws Exception
1992    {
1993        Request request = ContextHelper.getRequest(_context);
1994        
1995        // Retrieve the current workspace.
1996        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1997        
1998        try
1999        {
2000            // Force the workspace.
2001            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
2002            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
2003            
2004            getLogger().debug("Unindexing resource {} from Solr.", resourceId);
2005            
2006            doUnindexDocument(resourceId, workspaceName, solrClient);
2007            
2008            getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId);
2009        }
2010        catch (Exception e)
2011        {
2012            String error = String.format("Failed to unindex resource  %s in workspace %s", resourceId, workspaceName);
2013            getLogger().error(error, e);
2014            throw new IndexingException(error, e);
2015        }
2016        finally
2017        {
2018            // Restore workspace
2019            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
2020        }
2021    }
2022    
2023    /**
2024     * Add or update a resource into Solr index
2025     * @param node The resource to index
2026     * @param documentType The document type of the resource
2027     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 
2028     * @param solrClient The solr client to use
2029     * @throws Exception if an error occurs while indexing.
2030     */
2031    protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2032    {
2033        if (node instanceof ResourceCollection)
2034        {
2035            try (AmetysObjectIterable<AmetysObject> children = ((ResourceCollection) node).getChildren())
2036            {
2037                for (AmetysObject child : children)
2038                {
2039                    doIndexExplorerItem(child, documentType, resourceRoot, solrClient);
2040                }
2041            }
2042        }
2043        else if (node instanceof Resource)
2044        {
2045            doIndexResource((Resource) node, documentType, resourceRoot, solrClient);
2046        }
2047    }
2048    
2049    /**
2050     * Add or update a resource into Solr index
2051     * @param resource The resource to index
2052     * @param documentType The document type of the resource
2053     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
2054     * @param solrClient The solr client to use 
2055     * @throws Exception if an error occurs while indexing.
2056     */
2057    protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2058    {
2059        SolrInputDocument document = new SolrInputDocument();
2060        
2061        _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot);
2062        
2063        String workspaceName = _workspaceSelector.getWorkspace();
2064        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document);
2065        int status = solrResponse.getStatus();
2066        
2067        if (status != 0)
2068        {
2069            throw new IOException("Resource indexation: got status code '" + status + "'.");
2070        }
2071    }
2072    
2073    /**
2074     * Process a Solr commit operation in all workspaces.
2075     * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient}
2076     * @throws SolrServerException if there is an error on the server
2077     * @throws IOException if there is a communication error with the server
2078     */
2079    public void commit() throws SolrServerException, IOException
2080    {
2081        String[] workspaceNames;
2082        try
2083        {
2084            workspaceNames = _repository.getWorkspaces();
2085            for (String workspaceName : workspaceNames)
2086            {
2087                SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName);
2088                commit(workspaceName, solrClient);
2089            }
2090        }
2091        catch (RepositoryException e)
2092        {
2093            throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e);
2094        }
2095    }
2096    
2097    /**
2098     * Process a Solr commit operation in given workspace.
2099     * @param workspaceName The workspace's name
2100     * @param solrClient The solr client to use
2101     * @throws SolrServerException if there is an error on the server
2102     * @throws IOException if there is a communication error with the server
2103     */
2104    public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2105    {
2106        long time_0 = System.currentTimeMillis();
2107        
2108        // Commit
2109        UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName));
2110        int status = solrResponse.getStatus();
2111        
2112        if (status != 0)
2113        {
2114            throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2115        }
2116        
2117        getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0);
2118    }
2119    
2120    /**
2121     * Launch a solr index optimization.
2122     * @param workspaceName The workspace's name
2123     * @param solrClient The solr client to use
2124     * @throws SolrServerException if there is an error on the server
2125     * @throws IOException if there is a communication error with the server
2126     */
2127    public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2128    {
2129        UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName));
2130        int status = solrResponse.getStatus();
2131        
2132        if (status != 0)
2133        {
2134            throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2135        }
2136        
2137        getLogger().debug("Successful Solr optimize operation during an Ametys indexing process.");
2138    }
2139    
2140    /**
2141     * Delete all documents from the solr index.
2142     * @param workspaceName The workspace name
2143     * @param solrClient The solr client to use
2144     * @throws Exception if an error occurs while unindexing.
2145     */
2146    public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception
2147    {
2148        getLogger().debug("Deleting all documents from Solr.");
2149        
2150        String collection = _solrClientProvider.getCollectionName(workspaceName);
2151        solrClient.deleteByQuery(collection, "*:*");
2152        
2153        getLogger().debug("Successfully deleted all documents from Solr.");
2154    }
2155    
2156    /**
2157     * Delete a document from the Solr server.
2158     * @param id The id of the document to delete from Solr
2159     * @param workspaceName The workspace name
2160     * @param solrClient The solr client to use
2161     * @throws Exception if an error occurs while indexing.
2162     */
2163    protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception
2164    {
2165        UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id);
2166        int status = solrResponse.getStatus();
2167        
2168        if (status != 0)
2169        {
2170            throw new IOException("Deletion of document " + id + ": got status code '" + status + "'.");
2171        }
2172    }
2173    
2174    /**
2175     * Delete content attachments documents of a given content from the Solr server.
2176     * @param contentId The id of the content
2177     * @param workspaceName The workspace name
2178     * @param solrClient The solr client to use
2179     * @throws Exception if an error occurs while indexing.
2180     */
2181    protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception
2182    {
2183        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
2184        
2185        Query query = new ContentAttachmentQuery(contentId);
2186        UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build());
2187        int status = solrResponse.getStatus();
2188        
2189        if (status != 0)
2190        {
2191            throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'.");
2192        }
2193    }
2194    
2195//    /**
2196//     * Get the collection to use.
2197//     * @return The name of the collection to index into.
2198//     */
2199//    protected String getCollection()
2200//    {
2201//        return _solrCorePrefix + _workspaceSelector.getWorkspace();
2202//    }
2203    
2204    class IndexationResult
2205    {
2206        protected int _successCount;
2207        
2208        protected int _errorCount;
2209        
2210        /**
2211         * Constructor
2212         */
2213        public IndexationResult()
2214        {
2215            this(0, 0);
2216        }
2217        
2218        /**
2219         * Constructor
2220         * @param successCount The success count.
2221         * @param errorCount The error count.
2222         */
2223        public IndexationResult(int successCount, int errorCount)
2224        {
2225            this._successCount = successCount;
2226            this._errorCount = errorCount;
2227        }
2228        
2229        /**
2230         * Get the successCount.
2231         * @return the successCount
2232         */
2233        public int getSuccessCount()
2234        {
2235            return _successCount;
2236        }
2237        
2238        /**
2239         * Set the successCount.
2240         * @param successCount the successCount to set
2241         */
2242        public void setSuccessCount(int successCount)
2243        {
2244            this._successCount = successCount;
2245        }
2246        
2247        /**
2248         * Test if the indexation had errors.
2249         * @return true if the indexation had errors, false otherwise.
2250         */
2251        public boolean hasErrors()
2252        {
2253            return _errorCount > 0;
2254        }
2255        
2256        /**
2257         * Get the errorCount.
2258         * @return the errorCount
2259         */
2260        public int getErrorCount()
2261        {
2262            return _errorCount;
2263        }
2264        
2265        /**
2266         * Set the errorCount.
2267         * @param errorCount the errorCount to set
2268         */
2269        public void setErrorCount(int errorCount)
2270        {
2271            this._errorCount = errorCount;
2272        }
2273    }
2274    
2275    private static class SchemaRequestComparator implements Comparator<SchemaRequest.Update>
2276    {
2277        public int compare(SchemaRequest.Update u1, SchemaRequest.Update u2)
2278        {
2279            return Integer.compare(_getOrder(u1), _getOrder(u2));
2280        }
2281        
2282        private int _getOrder(SchemaRequest.Update update)
2283        {
2284            // Field types are needed first to define fields
2285            if (update instanceof SchemaRequest.AddFieldType)
2286            {
2287                return 1;
2288            }
2289            
2290            // Fields or dynamic fields can be defined at the same time
2291            if (update instanceof SchemaRequest.AddField || update instanceof SchemaRequest.AddDynamicField)
2292            {
2293                return 2;
2294            }
2295            
2296            // Copy fields can be based on fields or dynamic fields
2297            if (update instanceof SchemaRequest.AddCopyField)
2298            {
2299                return 3;
2300            }
2301            
2302            return 0;
2303        }
2304    }
2305}