001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.cms.content.indexing.solr;
017
018import java.io.IOException;
019import java.nio.ByteBuffer;
020import java.nio.CharBuffer;
021import java.nio.charset.CharsetDecoder;
022import java.nio.charset.CodingErrorAction;
023import java.nio.charset.StandardCharsets;
024import java.text.SimpleDateFormat;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collection;
028import java.util.Comparator;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.List;
032import java.util.Map;
033import java.util.Objects;
034import java.util.Set;
035import java.util.function.Function;
036import java.util.stream.Collectors;
037import java.util.stream.StreamSupport;
038
039import javax.jcr.RepositoryException;
040
041import org.apache.avalon.framework.activity.Initializable;
042import org.apache.avalon.framework.component.Component;
043import org.apache.avalon.framework.context.Context;
044import org.apache.avalon.framework.context.ContextException;
045import org.apache.avalon.framework.context.Contextualizable;
046import org.apache.avalon.framework.service.ServiceException;
047import org.apache.avalon.framework.service.ServiceManager;
048import org.apache.avalon.framework.service.Serviceable;
049import org.apache.cocoon.components.ContextHelper;
050import org.apache.cocoon.environment.Request;
051import org.apache.commons.lang3.ObjectUtils;
052import org.apache.commons.lang3.StringUtils;
053import org.apache.solr.client.solrj.SolrClient;
054import org.apache.solr.client.solrj.SolrQuery;
055import org.apache.solr.client.solrj.SolrResponse;
056import org.apache.solr.client.solrj.SolrServerException;
057import org.apache.solr.client.solrj.request.CoreAdminRequest;
058import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
059import org.apache.solr.client.solrj.request.schema.SchemaRequest;
060import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update;
061import org.apache.solr.client.solrj.response.CoreAdminResponse;
062import org.apache.solr.client.solrj.response.QueryResponse;
063import org.apache.solr.client.solrj.response.SolrResponseBase;
064import org.apache.solr.client.solrj.response.UpdateResponse;
065import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
066import org.apache.solr.client.solrj.response.schema.SchemaResponse;
067import org.apache.solr.client.solrj.util.ClientUtils;
068import org.apache.solr.common.SolrInputDocument;
069import org.apache.solr.common.util.NamedList;
070import org.slf4j.Logger;
071
072import org.ametys.cms.indexing.IndexingException;
073import org.ametys.cms.indexing.solr.ReloadAclCacheRequest;
074import org.ametys.cms.indexing.solr.UpdateAclCacheRequest;
075import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest;
076import org.ametys.cms.repository.Content;
077import org.ametys.cms.repository.ContentQueryHelper;
078import org.ametys.cms.repository.WorkflowAwareContent;
079import org.ametys.cms.rights.solrchecking.ReadAccessHelper;
080import org.ametys.cms.search.query.ContentAttachmentQuery;
081import org.ametys.cms.search.query.DocumentTypeQuery;
082import org.ametys.cms.search.query.OrQuery;
083import org.ametys.cms.search.query.Query;
084import org.ametys.cms.search.query.ResourceLocationQuery;
085import org.ametys.cms.search.solr.NoAutoCommitUpdateClient;
086import org.ametys.cms.search.solr.SolrClientProvider;
087import org.ametys.cms.search.solr.schema.CopyFieldDefinition;
088import org.ametys.cms.search.solr.schema.FieldDefinition;
089import org.ametys.cms.search.solr.schema.SchemaDefinition;
090import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider;
091import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint;
092import org.ametys.cms.search.solr.schema.SchemaFields;
093import org.ametys.cms.search.solr.schema.SchemaHelper;
094import org.ametys.core.group.GroupIdentity;
095import org.ametys.core.right.AllowedUsers;
096import org.ametys.core.user.UserIdentity;
097import org.ametys.core.util.DateUtils;
098import org.ametys.plugins.explorer.resources.Resource;
099import org.ametys.plugins.explorer.resources.ResourceCollection;
100import org.ametys.plugins.repository.AmetysObject;
101import org.ametys.plugins.repository.AmetysObjectIterable;
102import org.ametys.plugins.repository.AmetysObjectResolver;
103import org.ametys.plugins.repository.RepositoryConstants;
104import org.ametys.plugins.repository.TraversableAmetysObject;
105import org.ametys.plugins.repository.UnknownAmetysObjectException;
106import org.ametys.plugins.repository.collection.AmetysObjectCollection;
107import org.ametys.plugins.repository.provider.AbstractRepository;
108import org.ametys.plugins.repository.provider.JackrabbitRepository;
109import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector;
110import org.ametys.plugins.repository.provider.WorkspaceSelector;
111import org.ametys.runtime.config.Config;
112import org.ametys.runtime.plugin.component.AbstractLogEnabled;
113
114/**
115 * Solr indexer.
116 */
117public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable
118{
119    /** The component role. */
120    public static final String ROLE = SolrIndexer.class.getName();
121    
122    private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>();
123    
124    private static final String _CONFIGSET_NAME_PREFIX = "configset-";
125    
126    private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_");
127    private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general");
128    
129    private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766;
130    
131    /** The ametys object resolver. */
132    protected AmetysObjectResolver _resolver;
133    /** The schema definition provider extension point. */
134    protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP;
135    /** The schema helper. */
136    protected SchemaHelper _schemaHelper;
137    /** Solr Ametys contents indexer */
138    protected SolrContentIndexer _solrContentIndexer;
139    /** Solr workflow indexer. */
140    protected SolrWorkflowIndexer _solrWorkflowIndexer;
141    /** Solr resource indexer. */
142    protected SolrResourceIndexer _solrResourceIndexer;
143    
144    /** The Solr client provider */
145    protected SolrClientProvider _solrClientProvider;
146    
147    /** The solr core prefix. */
148    protected String _solrCorePrefix;
149    /** The Ametys internal URL used by Solr to query Ametys */
150    protected String _ametysInternalUrl;
151    
152    /** The workspace selector. */
153    protected WorkspaceSelector _workspaceSelector;
154    /** The JCR repository */
155    protected JackrabbitRepository _repository;
156    
157    /** The helper for read access */
158    protected ReadAccessHelper _readAccessHelper;
159    
160    /** The avalon context */
161    protected Context _context;
162
163    /**
164     * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone
165     * @return The date format for indexing dates
166     * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead
167     */
168    @Deprecated
169    public static SimpleDateFormat dateFormat()
170    {
171        if (__DATE_FORMAT.get() == null)
172        {
173            __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
174        }
175        return __DATE_FORMAT.get();
176    }
177    
178    /**
179     * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr.
180     * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept.
181     * @param value The string value to index
182     * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log.
183     * @param documentId The id of the document being indexed. Can be null if you do not want to log.
184     * @param fieldName The name of the field being indexed. Can be null if you do not want to log.
185     * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes)
186     */
187    public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName)
188    {
189        if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT)
190        {
191            // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length())
192            // As a result, value.getBytes().length <= limit
193            return value;
194        }
195        
196        // There is a doubt, the string may need to be truncated (or not)
197        byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
198        int bytesLength = valueBytes.length;
199        if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT)
200        {
201            return value;
202        }
203        
204        if (ObjectUtils.allNotNull(logger, documentId, fieldName))
205        {
206            logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT);
207        }
208        
209        // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974)
210        CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT);
211        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
212                                                       .onMalformedInput(CodingErrorAction.IGNORE);
213        decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true);
214        decoder.flush(charBuffer);
215        return new String(charBuffer.array(), 0, charBuffer.position());
216    }
217    
218    @Override
219    public void service(ServiceManager serviceManager) throws ServiceException
220    {
221        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
222        _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE);
223        _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE);
224        _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE);
225        _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE);
226        _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE);
227        _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE);
228        _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE);
229        _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE);
230        _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE);
231    }
232    
233    @Override
234    public void initialize() throws Exception
235    {
236        Config config = Config.getInstance();
237        _solrCorePrefix = config.getValue("cms.solr.core.prefix");
238        
239        _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url");
240        if (StringUtils.isBlank(_ametysInternalUrl))
241        {
242            // fallback to CMS URL as internal URL is an optional parameter
243            _ametysInternalUrl = config.getValue("cms.url");
244        }
245    }
246    
247    @Override
248    public void contextualize(Context context) throws ContextException
249    {
250        _context = context;
251    }
252    
253    /**
254     * Gets the 'autocommit' Solr client
255     * @param workspaceName The name of the workspace
256     * @return the Solr client
257     */
258    protected SolrClient _getAutoCommitSolrClient(String workspaceName)
259    {
260        return _solrClientProvider.getUpdateClient(workspaceName, true);
261    }
262    
263    /**
264     * Gets the 'no autocommit' Solr client
265     * @param workspaceName The name of the workspace
266     * @return the Solr client
267     */
268    protected SolrClient _getNoAutoCommitSolrClient(String workspaceName)
269    {
270        return _solrClientProvider.getUpdateClient(workspaceName, false);
271    }
272    
273    // for admin operations
274    private SolrClient _defaultSolrClient()
275    {
276        return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE);
277    }
278    
279    /**
280     * Get the names of the Solr cores.
281     * @return The names of the Solr cores.
282     * @throws IOException If an I/O error occurs.
283     * @throws SolrServerException If a Solr error occurs.
284     */
285    @SuppressWarnings("unchecked")
286    public Set<String> getCoreNames() throws IOException, SolrServerException
287    {
288        Set<String> coreNames = new HashSet<>();
289        
290        getLogger().debug("Getting core list.");
291        
292        SolrQuery query = new SolrQuery();
293        query.setRequestHandler("/admin/cores");
294        query.setParam("action", "STATUS");
295        
296        QueryResponse response = _defaultSolrClient().query(query);
297        
298        NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status");
299        for (Map.Entry<String, NamedList<?>> core : status)
300        {
301            String fullName = (String) core.getValue().get("name");
302            if (fullName.startsWith(_solrCorePrefix))
303            {
304                coreNames.add(fullName.substring(_solrCorePrefix.length()));
305            }
306        }
307        
308        return coreNames;
309    }
310    
311    /**
312     * Get the names of the Solr cores.
313     * @return The names of the Solr cores.
314     * @throws IOException If an I/O error occurs.
315     * @throws SolrServerException If a Solr error occurs.
316     */
317    protected Set<String> getRealCoreNames() throws IOException, SolrServerException
318    {
319        Set<String> coreNames = new HashSet<>();
320        
321        getLogger().debug("Getting core list.");
322        
323        CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient());
324        
325        NamedList<NamedList<Object>> status = response.getCoreStatus();
326        for (Map.Entry<String, NamedList<Object>> core : status)
327        {
328            String fullName = (String) core.getValue().get("name");
329            if (fullName.startsWith(_solrCorePrefix))
330            {
331                coreNames.add(fullName.substring(_solrCorePrefix.length()));
332            }
333        }
334        
335        return coreNames;
336    }
337    
338    /**
339     * Create a Solr core.
340     * @param name The name of the core to create.
341     * @throws IOException If an I/O error occurs.
342     * @throws SolrServerException If a Solr error occurs.
343     */
344    public void createCore(String name) throws IOException, SolrServerException
345    {
346        String fullName = _solrCorePrefix + name;
347        String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix);
348        _createConfigset(configsetName);
349        
350        getLogger().info("Creating core '{}' (full name: '{}').", name, fullName);
351        
352        SolrQuery query = new SolrQuery();
353        query.setRequestHandler("/admin/cores");
354        query.setParam("action", "CREATE");
355        query.setParam("name", fullName);
356        query.setParam("configSet", configsetName);
357        query.setParam("property.ametys.url", _ametysInternalUrl);
358        
359        QueryResponse response = _defaultSolrClient().query(query);
360        NamedList<?> results = response.getResponse();
361        
362        NamedList<?> error = (NamedList<?>) results.get("error");
363        if (error != null)
364        {
365            throw new IOException("Error creating the core: " + error.get("msg"));
366        }
367    }
368    
369    /**
370     * Updates the ametys.url property of the Solr cores.
371     */
372    public void updateAmetysUrlCoreProperty()
373    {
374        Set<String> coreNames;
375        try
376        {
377            coreNames = getCoreNames();
378        }
379        catch (SolrServerException | IOException e)
380        {
381            getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e);
382            return;
383        }
384        
385        for (String coreName : coreNames)
386        {
387            String collection = _solrClientProvider.getCollectionName(coreName);
388            SolrResponseBase response;
389            try
390            {
391                response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection);
392            }
393            catch (SolrServerException | IOException e)
394            {
395                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e);
396                continue;
397            }
398            
399            NamedList<Object> responseParams = response.getResponse();
400            if ("ok".equals(responseParams.get("result")))
401            {
402                Boolean valueChanged = responseParams.getBooleanArg("valueChanged");
403                if (valueChanged)
404                {
405                    getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName);
406                }
407                else
408                {
409                    getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName);
410                }
411            }
412            else
413            {
414                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName);
415            }
416        }
417    }
418    
419    private void _createConfigset(String name) throws IOException, SolrServerException
420    {
421        getLogger().info("Creating (if necessary) configset '{}'", name);
422        
423        // This request handler will check if configset exists. If not it will be created.
424        SolrQuery query = new SolrQuery();
425        query.setRequestHandler("/admin/cores");
426        query.setParam("action", "createConfigset");
427        query.setParam("name", name);
428        
429        QueryResponse response = _defaultSolrClient().query(query);
430        NamedList<?> results = response.getResponse();
431        
432        NamedList<?> error = (NamedList<?>) results.get("error");
433        if (error != null)
434        {
435            throw new IOException("Error creating the core: " + error.get("msg"));
436        }
437    }
438    
439    /**
440     * Delete a Solr core.
441     * @param name The name of the core to delete.
442     * @throws IOException If an I/O error occurs.
443     * @throws SolrServerException If a Solr error occurs.
444     */
445    public void deleteCore(String name) throws IOException, SolrServerException
446    {
447        String fullName = _solrCorePrefix + name;
448        
449        getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName);
450        
451        SolrQuery query = new SolrQuery();
452        query.setRequestHandler("/admin/cores");
453        query.setParam("action", "UNLOAD");
454        query.setParam("core", fullName);
455        query.setParam("deleteInstanceDir", "true");
456        
457        QueryResponse response = _defaultSolrClient().query(query);
458        NamedList<?> results = response.getResponse();
459        
460        NamedList<?> error = (NamedList<?>) results.get("error");
461        if (error != null)
462        {
463            throw new IOException("Error deleting core" + name + ": " + error.get("msg"));
464        }
465    }
466    
467    /**
468     * Send the schema.
469     * @throws IOException If a communication error occurs.
470     * @throws SolrServerException If a solr error occurs.
471     */
472    public void sendSchema() throws IOException, SolrServerException
473    {
474        getLogger().info("Computing and sending the schema to the solr server.");
475        
476        String workspaceName = _workspaceSelector.getWorkspace();
477        String collection = _solrClientProvider.getCollectionName(workspaceName);
478        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
479        
480//        SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema();
481        SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml");
482        
483        // TODO Clear the schema except fields marked ametysReadOnly="true".
484        
485        // Clear the current schema.
486        clearSchema(solrClient, collection);
487        
488        SchemaRequest schemaRequest = new SchemaRequest();
489        SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
490        
491        // The cleared schema contains only the basic fields which can't be deleted.
492        SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation();
493        SchemaFields schemaFields = new SchemaFields(clearedSchema);
494        
495        getLogger().debug("Schema after clear: \n{}", schemaFields.toString());
496        
497        // Add the static schema types and fields.
498        List<SchemaRequest.Update> updates = new ArrayList<>();
499        
500        // Set "add field" definitions from the static schema to the update list.
501        addStaticSchemaUpdates(updates, staticSchema, schemaFields);
502        
503        getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString());
504        
505        // Set "add field" definitions from the static schema to the update list.
506        addCustomUpdates(updates, schemaFields);
507        
508        reorderUpdates(updates);
509        
510        SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates);
511        SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
512        
513        getLogger().debug("Send schema response: {}", updateResponse.toString());
514        Object errors = updateResponse.getResponse().get("errors");
515        if (errors != null && errors instanceof List && !((List) errors).isEmpty())
516        {
517            String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString();
518            throw new SolrServerException(msg);
519        }
520    
521        getLogger().info("Schema sent to the solr server.");
522        
523        reloadCores();
524    }
525    
526    /**
527     * Compute the list of {@link Update} directives from the static schema.
528     * @param updates The list of {@link Update} directives to fill.
529     * @param staticSchema The static schema representation.
530     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
531     */
532    protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields)
533    {
534        List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes();
535        for (FieldTypeDefinition fieldType : fieldTypes)
536        {
537            String name = (String) fieldType.getAttributes().get("name");
538            if (!schemaFields.hasFieldType(name))
539            {
540                updates.add(new SchemaRequest.AddFieldType(fieldType));
541                schemaFields.addFieldType(name);
542            }
543        }
544        for (Map<String, Object> field : staticSchema.getFields())
545        {
546            String name = (String) field.get("name");
547            if (!schemaFields.hasField(name))
548            {
549                updates.add(new SchemaRequest.AddField(field));
550                schemaFields.addField(name);
551            }
552        }
553        for (Map<String, Object> field : staticSchema.getDynamicFields())
554        {
555            String name = (String) field.get("name");
556            if (!schemaFields.hasDynamicField(name))
557            {
558                updates.add(new SchemaRequest.AddDynamicField(field));
559                schemaFields.addDynamicField(name);
560            }
561        }
562        for (Map<String, Object> field : staticSchema.getCopyFields())
563        {
564            String source = (String) field.get("source");
565            String dest = (String) field.get("dest");
566            if (!schemaFields.hasCopyField(source, dest))
567            {
568                updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest)));
569                schemaFields.addCopyField(source, dest);
570            }
571        }
572    }
573    
574    /**
575     * Compute the list of custom {@link Update} directives.
576     * @param updates The list of {@link Update} directives to fill.
577     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
578     */
579    protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields)
580    {
581        // Add all our property-managed fields.
582        for (String providerId : _schemaDefProviderEP.getExtensionsIds())
583        {
584            SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId);
585            
586            for (SchemaDefinition definition : definitionProvider.getDefinitions())
587            {
588                if (!definitionExists(definition, schemaFields))
589                {
590                    SchemaRequest.Update update = getSchemaUpdate(definition);
591                    if (update != null)
592                    {
593                        updates.add(update);
594                    }
595                }
596            }
597        }
598        
599//        for (String propId : _sysPropEP.getExtensionsIds())
600//        {
601//            Collection<SchemaDefinition> definitions = _sysPropEP.getExtension(propId).getSchemaDefinitions();
602//            
603//            for (SchemaDefinition definition : definitions)
604//            {
605//                if (!definitionExists(definition, schemaFields))
606//                {
607//                    SchemaRequest.Update update = getSchemaUpdate(definition);
608//                    if (update != null)
609//                    {
610//                        updates.add(update);
611//                    }
612//                }
613//            }
614//        }
615    }
616    
617    /**
618     * Reorder the list of {@link Update} directives.
619     * @param updates The list of {@link Update} directives to reorder.
620     */
621    protected void reorderUpdates(List<SchemaRequest.Update> updates)
622    {
623        updates.sort(Comparator.comparing(u ->
624        {
625            if (u instanceof SchemaRequest.AddCopyField)
626            {
627                // copyField at the end of the list to avoid an exception of an undeclared 'source' or 'dest' whereas it is declared later
628                return 1;
629            }
630            else
631            {
632                return 0;
633            }
634        }));
635    }
636    
637    /**
638     * Test if the given schema definition exists in the given {@link SchemaFields} reference.
639     * @param definition The schema definition to test.
640     * @param schemaFields The current schema fields.
641     * @return true if the SchemaFields contain the schema definition.
642     */
643    protected boolean definitionExists(SchemaDefinition definition, SchemaFields schemaFields)
644    {
645        if (definition instanceof FieldDefinition)
646        {
647            FieldDefinition fieldDef = (FieldDefinition) definition;
648            if (fieldDef.isDynamic())
649            {
650                return schemaFields.hasField(fieldDef.getName());
651            }
652            else
653            {
654                return schemaFields.hasDynamicField(fieldDef.getName());
655            }
656        }
657        else if (definition instanceof CopyFieldDefinition)
658        {
659            CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition;
660            return schemaFields.hasCopyField(fieldDef.getSource(), fieldDef.getDestination());
661        }
662        
663        return false;
664    }
665    
666    /**
667     * Delete all the fields of the existing schema in the given collection.
668     * @param solrClient The Solr client
669     * @param collection The collection.
670     * @throws IOException If a communication error occurs.
671     * @throws SolrServerException If a solr error occurs.
672     */
673    protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException
674    {
675        try
676        {
677            getLogger().info("Clearing the existing schema on the solr server.");
678            
679            SchemaRequest schemaRequest = new SchemaRequest();
680            SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
681            
682            SchemaRepresentation schema = schemaResponse.getSchemaRepresentation();
683            
684            List<SchemaRequest.Update> deletions = new ArrayList<>();
685            
686            // First the copy fields, then dynamic and simple fields, and field types in the end.
687            for (Map<String, Object> field : schema.getCopyFields())
688            {
689                String source = (String) field.get("source");
690                String dest = (String) field.get("dest");
691                deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest)));
692            }
693            for (Map<String, Object> field : schema.getDynamicFields())
694            {
695                String name = (String) field.get("name");
696                deletions.add(new SchemaRequest.DeleteDynamicField(name));
697            }
698            for (Map<String, Object> field : schema.getFields())
699            {
700                String name = (String) field.get("name");
701                if (!_READ_ONLY_FIELDS.contains(name))
702                {
703                    deletions.add(new SchemaRequest.DeleteField(name));
704                }
705            }
706            for (FieldTypeDefinition fieldType : schema.getFieldTypes())
707            {
708                String name = (String) fieldType.getAttributes().get("name");
709                if (!_READ_ONLY_FIELDTYPES.contains(name))
710                {
711                    deletions.add(new SchemaRequest.DeleteFieldType(name));
712                }
713            }
714            
715            SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions);
716            SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
717            
718            Object errors = updateResponse.getResponse().get("errors");
719            if (errors != null && errors instanceof List && !((List) errors).isEmpty())
720            {
721                String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString();
722                throw new SolrServerException(msg);
723            }
724            else
725            {
726                getLogger().debug("Clear schema response: {}", updateResponse.toString());
727            }
728            
729            getLogger().info("Solr schema cleared.");
730        }
731        catch (SolrServerException | IOException e)
732        {
733            getLogger().error("Error clearing schema in collection " + collection, e);
734            throw e;
735        }
736    }
737    
738    /**
739     * Get the schema {@link Update} directive from the given schema definition.
740     * @param definition The schema definition to add.
741     * @return The add {@link Update} directive.
742     */
743    protected SchemaRequest.Update getSchemaUpdate(SchemaDefinition definition)
744    {
745        SchemaRequest.Update update = null;
746        
747        if (definition instanceof FieldDefinition)
748        {
749            FieldDefinition fieldDef = (FieldDefinition) definition;
750            
751            // Force indexed and stored attributes to true.
752            Map<String, Object> attributes = new HashMap<>();
753            attributes.put("name", fieldDef.getName());
754            attributes.put("type", fieldDef.getType());
755            attributes.put("multiValued", fieldDef.isMultiValued());
756            attributes.put("docValues", fieldDef.isDocValues());
757            attributes.put("indexed", Boolean.TRUE);
758            attributes.put("stored", Boolean.TRUE);
759            
760            update = new SchemaRequest.AddField(attributes);
761        }
762        else if (definition instanceof CopyFieldDefinition)
763        {
764            CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition;
765            
766            String source = fieldDef.getSource();
767            String dest = fieldDef.getDestination();
768            
769            update = new SchemaRequest.AddCopyField(source, Arrays.asList(dest));
770        }
771        
772        return update;
773    }
774    
775    /**
776     * Reload the solr cores.
777     * @throws IOException If a communication error occurs.
778     * @throws SolrServerException If a solr error occurs.
779     */
780    protected void reloadCores() throws IOException, SolrServerException
781    {
782        getLogger().info("Reloading solr cores.");
783        
784        for (String coreName : getCoreNames())
785        {
786            String fullName = _solrCorePrefix + coreName;
787            
788            CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient());
789            
790            getLogger().debug("Reload core response: {}", reloadResponse.toString());
791        }
792        
793        getLogger().info("All cores reloaded.");
794    }
795    
796    /**
797     * Reloads the ACL Solr cache for all users
798     * @throws IOException If an I/O error occurs.
799     * @throws SolrServerException If a Solr error occurs.
800     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
801     */
802    public void reloadAclCache() throws IOException, SolrServerException, RepositoryException
803    {
804        String[] workspaceNames = _repository.getWorkspaces();
805        for (String workspaceName : workspaceNames)
806        {
807            reloadAclCache(workspaceName);
808        }
809    }
810    
811    /**
812     * Reloads the ACL Solr cache for all users
813     * @param workspaceName The workspace name
814     * @throws IOException If an I/O error occurs.
815     * @throws SolrServerException If a Solr error occurs.
816     */
817    public void reloadAclCache(String workspaceName) throws IOException, SolrServerException
818    {
819        reloadAclCache(workspaceName, false);
820    }
821    
822    /**
823     * Reloads the ACL Solr cache for all users
824     * @param workspaceName The workspace name
825     * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache)
826     * @throws IOException If an I/O error occurs.
827     * @throws SolrServerException If a Solr error occurs.
828     */
829    public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException
830    {
831        getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
832        
833        String collection = _solrClientProvider.getCollectionName(workspaceName);
834        SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection);
835        NamedList<Object> responseObj = responseBase.getResponse();
836        
837        if ("ok".equals(responseObj.get("result")))
838        {
839            getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
840        }
841        else
842        {
843            Object error = responseObj.get("error");
844            getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error);
845        }
846    }
847    
848    /**
849     * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces.
850     * @param objects the {@link AmetysObject}s to update.
851     * @throws IOException If an I/O error occurs.
852     * @throws SolrServerException If a Solr error occurs.
853     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
854     */
855    public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException
856    {
857        String[] workspaceNames = _repository.getWorkspaces();
858        for (String workspaceName : workspaceNames)
859        {
860            updateAclCache(objects, workspaceName);
861        }
862    }
863    
864    /**
865     * Updates the ACL Solr cache for some {@link AmetysObject}s.
866     * @param objects the {@link AmetysObject}s to update.
867     * @param workspaceName The workspace name
868     * @throws IOException If an I/O error occurs.
869     * @throws SolrServerException If a Solr error occurs.
870     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
871     */
872    public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException
873    {
874        Map<String, Map<String, Object>> solrParams = new HashMap<>();
875        
876        for (AmetysObject object : objects)
877        {
878            AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object);
879            
880            solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(),
881                                                  "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(),
882                                                  "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
883                                                  "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
884                                                  "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()),
885                                                  "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList())));
886        }
887        
888        String collection = _solrClientProvider.getCollectionName(workspaceName);
889        SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection);
890        NamedList<Object> responseObj = response.getResponse();
891        
892        if ("ok".equals(responseObj.get("result")))
893        {
894            if (getLogger().isInfoEnabled())
895            {
896                getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet());
897            }
898        }
899        else
900        {
901            @SuppressWarnings("unchecked")
902            List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects");
903            getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects);
904        }
905    }
906    
907    /**
908     * Index all the contents in a given workspace.
909     * @param workspaceName the workspace where to index
910     * @param indexAttachments to index content attachments
911     * @param solrClient The solr client to use
912     * @return The indexation result as a Map.
913     * @throws Exception if an error occurs while indexing.
914     */
915    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
916    {
917        Request request = ContextHelper.getRequest(_context);
918        
919        // Retrieve the current workspace.
920        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
921        
922        try
923        {
924            // Force the workspace.
925            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
926            
927            getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName);
928            
929            long start = System.currentTimeMillis();
930            
931            // Delete all contents
932            unindexAllContents(workspaceName, indexAttachments, solrClient);
933            
934            String query = ContentQueryHelper.getContentXPathQuery(null);
935            AmetysObjectIterable<Content> contents = _resolver.query(query);
936            
937            IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient);
938            
939            long end = System.currentTimeMillis();
940            
941            if (!result.hasErrors())
942            {
943                getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
944            }
945            else
946            {
947                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
948            }
949            
950            Map<String, Object> results = new HashMap<>();
951            results.put("successCount", result.getSuccessCount());
952            if (result.hasErrors())
953            {
954                results.put("errorCount", result.getErrorCount());
955            }
956            
957            return results;
958        }
959        catch (Exception e)
960        {
961            String error = String.format("Failed to index all contents in workspace %s", workspaceName);
962            getLogger().error(error, e);
963            throw new IndexingException(error, e);
964        }
965        finally
966        {
967            // Restore context
968            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
969        }
970    }
971    
972    /**
973     * Unindex all content documents.
974     * @param workspaceName The workspace name
975     * @param unindexAttachments also unindex content attachments
976     * @param solrClient The solr client to use
977     * @throws Exception if an error occurs while unindexing.
978     */
979    protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
980    {
981        String collection = _solrClientProvider.getCollectionName(workspaceName);
982        
983        Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT);
984        Query query;
985        if (unindexAttachments)
986        {
987            Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE);
988            Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE);
989            query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents);
990        }
991        else
992        {
993            query = contents;
994        }
995        solrClient.deleteByQuery(collection, query.build());
996    }
997    
998    /**
999     * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit
1000     * @param collectionId The id of collection 
1001     * @param indexAttachments to index content attachments
1002     * @throws Exception if an error occurs while indexing.
1003     */
1004    public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception
1005    {
1006        String[] workspaceNames = _repository.getWorkspaces();
1007        for (String workspaceName : workspaceNames)
1008        {
1009            indexSubcontents(collectionId, workspaceName, indexAttachments);
1010        }
1011    }
1012    
1013    /**
1014     * Index the child contents of a {@link AmetysObjectCollection}
1015     * @param collectionId The id of collection 
1016     * @param workspaceName the workspace where to index
1017     * @param indexAttachments to index content attachments
1018     * @throws Exception if an error occurs while unindexing.
1019     */
1020    public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception
1021    {
1022        Request request = ContextHelper.getRequest(_context);
1023        
1024        // Retrieve the current workspace.
1025        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1026        
1027        try
1028        {
1029            // Force the workspace.
1030            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1031            
1032            if (_resolver.hasAmetysObjectForId(collectionId))
1033            {
1034                SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1035                AmetysObjectCollection collection = _resolver.resolveById(collectionId);
1036                AmetysObjectIterable<AmetysObject> children = collection.getChildren();
1037                
1038                for (AmetysObject child : children)
1039                {
1040                    if (child instanceof Content)
1041                    {
1042                        Content content = (Content) child;
1043                        
1044                        _doIndexContent(content, workspaceName, indexAttachments, solrClient);
1045                    }
1046                }
1047            }
1048        }
1049        finally
1050        {
1051            // Restore context
1052            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1053        }
1054    }
1055    
1056    /**
1057     * Add or update a content into Solr index on all workspaces and commit
1058     * @param contentId The id of the content to index
1059     * @param indexAttachments to index content attachments
1060     * @throws Exception if an error occurs while indexing.
1061     */
1062    public void indexContent(String contentId, boolean indexAttachments) throws Exception
1063    {
1064        String[] workspaceNames = _repository.getWorkspaces();
1065        for (String workspaceName : workspaceNames)
1066        {
1067            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1068            indexContent(contentId, workspaceName, indexAttachments, solrClient);
1069        }
1070    }
1071    
1072    /**
1073     * Add or update a content into Solr index
1074     * @param contentId The id of the content to index
1075     * @param workspaceName the workspace where to index
1076     * @param indexAttachments to index content attachments
1077     * @throws Exception if an error occurs while indexing.
1078     */
1079    public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception
1080    {
1081        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1082        indexContent(contentId, workspaceName, indexAttachments, solrClient);
1083    }
1084    
1085    /**
1086     * Add or update a content into Solr index
1087     * @param contentId The id of the content to index
1088     * @param workspaceName the workspace where to index
1089     * @param indexAttachments to index content attachments
1090     * @param solrClient The solr client to use
1091     * @throws Exception if an error occurs while indexing.
1092     */
1093    public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1094    {
1095        Request request = ContextHelper.getRequest(_context);
1096        
1097        // Retrieve the current workspace.
1098        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1099        
1100        try
1101        {
1102            // Force the workspace.
1103            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1104            
1105            if (_resolver.hasAmetysObjectForId(contentId))
1106            {
1107                Content content = _resolver.resolveById(contentId);
1108                _doIndexContent(content, workspaceName, indexAttachments, solrClient);
1109            }
1110        }
1111        finally
1112        {
1113            // Restore context
1114            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1115        }
1116    }
1117    
1118    private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException
1119    {
1120        try
1121        {
1122            long time_0 = System.currentTimeMillis();
1123            
1124            getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName);
1125            
1126            deleteRepeaterDocs(content.getId(), workspaceName, solrClient);
1127            doIndexContent(content, workspaceName, solrClient);
1128            doIndexContentWorkflow(content, workspaceName, solrClient);
1129            if (indexAttachments)
1130            {
1131                indexContentAttachments(content.getRootAttachments(), content, solrClient);
1132            }
1133            
1134            getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0);
1135        }
1136        catch (Exception e)
1137        {
1138            String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName);
1139            getLogger().error(error, e);
1140            throw new IndexingException(error, e);
1141        }
1142    }
1143    
1144    /**
1145     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1146     * @param contents the collection of contents to index.
1147     * @return the indexation result.
1148     * @throws Exception if an error occurs while indexing.
1149     */
1150    public IndexationResult indexContents(Iterable<Content> contents) throws Exception
1151    {
1152        IndexationResult result = new IndexationResult();
1153        
1154        String[] workspaceNames = _repository.getWorkspaces();
1155        for (String workspaceName : workspaceNames)
1156        {
1157            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1158            IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient);
1159            result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount());
1160        }
1161        
1162        return result;
1163    }
1164    
1165    /**
1166     * Send a collection of contents for indexation in the solr server.
1167     * @param contents the collection of contents to index.
1168     * @param workspaceName the workspace where to index
1169     * @param indexAttachments to index content attachments
1170     * @param solrClient The solr client to use
1171     * @return the indexation result.
1172     * @throws Exception if an error occurs while indexing.
1173     */
1174    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1175    {
1176        Request request = ContextHelper.getRequest(_context);
1177        
1178        // Retrieve the current workspace.
1179        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1180        
1181        try
1182        {
1183            // Force the workspace.
1184            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1185            
1186            getLogger().info("Starting indexation of several contents for workspace {}", workspaceName);
1187            
1188            long start = System.currentTimeMillis();
1189            
1190            List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false)
1191                .map(Content::getId)
1192                .map(this::_resolveSilently)
1193                .filter(Objects::nonNull)
1194                .collect(Collectors.toList());
1195                
1196            IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient);
1197            
1198            long end = System.currentTimeMillis();
1199            
1200            if (!result.hasErrors())
1201            {
1202                getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1203            }
1204            else
1205            {
1206                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1207            }
1208            
1209            return result;
1210        }
1211        catch (Exception e)
1212        {
1213            String error = String.format("Failed to index several contents in workspace %s", workspaceName);
1214            getLogger().error(error, e);
1215            throw new IndexingException(error, e);
1216        }
1217        finally
1218        {
1219            // Restore context
1220            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1221        }
1222            
1223    }
1224    
1225    private Content _resolveSilently(String contentId)
1226    {
1227        try
1228        {
1229            return _resolver.resolveById(contentId);
1230        }
1231        catch (UnknownAmetysObjectException e) 
1232        {
1233            return null;
1234        }
1235    }
1236    
1237    /**
1238     * Send some contents for indexation in the solr server.
1239     * @param contents the contents to index.
1240     * @param workspaceName The workspace name
1241     * @param indexAttachments to index content attachments
1242     * @param solrClient The solr client to use
1243     * @return the indexation result.
1244     * @throws Exception if an error occurs committing the results.
1245     */
1246    protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1247    {
1248        int successCount = 0;
1249        int errorCount = 0;
1250        for (Content content : contents)
1251        {
1252            try
1253            {
1254                doIndexContent(content, workspaceName, solrClient);
1255                doIndexContentWorkflow(content, workspaceName, solrClient);
1256                if (indexAttachments)
1257                {
1258                    indexContentAttachments(content.getRootAttachments(), content, solrClient);
1259                }
1260                successCount++;
1261            }
1262            catch (Exception e)
1263            {
1264                getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e);
1265                errorCount++;
1266            }
1267        }
1268        
1269        return new IndexationResult(successCount, errorCount);
1270    }
1271    
1272    /**
1273     * Update the value of a specific system property in a content document.
1274     * @param content The content to update.
1275     * @param propertyId The system property ID.
1276     * @param workspaceName The workspace name
1277     * @throws Exception if an error occurs while indexing.
1278     */
1279    public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception
1280    {
1281        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1282        updateSystemProperty(content, propertyId, workspaceName, solrClient);
1283    }
1284    
1285    /**
1286     * Update the value of a specific system property in a content document.
1287     * @param content The content to update.
1288     * @param propertyId The system property ID.
1289     * @param workspaceName The workspace name
1290     * @param solrClient The solr client to use
1291     * @throws Exception if an error occurs while indexing.
1292     */
1293    public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1294    {
1295        getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content);
1296        
1297        SolrInputDocument document = new SolrInputDocument();
1298        boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document);
1299        
1300        if (!hasUpdate)
1301        {
1302            getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content);
1303            return;
1304        }
1305        
1306        String collection = _solrClientProvider.getCollectionName(workspaceName);
1307        UpdateResponse solrResponse = solrClient.add(collection, document);
1308        int status = solrResponse.getStatus();
1309        
1310        if (status != 0)
1311        {
1312            throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'.");
1313        }
1314        
1315        getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content);
1316    }
1317    
1318    /**
1319     * Remove a content from Solr index for all workspaces and commit
1320     * @param contentId The id of content to unindex
1321     * @param unindexAttachments also unindex content attachments
1322     * @throws Exception if an error occurs while indexing.
1323     */
1324    public void unindexContent(String contentId, boolean unindexAttachments) throws Exception
1325    {
1326        String[] workspaceNames = _repository.getWorkspaces();
1327        for (String workspaceName : workspaceNames)
1328        {
1329            unindexContent(contentId, workspaceName, unindexAttachments);
1330        }
1331    }
1332    
1333    /**
1334     * Remove a content from Solr index
1335     * @param contentId The id of content to unindex
1336     * @param workspaceName The workspace where to work in 
1337     * @param unindexAttachments also unindex content attachments
1338     * @throws Exception if an error occurs while indexing.
1339     */
1340    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception
1341    {
1342        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1343        unindexContent(contentId, workspaceName, unindexAttachments, solrClient);
1344    }
1345    
1346    /**
1347     * Remove a content from Solr index
1348     * @param contentId The id of content to unindex
1349     * @param workspaceName The workspace where to work in 
1350     * @param unindexAttachments also unindex content attachments
1351     * @param solrClient The solr client to use
1352     * @throws Exception if an error occurs while indexing.
1353     */
1354    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
1355    {
1356        Request request = ContextHelper.getRequest(_context);
1357        
1358        // Retrieve the current workspace.
1359        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1360        
1361        try
1362        {
1363            // Force the workspace.
1364            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1365            
1366            getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName);
1367            
1368            deleteRepeaterDocs(contentId, workspaceName, solrClient);
1369            doUnindexDocument(contentId, workspaceName, solrClient);
1370            if (unindexAttachments)
1371            {
1372                doUnindexContentAttachments(contentId, workspaceName, solrClient);
1373            }
1374            _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1375            
1376            getLogger().debug("Succesfully deleted content {} from Solr.", contentId);
1377        }
1378        catch (Exception e)
1379        {
1380            String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName);
1381            getLogger().error(error, e);
1382            throw new IndexingException(error, e);
1383        }
1384        finally
1385        {
1386            // Restore workspace
1387            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1388        }
1389    }
1390    
1391    /**
1392     * Remove a content from Solr index for all workspaces and commit
1393     * @param contentIds The id of content to unindex
1394     * @throws Exception if an error occurs while indexing.
1395     */
1396    public void unindexContents(Collection<String> contentIds) throws Exception
1397    {
1398        String[] workspaceNames = _repository.getWorkspaces();
1399        for (String workspaceName : workspaceNames)
1400        {
1401            unindexContents(contentIds, workspaceName);
1402        }
1403    }
1404    
1405    /**
1406     * Remove a content from Solr index
1407     * @param contentIds The id of content to unindex
1408     * @param workspaceName The workspace where to work in 
1409     * @throws Exception if an error occurs while indexing.
1410     */
1411    public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception
1412    {
1413        Request request = ContextHelper.getRequest(_context);
1414        
1415        // Retrieve the current workspace.
1416        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1417        
1418        try
1419        {
1420            // Force the workspace.
1421            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1422            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1423            
1424            getLogger().debug("Unindexing several contents from Solr.");
1425            
1426            for (String contentId : contentIds)
1427            {
1428                deleteRepeaterDocs(contentId, workspaceName, solrClient);
1429                doUnindexDocument(contentId, workspaceName, solrClient);
1430                _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1431            }
1432            
1433            getLogger().debug("Succesfully unindexed content from Solr.");
1434        }
1435        catch (Exception e)
1436        {
1437            String error = String.format("Failed to unindex several contents in workspace %s", workspaceName);
1438            getLogger().error(error, e);
1439            throw new IndexingException(error, e);
1440        }
1441        finally
1442        {
1443            // Restore workspace
1444            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1445        }
1446    }
1447    
1448    /**
1449     * Add or update a content into Solr index
1450     * @param content The content to index
1451     * @param workspaceName The workspace where to index
1452     * @param solrClient The solr client to use
1453     * @throws Exception if an error occurs while indexing.
1454     */
1455    protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception
1456    {
1457        long time_0 = System.currentTimeMillis();
1458        
1459        SolrInputDocument document = new SolrInputDocument();
1460        List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document);
1461        
1462        long time_1 = System.currentTimeMillis();
1463        getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0);
1464        
1465        indexAclInitValues(content, document);
1466        
1467        long time_2 = System.currentTimeMillis();
1468        getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1);
1469        
1470        List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments);
1471        documents.add(document);
1472        
1473        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents);
1474        int status = solrResponse.getStatus();
1475        
1476        long time_3 = System.currentTimeMillis();
1477        getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2);
1478        
1479        if (status != 0)
1480        {
1481            throw new IOException("Content indexation: got status code '" + status + "'.");
1482        }
1483    }
1484    
1485    /**
1486     * Indexes read-ACl initial values for object
1487     * @param ametysObject The object
1488     * @param document The Solr document
1489     */
1490    public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document)
1491    {
1492        // Indexation of AmetysObject property
1493        document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true);
1494        
1495        // Indexation of initValues for AllowedUsers
1496        AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject);
1497        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed());
1498        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed());
1499        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString);
1500        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString);
1501        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString);
1502        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString);
1503    }
1504    
1505    private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier)
1506    {
1507        if (values == null)
1508        {
1509            return;
1510        }
1511        
1512        for (T value : values)
1513        {
1514            document.addField(fieldName, stringifier.apply(value));
1515        }
1516    }
1517    
1518    /**
1519     * Index the whole workflow of a content.
1520     * @param content The content.
1521     * @param workspaceName The workspace name
1522     * @param solrClient The solr client to use
1523     * @throws Exception if an error occurs while indexing.
1524     */
1525    protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception
1526    {
1527        if (content instanceof WorkflowAwareContent)
1528        {
1529            _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient);
1530        }
1531    }
1532    
1533    /**
1534     * Index content attachments as new entries in the idnex
1535     * @param collection the collection of attachments
1536     * @param content the content whose attachments will be indexed
1537     * @throws Exception if something goes wrong when indexing the attachments of the content
1538     */
1539    public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception
1540    {
1541        Request request = ContextHelper.getRequest(_context);
1542        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1543        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1544        indexContentAttachments(collection, content, solrClient);
1545    }
1546    
1547    /**
1548     * Index content attachments as new entries in the idnex
1549     * @param collection the collection of attachments
1550     * @param content the content whose attachments will be indexed
1551     * @param solrClient The solr client to use
1552     * @throws Exception if something goes wrong when indexing the attachments of the content
1553     */
1554    public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception
1555    {
1556        if (collection == null)
1557        {
1558            return;
1559        }
1560        
1561        try (AmetysObjectIterable<AmetysObject> children = collection.getChildren())
1562        {
1563            for (AmetysObject object : children)
1564            {
1565                if (object instanceof ResourceCollection)
1566                {
1567                    indexContentAttachments((ResourceCollection) object, content, solrClient);
1568                }
1569                else if (object instanceof Resource)
1570                {
1571                    Resource resource = (Resource) object;
1572                    indexContentAttachment(resource, content, solrClient);
1573                }
1574            }
1575        }
1576    }
1577    
1578    /**
1579     * Index a content attachment
1580     * @param resource the content attachment as a {@link Resource}
1581     * @param content the content whose attachment is going to be indexed
1582     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1583     */
1584    public void indexContentAttachment(Resource resource, Content content) throws Exception
1585    {
1586        Request request = ContextHelper.getRequest(_context);
1587        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1588        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1589        indexContentAttachment(resource, content, solrClient);
1590    }
1591    
1592    /**
1593     * Index a content attachment
1594     * @param resource the content attachment as a {@link Resource}
1595     * @param content the content whose attachment is going to be indexed
1596     * @param solrClient The solr client to use
1597     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1598     */
1599    public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception
1600    {
1601        SolrInputDocument document = new SolrInputDocument();
1602        
1603        // Prepare resource doc
1604        _indexContentAttachment(resource, document, content);
1605        
1606        // Indexation of the document
1607        _indexResourceDocument(resource, document, solrClient);
1608    }
1609    
1610    private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception
1611    {
1612        String language = content.getLanguage();
1613        
1614        _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language);
1615        
1616        // Need the id of the content for unindexing attachment during the unindexing of the content
1617        document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId());
1618    }
1619    
1620    /**
1621     * Index a populated solr input document of type Resource.
1622     * @param resource the resource from which the input document is created
1623     * @param document the input document
1624     * @param solrClient The solr client to use
1625     * @throws SolrServerException if there is an error on the server
1626     * @throws IOException if there is a communication error with the server
1627     */
1628    protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException
1629    {
1630        String resourceId = resource.getId();
1631        
1632        // Retrieve appropriate collection name
1633        Request request = ContextHelper.getRequest(_context);
1634        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1635        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
1636        
1637        // Add document
1638        UpdateResponse solrResponse = solrClient.add(collectionName, document);
1639        int status = solrResponse.getStatus();
1640        
1641        if (status != 0)
1642        {
1643            throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId);
1644        }
1645        
1646        getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId);
1647    }
1648    
1649    /**
1650     * Delete repeater documents of a specified content.
1651     * @param workspaceName The workspace name
1652     * @param contentId the content ID.
1653     * @param solrClient The solr client to use
1654     * @throws Exception if an error occurs while indexing.
1655     */
1656    protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception
1657    {
1658        long time_0 = System.currentTimeMillis();
1659        
1660        // _documentType:repeater AND id:content\://xxx/*
1661        StringBuilder query = new StringBuilder();
1662        query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER)
1663             .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*");
1664        
1665        solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString());
1666        
1667        getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0);
1668    }
1669    
1670    /**
1671     * Index all the resources in a given workspace.
1672     * @param workspaceName The workspace where to index
1673     * @param solrClient The solr client to use
1674     * @return The indexation result as a Map.
1675     * @throws Exception if an error occurs while indexing.
1676     */
1677    public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient) throws Exception
1678    {
1679        Request request = ContextHelper.getRequest(_context);
1680        
1681        // Retrieve the current workspace.
1682        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1683        
1684        try
1685        {
1686            // Force the workspace.
1687            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1688            
1689            getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName);
1690            
1691            long start = System.currentTimeMillis();
1692            
1693            // Delete all resources
1694            _unindexAllResources(workspaceName, solrClient);
1695            
1696            Map<String, Object> results = new HashMap<>();
1697            
1698            try
1699            {
1700                TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources");
1701                AmetysObjectIterable<Resource> resources = resourceRoot.getChildren();
1702                
1703                IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient);
1704                
1705                long end = System.currentTimeMillis();
1706                
1707                if (!result.hasErrors())
1708                {
1709                    getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1710                }
1711                else
1712                {
1713                    getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1714                }
1715                
1716                results.put("successCount", result.getSuccessCount());
1717                if (result.hasErrors())
1718                {
1719                    results.put("errorCount", result.getErrorCount());
1720                }
1721            }
1722            catch (UnknownAmetysObjectException e)
1723            {
1724                getLogger().info("There is no root for resources in current workspace.");
1725            }
1726            
1727            return results;
1728        }
1729        catch (Exception e)
1730        {
1731            String error = String.format("Failed to index all resources in workspace %s", workspaceName);
1732            getLogger().error(error, e);
1733            throw new IndexingException(error, e);
1734        }
1735        finally
1736        {
1737            // Restore context
1738            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1739        }
1740    }
1741    
1742    /**
1743     * Send a collection of contents for indexation in the solr server.
1744     * @param resources the collection of contents to index.
1745     * @param documentType The document type of the resource
1746     * @param workspaceName The workspace where to index
1747     * @return the indexation result.
1748     * @throws Exception if an error occurs while indexing.
1749     */
1750    public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception
1751    {
1752        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1753        return indexResources(resources, documentType, null, workspaceName, solrClient);
1754    }
1755    
1756    /**
1757     * Send a collection of contents for indexation in the solr server.
1758     * @param resources the collection of contents to index.
1759     * @param documentType The document type of the resource
1760     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1761     * @param workspaceName The workspace where to index
1762     * @param solrClient The solr client to use
1763     * @return the indexation result.
1764     * @throws Exception if an error occurs while indexing.
1765     */
1766    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception
1767    {
1768        Request request = ContextHelper.getRequest(_context);
1769        
1770        // Retrieve the current workspace.
1771        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1772        
1773        try
1774        {
1775            // Force the workspace.
1776            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1777            
1778            getLogger().info("Starting indexation of several resources for workspace {}", workspaceName);
1779            
1780            long start = System.currentTimeMillis();
1781            
1782            IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient);
1783            
1784            long end = System.currentTimeMillis();
1785            
1786            if (!result.hasErrors())
1787            {
1788                getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1789            }
1790            else
1791            {
1792                getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1793            }
1794            
1795            return result;
1796        }
1797        catch (Exception e)
1798        {
1799            String error = String.format("Failed to index several resources in workspace %s", workspaceName);
1800            getLogger().error(error, e);
1801            throw new IndexingException(error, e);
1802        }
1803        finally
1804        {
1805            // Restore context
1806            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1807        }
1808    }
1809    
1810    /**
1811     * Send some resources for indexation in the solr server.
1812     * @param resources the resources to index.
1813     * @param documentType The document type of the resource
1814     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 
1815     * @param workspaceName The workspace where to index
1816     * @param solrClient The solr client to use
1817     * @return the indexation result.
1818     * @throws Exception if an error occurs committing the results.
1819     */
1820    protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception
1821    {
1822        int successCount = 0;
1823        int errorCount = 0;
1824        for (AmetysObject resource : resources)
1825        {
1826            try
1827            {
1828                doIndexExplorerItem(resource, documentType, resourceRoot, solrClient);
1829                successCount++;
1830            }
1831            catch (Exception e)
1832            {
1833                getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e);
1834                errorCount++;
1835            }
1836        }
1837        
1838        return new IndexationResult(successCount, errorCount);
1839    }
1840    
1841    /**
1842     * Add or update a resource into Solr index
1843     * @param resource The resource to index
1844     * @param documentType The document type of the resource
1845     * @param workspaceName The workspace where to index
1846     * @throws Exception if an error occurs while indexing.
1847     */
1848    public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception
1849    {
1850        Request request = ContextHelper.getRequest(_context);
1851        
1852        // Retrieve the current workspace.
1853        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1854        
1855        try
1856        {
1857            // Force the workspace.
1858            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1859            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1860            
1861            getLogger().debug("Indexing resource {} into Solr.", resource.getId());
1862            
1863            doIndexResource(resource, documentType, null, solrClient);
1864            
1865            getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId());
1866        }
1867        catch (Exception e)
1868        {
1869            String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName);
1870            getLogger().error(error, e);
1871            throw new IndexingException(error, e);
1872        }
1873        finally
1874        {
1875            // Restore context
1876            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1877        }
1878    }
1879    
1880    private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception
1881    {
1882        getLogger().debug("Unindexing all resources from Solr.");
1883        
1884        String collection = _solrClientProvider.getCollectionName(workspaceName);
1885        solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE);
1886        
1887        getLogger().debug("Succesfully deleted all resource documents from Solr.");
1888    }
1889    
1890    /**
1891     * Delete all resource documents at a given path for all workspaces and commit
1892     * @param rootId The resource root ID, must not be null.
1893     * @param path The resource path relative to the given root, must start with a slash.
1894     * @throws Exception If an error occurs while unindexing.
1895     */
1896    public void unindexResourcesByPath(String rootId, String path) throws Exception
1897    {
1898        String[] workspaceNames = _repository.getWorkspaces();
1899        for (String workspaceName : workspaceNames)
1900        {
1901            unindexResourcesByPath(rootId, path, workspaceName);
1902        }
1903    }
1904    
1905    /**
1906     * Delete all resource documents at a given path.
1907     * @param rootId The resource root ID, must not be null.
1908     * @param path The resource path relative to the given root, must start with a slash.
1909     * @param workspaceName The workspace where to work in 
1910     * @throws Exception If an error occurs while unindexing.
1911     */
1912    public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception
1913    {
1914        Request request = ContextHelper.getRequest(_context);
1915        
1916        // Retrieve the current workspace.
1917        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1918        
1919        try
1920        {
1921            // Force the workspace.
1922            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1923            
1924            getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId);
1925            
1926            Query query = new ResourceLocationQuery(rootId, path);
1927            
1928            String collection = _solrClientProvider.getCollectionName(workspaceName);
1929            _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build());
1930            
1931            getLogger().debug("Succesfully deleted resource document from Solr.");
1932        }
1933        catch (Exception e)
1934        {
1935            String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName);
1936            getLogger().error(error, e);
1937            throw new IndexingException(error, e);
1938        }
1939        finally
1940        {
1941            // Restore workspace
1942            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1943        }
1944    }
1945    
1946    /**
1947     * Remove a resource from Solr index for all workspaces and commit
1948     * @param resourceId The id of resource to unindex
1949     * @throws Exception if an error occurs while indexing.
1950     */
1951    public void unindexResource(String resourceId) throws Exception
1952    {
1953        String[] workspaceNames = _repository.getWorkspaces();
1954        for (String workspaceName : workspaceNames)
1955        {
1956            unindexResource(resourceId, workspaceName);
1957        }
1958    }
1959    
1960    /**
1961     * Remove a resource from the Solr index.
1962     * @param resourceId The id of resource to unindex
1963     * @param workspaceName The workspace where to work in 
1964     * @throws Exception if an error occurs while unindexing.
1965     */
1966    public void unindexResource(String resourceId, String workspaceName) throws Exception
1967    {
1968        Request request = ContextHelper.getRequest(_context);
1969        
1970        // Retrieve the current workspace.
1971        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1972        
1973        try
1974        {
1975            // Force the workspace.
1976            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1977            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1978            
1979            getLogger().debug("Unindexing resource {} from Solr.", resourceId);
1980            
1981            doUnindexDocument(resourceId, workspaceName, solrClient);
1982            
1983            getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId);
1984        }
1985        catch (Exception e)
1986        {
1987            String error = String.format("Failed to unindex resource  %s in workspace %s", resourceId, workspaceName);
1988            getLogger().error(error, e);
1989            throw new IndexingException(error, e);
1990        }
1991        finally
1992        {
1993            // Restore workspace
1994            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1995        }
1996    }
1997    
1998    /**
1999     * Add or update a resource into Solr index
2000     * @param node The resource to index
2001     * @param documentType The document type of the resource
2002     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 
2003     * @param solrClient The solr client to use
2004     * @throws Exception if an error occurs while indexing.
2005     */
2006    protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2007    {
2008        if (node instanceof ResourceCollection)
2009        {
2010            try (AmetysObjectIterable<AmetysObject> children = ((ResourceCollection) node).getChildren())
2011            {
2012                for (AmetysObject child : children)
2013                {
2014                    doIndexExplorerItem(child, documentType, resourceRoot, solrClient);
2015                }
2016            }
2017        }
2018        else if (node instanceof Resource)
2019        {
2020            doIndexResource((Resource) node, documentType, resourceRoot, solrClient);
2021        }
2022    }
2023    
2024    /**
2025     * Add or update a resource into Solr index
2026     * @param resource The resource to index
2027     * @param documentType The document type of the resource
2028     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
2029     * @param solrClient The solr client to use 
2030     * @throws Exception if an error occurs while indexing.
2031     */
2032    protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2033    {
2034        SolrInputDocument document = new SolrInputDocument();
2035        
2036        _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot);
2037        
2038        String workspaceName = _workspaceSelector.getWorkspace();
2039        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document);
2040        int status = solrResponse.getStatus();
2041        
2042        if (status != 0)
2043        {
2044            throw new IOException("Resource indexation: got status code '" + status + "'.");
2045        }
2046    }
2047    
2048    /**
2049     * Process a Solr commit operation in all workspaces.
2050     * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient}
2051     * @throws SolrServerException if there is an error on the server
2052     * @throws IOException if there is a communication error with the server
2053     */
2054    public void commit() throws SolrServerException, IOException
2055    {
2056        String[] workspaceNames;
2057        try
2058        {
2059            workspaceNames = _repository.getWorkspaces();
2060            for (String workspaceName : workspaceNames)
2061            {
2062                SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName);
2063                commit(workspaceName, solrClient);
2064            }
2065        }
2066        catch (RepositoryException e)
2067        {
2068            throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e);
2069        }
2070    }
2071    
2072    /**
2073     * Process a Solr commit operation in given workspace.
2074     * @param workspaceName The workspace's name
2075     * @param solrClient The solr client to use
2076     * @throws SolrServerException if there is an error on the server
2077     * @throws IOException if there is a communication error with the server
2078     */
2079    public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2080    {
2081        long time_0 = System.currentTimeMillis();
2082        
2083        // Commit
2084        UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName));
2085        int status = solrResponse.getStatus();
2086        
2087        if (status != 0)
2088        {
2089            throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2090        }
2091        
2092        getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0);
2093    }
2094    
2095    /**
2096     * Launch a solr index optimization.
2097     * @param workspaceName The workspace's name
2098     * @param solrClient The solr client to use
2099     * @throws SolrServerException if there is an error on the server
2100     * @throws IOException if there is a communication error with the server
2101     */
2102    public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2103    {
2104        UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName));
2105        int status = solrResponse.getStatus();
2106        
2107        if (status != 0)
2108        {
2109            throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2110        }
2111        
2112        getLogger().debug("Successful Solr optimize operation during an Ametys indexing process.");
2113    }
2114    
2115    /**
2116     * Delete all documents from the solr index.
2117     * @param workspaceName The workspace name
2118     * @param solrClient The solr client to use
2119     * @throws Exception if an error occurs while unindexing.
2120     */
2121    public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception
2122    {
2123        getLogger().debug("Deleting all documents from Solr.");
2124        
2125        String collection = _solrClientProvider.getCollectionName(workspaceName);
2126        solrClient.deleteByQuery(collection, "*:*");
2127        
2128        getLogger().debug("Successfully deleted all documents from Solr.");
2129    }
2130    
2131    /**
2132     * Delete a document from the Solr server.
2133     * @param id The id of the document to delete from Solr
2134     * @param workspaceName The workspace name
2135     * @param solrClient The solr client to use
2136     * @throws Exception if an error occurs while indexing.
2137     */
2138    protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception
2139    {
2140        UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id);
2141        int status = solrResponse.getStatus();
2142        
2143        if (status != 0)
2144        {
2145            throw new IOException("Deletion of document " + id + ": got status code '" + status + "'.");
2146        }
2147    }
2148    
2149    /**
2150     * Delete content attachments documents of a given content from the Solr server.
2151     * @param contentId The id of the content
2152     * @param workspaceName The workspace name
2153     * @param solrClient The solr client to use
2154     * @throws Exception if an error occurs while indexing.
2155     */
2156    protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception
2157    {
2158        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
2159        
2160        Query query = new ContentAttachmentQuery(contentId);
2161        UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build());
2162        int status = solrResponse.getStatus();
2163        
2164        if (status != 0)
2165        {
2166            throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'.");
2167        }
2168    }
2169    
2170//    /**
2171//     * Get the collection to use.
2172//     * @return The name of the collection to index into.
2173//     */
2174//    protected String getCollection()
2175//    {
2176//        return _solrCorePrefix + _workspaceSelector.getWorkspace();
2177//    }
2178    
2179    class IndexationResult
2180    {
2181        protected int _successCount;
2182        
2183        protected int _errorCount;
2184        
2185        /**
2186         * Constructor
2187         */
2188        public IndexationResult()
2189        {
2190            this(0, 0);
2191        }
2192        
2193        /**
2194         * Constructor
2195         * @param successCount The success count.
2196         * @param errorCount The error count.
2197         */
2198        public IndexationResult(int successCount, int errorCount)
2199        {
2200            this._successCount = successCount;
2201            this._errorCount = errorCount;
2202        }
2203        
2204        /**
2205         * Get the successCount.
2206         * @return the successCount
2207         */
2208        public int getSuccessCount()
2209        {
2210            return _successCount;
2211        }
2212        
2213        /**
2214         * Set the successCount.
2215         * @param successCount the successCount to set
2216         */
2217        public void setSuccessCount(int successCount)
2218        {
2219            this._successCount = successCount;
2220        }
2221        
2222        /**
2223         * Test if the indexation had errors.
2224         * @return true if the indexation had errors, false otherwise.
2225         */
2226        public boolean hasErrors()
2227        {
2228            return _errorCount > 0;
2229        }
2230        
2231        /**
2232         * Get the errorCount.
2233         * @return the errorCount
2234         */
2235        public int getErrorCount()
2236        {
2237            return _errorCount;
2238        }
2239        
2240        /**
2241         * Set the errorCount.
2242         * @param errorCount the errorCount to set
2243         */
2244        public void setErrorCount(int errorCount)
2245        {
2246            this._errorCount = errorCount;
2247        }
2248    }
2249    
2250}