001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.cms.content.indexing.solr;
017
018import java.io.IOException;
019import java.nio.ByteBuffer;
020import java.nio.CharBuffer;
021import java.nio.charset.CharsetDecoder;
022import java.nio.charset.CodingErrorAction;
023import java.nio.charset.StandardCharsets;
024import java.text.SimpleDateFormat;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collection;
028import java.util.Collections;
029import java.util.Comparator;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Objects;
035import java.util.Set;
036import java.util.concurrent.Future;
037import java.util.function.Function;
038import java.util.stream.Collectors;
039import java.util.stream.StreamSupport;
040
041import javax.jcr.RepositoryException;
042
043import org.apache.avalon.framework.activity.Initializable;
044import org.apache.avalon.framework.component.Component;
045import org.apache.avalon.framework.context.Context;
046import org.apache.avalon.framework.context.ContextException;
047import org.apache.avalon.framework.context.Contextualizable;
048import org.apache.avalon.framework.service.ServiceException;
049import org.apache.avalon.framework.service.ServiceManager;
050import org.apache.avalon.framework.service.Serviceable;
051import org.apache.cocoon.Constants;
052import org.apache.cocoon.components.ContextHelper;
053import org.apache.cocoon.environment.Request;
054import org.apache.commons.collections4.IterableUtils;
055import org.apache.commons.lang3.ObjectUtils;
056import org.apache.commons.lang3.StringUtils;
057import org.apache.solr.client.solrj.SolrClient;
058import org.apache.solr.client.solrj.SolrResponse;
059import org.apache.solr.client.solrj.SolrServerException;
060import org.apache.solr.client.solrj.request.CoreAdminRequest;
061import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
062import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
063import org.apache.solr.client.solrj.request.schema.SchemaRequest;
064import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update;
065import org.apache.solr.client.solrj.response.CoreAdminResponse;
066import org.apache.solr.client.solrj.response.SolrResponseBase;
067import org.apache.solr.client.solrj.response.UpdateResponse;
068import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
069import org.apache.solr.client.solrj.response.schema.SchemaResponse;
070import org.apache.solr.client.solrj.util.ClientUtils;
071import org.apache.solr.common.SolrInputDocument;
072import org.apache.solr.common.params.CoreAdminParams;
073import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
074import org.apache.solr.common.params.ModifiableSolrParams;
075import org.apache.solr.common.params.SolrParams;
076import org.apache.solr.common.util.NamedList;
077import org.slf4j.Logger;
078
079import org.ametys.cms.indexing.IndexingException;
080import org.ametys.cms.indexing.solr.AbstractIndexerCallable;
081import org.ametys.cms.indexing.solr.IndexationResult;
082import org.ametys.cms.indexing.solr.ReloadAclCacheRequest;
083import org.ametys.cms.indexing.solr.ThreadIndexerHelper;
084import org.ametys.cms.indexing.solr.UpdateAclCacheRequest;
085import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest;
086import org.ametys.cms.repository.Content;
087import org.ametys.cms.repository.ContentQueryHelper;
088import org.ametys.cms.repository.WorkflowAwareContent;
089import org.ametys.cms.rights.solrchecking.ReadAccessHelper;
090import org.ametys.cms.search.query.ContentAttachmentQuery;
091import org.ametys.cms.search.query.DocumentTypeQuery;
092import org.ametys.cms.search.query.OrQuery;
093import org.ametys.cms.search.query.Query;
094import org.ametys.cms.search.query.ResourceLocationQuery;
095import org.ametys.cms.search.solr.NoAutoCommitUpdateClient;
096import org.ametys.cms.search.solr.SolrClientProvider;
097import org.ametys.cms.search.solr.schema.SchemaDefinition;
098import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider;
099import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint;
100import org.ametys.cms.search.solr.schema.SchemaFields;
101import org.ametys.cms.search.solr.schema.SchemaHelper;
102import org.ametys.core.group.GroupIdentity;
103import org.ametys.core.right.AllowedUsers;
104import org.ametys.core.schedule.progression.ContainerProgressionTracker;
105import org.ametys.core.schedule.progression.ProgressionTrackerFactory;
106import org.ametys.core.schedule.progression.SimpleProgressionTracker;
107import org.ametys.core.user.UserIdentity;
108import org.ametys.core.util.DateUtils;
109import org.ametys.plugins.explorer.resources.Resource;
110import org.ametys.plugins.explorer.resources.ResourceCollection;
111import org.ametys.plugins.repository.AmetysObject;
112import org.ametys.plugins.repository.AmetysObjectIterable;
113import org.ametys.plugins.repository.AmetysObjectResolver;
114import org.ametys.plugins.repository.RepositoryConstants;
115import org.ametys.plugins.repository.TraversableAmetysObject;
116import org.ametys.plugins.repository.UnknownAmetysObjectException;
117import org.ametys.plugins.repository.collection.AmetysObjectCollection;
118import org.ametys.plugins.repository.provider.AbstractRepository;
119import org.ametys.plugins.repository.provider.JackrabbitRepository;
120import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector;
121import org.ametys.plugins.repository.provider.WorkspaceSelector;
122import org.ametys.runtime.config.Config;
123import org.ametys.runtime.i18n.I18nizableText;
124import org.ametys.runtime.plugin.component.AbstractLogEnabled;
125
126/**
127 * Solr indexer.
128 */
129public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable
130{
131    /** The component role. */
132    public static final String ROLE = SolrIndexer.class.getName();
133    
134    private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>();
135    
136    private static final String _CONFIGSET_NAME_PREFIX = "configset-";
137    
138    private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_");
139    private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general");
140    
141    private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766;
142    
143    /** The service manager. */
144    protected ServiceManager _manager;
145    /** The ametys object resolver. */
146    protected AmetysObjectResolver _resolver;
147    /** The schema definition provider extension point. */
148    protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP;
149    /** The schema helper. */
150    protected SchemaHelper _schemaHelper;
151    /** Solr Ametys contents indexer */
152    protected SolrContentIndexer _solrContentIndexer;
153    /** Solr workflow indexer. */
154    protected SolrWorkflowIndexer _solrWorkflowIndexer;
155    /** Solr resource indexer. */
156    protected SolrResourceIndexer _solrResourceIndexer;
157    
158    /** The Solr client provider */
159    protected SolrClientProvider _solrClientProvider;
160    
161    /** The solr core prefix. */
162    protected String _solrCorePrefix;
163    /** The Ametys internal URL used by Solr to query Ametys */
164    protected String _ametysInternalUrl;
165    
166    /** The workspace selector. */
167    protected WorkspaceSelector _workspaceSelector;
168    /** The JCR repository */
169    protected JackrabbitRepository _repository;
170    /** The helper for read access */
171    protected ReadAccessHelper _readAccessHelper;
172    /** The thread indexer helper */
173    protected ThreadIndexerHelper _threadIndexerHelper;
174    
175    /** The avalon context */
176    protected Context _context;
177    /** Cocoon Context */
178    protected org.apache.cocoon.environment.Context _cocoonContext;
179
180    /**
181     * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone
182     * @return The date format for indexing dates
183     * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead
184     */
185    @Deprecated
186    public static SimpleDateFormat dateFormat()
187    {
188        if (__DATE_FORMAT.get() == null)
189        {
190            __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
191        }
192        return __DATE_FORMAT.get();
193    }
194    
195    /**
196     * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr.
197     * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept.
198     * @param value The string value to index
199     * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log.
200     * @param documentId The id of the document being indexed. Can be null if you do not want to log.
201     * @param fieldName The name of the field being indexed. Can be null if you do not want to log.
202     * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes)
203     */
204    public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName)
205    {
206        if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT)
207        {
208            // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length())
209            // As a result, value.getBytes().length <= limit
210            return value;
211        }
212        
213        // There is a doubt, the string may need to be truncated (or not)
214        byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
215        int bytesLength = valueBytes.length;
216        if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT)
217        {
218            return value;
219        }
220        
221        if (ObjectUtils.allNotNull(logger, documentId, fieldName))
222        {
223            logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT);
224        }
225        
226        // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974)
227        CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT);
228        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
229                                                       .onMalformedInput(CodingErrorAction.IGNORE);
230        decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true);
231        decoder.flush(charBuffer);
232        return new String(charBuffer.array(), 0, charBuffer.position());
233    }
234    
235    @Override
236    public void service(ServiceManager serviceManager) throws ServiceException
237    {
238        _manager = serviceManager;
239        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
240        _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE);
241        _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE);
242        _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE);
243        _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE);
244        _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE);
245        _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE);
246        _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE);
247        _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE);
248        _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE);
249        _threadIndexerHelper = (ThreadIndexerHelper) serviceManager.lookup(ThreadIndexerHelper.ROLE);
250    }
251    
252    @Override
253    public void initialize() throws Exception
254    {
255        Config config = Config.getInstance();
256        _solrCorePrefix = config.getValue("cms.solr.core.prefix");
257        
258        _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url");
259        if (StringUtils.isBlank(_ametysInternalUrl))
260        {
261            // fallback to CMS URL as internal URL is an optional parameter
262            _ametysInternalUrl = config.getValue("cms.url");
263        }
264    }
265    
266    @Override
267    public void contextualize(Context context) throws ContextException
268    {
269        _context = context;
270        _cocoonContext = (org.apache.cocoon.environment.Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
271    }
272    
273    /**
274     * Gets the 'autocommit' Solr client
275     * @param workspaceName The name of the workspace
276     * @return the Solr client
277     */
278    protected SolrClient _getAutoCommitSolrClient(String workspaceName)
279    {
280        return _solrClientProvider.getUpdateClient(workspaceName, true);
281    }
282    
283    /**
284     * Gets the 'no autocommit' Solr client
285     * @param workspaceName The name of the workspace
286     * @return the Solr client
287     */
288    protected SolrClient _getNoAutoCommitSolrClient(String workspaceName)
289    {
290        return _solrClientProvider.getUpdateClient(workspaceName, false);
291    }
292    
293    // for admin operations
294    private SolrClient _defaultSolrClient()
295    {
296        return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE);
297    }
298    
299    /**
300     * Get the names of the Solr cores.
301     * @return The names of the Solr cores.
302     * @throws IOException If an I/O error occurs.
303     * @throws SolrServerException If a Solr error occurs.
304     */
305    public Set<String> getCoreNames() throws IOException, SolrServerException
306    {
307        Set<String> coreNames = new HashSet<>();
308        
309        getLogger().debug("Getting core list.");
310        
311        CoreAdminRequest req = new CoreAdminRequest();
312        req.setAction(CoreAdminAction.STATUS);
313        
314        NamedList<NamedList<Object>> status = req.process(_defaultSolrClient()).getCoreStatus();
315        for (Map.Entry<String, NamedList<Object>> core : status)
316        {
317            String fullName = (String) core.getValue().get("name");
318            if (fullName.startsWith(_solrCorePrefix))
319            {
320                coreNames.add(fullName.substring(_solrCorePrefix.length()));
321            }
322        }
323        
324        return coreNames;
325    }
326    
327    /**
328     * Get the names of the Solr cores.
329     * @return The names of the Solr cores.
330     * @throws IOException If an I/O error occurs.
331     * @throws SolrServerException If a Solr error occurs.
332     */
333    protected Set<String> getRealCoreNames() throws IOException, SolrServerException
334    {
335        Set<String> coreNames = new HashSet<>();
336        
337        getLogger().debug("Getting core list.");
338        
339        CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient());
340        
341        NamedList<NamedList<Object>> status = response.getCoreStatus();
342        for (Map.Entry<String, NamedList<Object>> core : status)
343        {
344            String fullName = (String) core.getValue().get("name");
345            if (fullName.startsWith(_solrCorePrefix))
346            {
347                coreNames.add(fullName.substring(_solrCorePrefix.length()));
348            }
349        }
350        
351        return coreNames;
352    }
353    
354    /**
355     * Create a Solr core.
356     * @param name The name of the core to create.
357     * @throws IOException If an I/O error occurs.
358     * @throws SolrServerException If a Solr error occurs.
359     */
360    public void createCore(String name) throws IOException, SolrServerException
361    {
362        Set<String> cores = getCoreNames();
363        if (!cores.contains(name))
364        {
365            String fullName = _solrCorePrefix + name;
366            String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix);
367            _createConfigset(configsetName);
368            
369            getLogger().info("Creating core '{}' (full name: '{}').", name, fullName);
370            
371            Create createRequest = new Create() {
372                @Override
373                public SolrParams getParams()
374                {
375                    ModifiableSolrParams params = new ModifiableSolrParams();
376                    
377                    params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString());
378                    params.set(CoreAdminParams.NAME, fullName);
379                    params.set(CoreAdminParams.CONFIGSET, configsetName);
380                    params.set("property.ametys.url", _ametysInternalUrl);
381                    
382                    return params;
383                }
384            };
385            
386            NamedList<?> results = createRequest.process(_defaultSolrClient()).getResponse();
387            
388            NamedList<?> error = (NamedList<?>) results.get("error");
389            if (error != null)
390            {
391                throw new IOException("Error creating the core: " + error.get("msg"));
392            }
393        }
394        else
395        {
396            if (getLogger().isDebugEnabled())
397            {
398                getLogger().debug("Core '" + name + "' already exists, skipping it.");
399            }
400        }
401    }
402    
403    /**
404     * Updates the ametys.url property of the Solr cores.
405     */
406    public void updateAmetysUrlCoreProperty()
407    {
408        Set<String> coreNames;
409        try
410        {
411            coreNames = getCoreNames();
412        }
413        catch (SolrServerException | IOException e)
414        {
415            getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e);
416            return;
417        }
418        
419        for (String coreName : coreNames)
420        {
421            String collection = _solrClientProvider.getCollectionName(coreName);
422            SolrResponseBase response;
423            try
424            {
425                response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection);
426            }
427            catch (SolrServerException | IOException e)
428            {
429                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e);
430                continue;
431            }
432            
433            NamedList<Object> responseParams = response.getResponse();
434            if ("ok".equals(responseParams.get("result")))
435            {
436                Boolean valueChanged = responseParams.getBooleanArg("valueChanged");
437                if (valueChanged)
438                {
439                    getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName);
440                }
441                else
442                {
443                    getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName);
444                }
445            }
446            else
447            {
448                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName);
449            }
450        }
451    }
452    
453    private void _createConfigset(String name) throws IOException, SolrServerException
454    {
455        getLogger().info("Creating (if necessary) configset '{}'", name);
456        
457        // This request handler will check if configset exists. If not it will be created.
458        CoreAdminRequest request = new CoreAdminRequest() {
459            @Override
460            public SolrParams getParams()
461            {
462                ModifiableSolrParams params = new ModifiableSolrParams();
463                params.set(CoreAdminParams.ACTION, "createConfigset");
464                params.set(CoreAdminParams.NAME, name);
465                return params;
466            }
467        };
468        
469        NamedList<?> results = request.process(_defaultSolrClient()).getResponse();
470        
471        NamedList<?> error = (NamedList<?>) results.get("error");
472        if (error != null)
473        {
474            throw new IOException("Error creating the core: " + error.get("msg"));
475        }
476    }
477    
478    /**
479     * Delete a Solr core.
480     * @param name The name of the core to delete.
481     * @throws IOException If an I/O error occurs.
482     * @throws SolrServerException If a Solr error occurs.
483     */
484    public void deleteCore(String name) throws IOException, SolrServerException
485    {
486        String fullName = _solrCorePrefix + name;
487        
488        getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName);
489        
490        CoreAdminResponse response = CoreAdminRequest.unloadCore(fullName, true, true, _defaultSolrClient());
491        NamedList<?> results = response.getResponse();
492        
493        NamedList<?> error = (NamedList<?>) results.get("error");
494        if (error != null)
495        {
496            throw new IOException("Error deleting core" + name + ": " + error.get("msg"));
497        }
498    }
499    
500    /**
501     * Send the schema.
502     * @throws IOException If a communication error occurs.
503     * @throws SolrServerException If a solr error occurs.
504     */
505    public void sendSchema() throws IOException, SolrServerException
506    {
507        getLogger().info("Computing and sending the schema to the solr server.");
508        
509        String workspaceName = _workspaceSelector.getWorkspace();
510        String collection = _solrClientProvider.getCollectionName(workspaceName);
511        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
512        
513//        SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema();
514        SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml");
515        
516        // TODO Clear the schema except fields marked ametysReadOnly="true".
517        
518        // Clear the current schema.
519        clearSchema(solrClient, collection);
520        
521        SchemaRequest schemaRequest = new SchemaRequest();
522        SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
523        
524        // The cleared schema contains only the basic fields which can't be deleted.
525        SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation();
526        SchemaFields schemaFields = new SchemaFields(clearedSchema);
527        
528        getLogger().debug("Schema after clear: \n{}", schemaFields.toString());
529        
530        // Add the static schema types and fields.
531        List<SchemaRequest.Update> updates = new ArrayList<>();
532        
533        // Set "add field" definitions from the static schema to the update list.
534        addStaticSchemaUpdates(updates, staticSchema, schemaFields);
535        
536        getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString());
537        
538        // Set "add field" definitions from the static schema to the update list.
539        addCustomUpdates(updates, schemaFields);
540        
541        updates.sort(new SchemaRequestComparator());
542        
543        SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates);
544        SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
545        
546        getLogger().debug("Send schema response: {}", updateResponse.toString());
547        Object errors = updateResponse.getResponse().get("errors");
548        if (errors != null && errors instanceof List && !((List) errors).isEmpty())
549        {
550            String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString();
551            throw new SolrServerException(msg);
552        }
553    
554        getLogger().info("Schema sent to the solr server.");
555        
556        reloadCores();
557    }
558    
559    /**
560     * Compute the list of {@link Update} directives from the static schema.
561     * @param updates The list of {@link Update} directives to fill.
562     * @param staticSchema The static schema representation.
563     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
564     */
565    protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields)
566    {
567        List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes();
568        for (FieldTypeDefinition fieldType : fieldTypes)
569        {
570            String name = (String) fieldType.getAttributes().get("name");
571            if (!schemaFields.hasFieldType(name))
572            {
573                updates.add(new SchemaRequest.AddFieldType(fieldType));
574                schemaFields.addFieldType(name);
575            }
576        }
577        for (Map<String, Object> field : staticSchema.getFields())
578        {
579            String name = (String) field.get("name");
580            if (!schemaFields.hasField(name))
581            {
582                updates.add(new SchemaRequest.AddField(field));
583                schemaFields.addField(name);
584            }
585        }
586        for (Map<String, Object> field : staticSchema.getDynamicFields())
587        {
588            String name = (String) field.get("name");
589            if (!schemaFields.hasDynamicField(name))
590            {
591                updates.add(new SchemaRequest.AddDynamicField(field));
592                schemaFields.addDynamicField(name);
593            }
594        }
595        for (Map<String, Object> field : staticSchema.getCopyFields())
596        {
597            String source = (String) field.get("source");
598            String dest = (String) field.get("dest");
599            if (!schemaFields.hasCopyField(source, dest))
600            {
601                updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest)));
602                schemaFields.addCopyField(source, dest);
603            }
604        }
605    }
606    
607    /**
608     * Compute the list of custom {@link Update} directives.
609     * @param updates The list of {@link Update} directives to fill.
610     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
611     */
612    protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields)
613    {
614        // Add all our property-managed fields.
615        for (String providerId : _schemaDefProviderEP.getExtensionsIds())
616        {
617            SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId);
618            
619            for (SchemaDefinition definition : definitionProvider.getDefinitions())
620            {
621                if (!definition.exists(schemaFields))
622                {
623                    SchemaRequest.Update update = definition.getSchemaUpdate();
624                    if (update != null)
625                    {
626                        updates.add(update);
627                    }
628                }
629            }
630        }
631    }
632    
633    /**
634     * Delete all the fields of the existing schema in the given collection.
635     * @param solrClient The Solr client
636     * @param collection The collection.
637     * @throws IOException If a communication error occurs.
638     * @throws SolrServerException If a solr error occurs.
639     */
640    protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException
641    {
642        try
643        {
644            getLogger().info("Clearing the existing schema on the solr server.");
645            
646            SchemaRequest schemaRequest = new SchemaRequest();
647            SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
648            
649            SchemaRepresentation schema = schemaResponse.getSchemaRepresentation();
650            
651            List<SchemaRequest.Update> deletions = new ArrayList<>();
652            
653            // First the copy fields, then dynamic and simple fields, and field types in the end.
654            for (Map<String, Object> field : schema.getCopyFields())
655            {
656                String source = (String) field.get("source");
657                String dest = (String) field.get("dest");
658                deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest)));
659            }
660            for (Map<String, Object> field : schema.getDynamicFields())
661            {
662                String name = (String) field.get("name");
663                deletions.add(new SchemaRequest.DeleteDynamicField(name));
664            }
665            for (Map<String, Object> field : schema.getFields())
666            {
667                String name = (String) field.get("name");
668                if (!_READ_ONLY_FIELDS.contains(name))
669                {
670                    deletions.add(new SchemaRequest.DeleteField(name));
671                }
672            }
673            for (FieldTypeDefinition fieldType : schema.getFieldTypes())
674            {
675                String name = (String) fieldType.getAttributes().get("name");
676                if (!_READ_ONLY_FIELDTYPES.contains(name))
677                {
678                    deletions.add(new SchemaRequest.DeleteFieldType(name));
679                }
680            }
681            
682            SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions);
683            SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
684            
685            Object errors = updateResponse.getResponse().get("errors");
686            if (errors != null && errors instanceof List && !((List) errors).isEmpty())
687            {
688                String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString();
689                throw new SolrServerException(msg);
690            }
691            else
692            {
693                getLogger().debug("Clear schema response: {}", updateResponse.toString());
694            }
695            
696            getLogger().info("Solr schema cleared.");
697        }
698        catch (SolrServerException | IOException e)
699        {
700            getLogger().error("Error clearing schema in collection " + collection, e);
701            throw e;
702        }
703    }
704    
705    /**
706     * Reload the solr cores.
707     * @throws IOException If a communication error occurs.
708     * @throws SolrServerException If a solr error occurs.
709     */
710    protected void reloadCores() throws IOException, SolrServerException
711    {
712        getLogger().info("Reloading solr cores.");
713        
714        for (String coreName : getCoreNames())
715        {
716            String fullName = _solrCorePrefix + coreName;
717            
718            CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient());
719            
720            getLogger().debug("Reload core response: {}", reloadResponse.toString());
721        }
722        
723        getLogger().info("All cores reloaded.");
724    }
725    
726    /**
727     * Reloads the ACL Solr cache for all users
728     * @throws IOException If an I/O error occurs.
729     * @throws SolrServerException If a Solr error occurs.
730     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
731     */
732    public void reloadAclCache() throws IOException, SolrServerException, RepositoryException
733    {
734        String[] workspaceNames = _repository.getWorkspaces();
735        for (String workspaceName : workspaceNames)
736        {
737            reloadAclCache(workspaceName);
738        }
739    }
740    
741    /**
742     * Reloads the ACL Solr cache for all users
743     * @param workspaceName The workspace name
744     * @throws IOException If an I/O error occurs.
745     * @throws SolrServerException If a Solr error occurs.
746     */
747    public void reloadAclCache(String workspaceName) throws IOException, SolrServerException
748    {
749        reloadAclCache(workspaceName, false);
750    }
751    
752    /**
753     * Reloads the ACL Solr cache for all users
754     * @param workspaceName The workspace name
755     * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache)
756     * @throws IOException If an I/O error occurs.
757     * @throws SolrServerException If a Solr error occurs.
758     */
759    public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException
760    {
761        getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
762        
763        String collection = _solrClientProvider.getCollectionName(workspaceName);
764        SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection);
765        NamedList<Object> responseObj = responseBase.getResponse();
766        
767        if ("ok".equals(responseObj.get("result")))
768        {
769            getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
770        }
771        else
772        {
773            Object error = responseObj.get("error");
774            getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error);
775        }
776    }
777    
778    /**
779     * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces.
780     * @param objects the {@link AmetysObject}s to update.
781     * @throws IOException If an I/O error occurs.
782     * @throws SolrServerException If a Solr error occurs.
783     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
784     */
785    public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException
786    {
787        String[] workspaceNames = _repository.getWorkspaces();
788        for (String workspaceName : workspaceNames)
789        {
790            updateAclCache(objects, workspaceName);
791        }
792    }
793    
794    /**
795     * Updates the ACL Solr cache for some {@link AmetysObject}s.
796     * @param objects the {@link AmetysObject}s to update.
797     * @param workspaceName The workspace name
798     * @throws IOException If an I/O error occurs.
799     * @throws SolrServerException If a Solr error occurs.
800     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
801     */
802    public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException
803    {
804        Map<String, Map<String, Object>> solrParams = new HashMap<>();
805        
806        for (AmetysObject object : objects)
807        {
808            AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object);
809            
810            solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(),
811                                                  "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(),
812                                                  "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
813                                                  "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
814                                                  "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()),
815                                                  "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList())));
816        }
817        
818        String collection = _solrClientProvider.getCollectionName(workspaceName);
819        SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection);
820        NamedList<Object> responseObj = response.getResponse();
821        
822        if ("ok".equals(responseObj.get("result")))
823        {
824            if (getLogger().isInfoEnabled())
825            {
826                getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet());
827            }
828        }
829        else
830        {
831            @SuppressWarnings("unchecked")
832            List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects");
833            getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects);
834        }
835    }
836    
837    /**
838     * Index all the contents in a given workspace.
839     * @param workspaceName the workspace where to index
840     * @param indexAttachments to index content attachments
841     * @param solrClient The solr client to use
842     * @return The indexation result as a Map.
843     * @throws Exception if an error occurs while indexing.
844     */
845    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
846    {
847        return indexAllContents(workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index all contents", getLogger()));
848    }
849    
850    /**
851     * Index all the contents in a given workspace.
852     * @param workspaceName the workspace where to index
853     * @param indexAttachments to index content attachments
854     * @param solrClient The solr client to use
855     * @param progressionTracker The progression of the indexation
856     * @return The indexation result as a Map.
857     * @throws Exception if an error occurs while indexing.
858     */
859    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
860    {
861        Request request = ContextHelper.getRequest(_context);
862        
863        // Retrieve the current workspace.
864        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
865        
866        try
867        {
868            // Force the workspace.
869            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
870            
871            getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName);
872            
873            long start = System.currentTimeMillis();
874            
875            // Delete all contents
876            unindexAllContents(workspaceName, indexAttachments, solrClient);
877
878            String query = ContentQueryHelper.getContentXPathQuery(null);
879            AmetysObjectIterable<Content> contents = _resolver.query(query);
880            
881            IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient, progressionTracker);
882            
883            long end = System.currentTimeMillis();
884            
885            if (!result.hasErrors())
886            {
887                getLogger().info("{} contents indexed without error in {} milliseconds.", result.successCount(), end - start);
888            }
889            else
890            {
891                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount());
892            }
893            
894            Map<String, Object> results = new HashMap<>();
895            results.put("successCount", result.successCount());
896            if (result.hasErrors())
897            {
898                results.put("errorCount", result.errorCount());
899            }
900            
901            return results;
902        }
903        catch (Exception e)
904        {
905            String error = String.format("Failed to index all contents in workspace %s", workspaceName);
906            getLogger().error(error, e);
907            throw new IndexingException(error, e);
908        }
909        finally
910        {
911            // Restore context
912            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
913        }
914    }
915    /**
916     * Unindex all content documents.
917     * @param workspaceName The workspace name
918     * @param unindexAttachments also unindex content attachments
919     * @param solrClient The solr client to use
920     * @throws Exception if an error occurs while unindexing.
921     */
922    protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
923    {
924        String collection = _solrClientProvider.getCollectionName(workspaceName);
925        
926        Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT);
927        Query query;
928        if (unindexAttachments)
929        {
930            Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE);
931            Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE);
932            query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents);
933        }
934        else
935        {
936            query = contents;
937        }
938        solrClient.deleteByQuery(collection, query.build());
939    }
940    
941    /**
942     * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit
943     * @param collectionId The id of collection
944     * @param indexAttachments to index content attachments
945     * @throws Exception if an error occurs while indexing.
946     */
947    public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception
948    {
949        String[] workspaceNames = _repository.getWorkspaces();
950        for (String workspaceName : workspaceNames)
951        {
952            indexSubcontents(collectionId, workspaceName, indexAttachments);
953        }
954    }
955    
956    /**
957     * Index the child contents of a {@link AmetysObjectCollection}
958     * @param collectionId The id of collection
959     * @param workspaceName the workspace where to index
960     * @param indexAttachments to index content attachments
961     * @throws Exception if an error occurs while unindexing.
962     */
963    public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception
964    {
965        Request request = ContextHelper.getRequest(_context);
966        
967        // Retrieve the current workspace.
968        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
969        
970        try
971        {
972            // Force the workspace.
973            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
974            
975            if (_resolver.hasAmetysObjectForId(collectionId))
976            {
977                SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
978                AmetysObjectCollection collection = _resolver.resolveById(collectionId);
979                AmetysObjectIterable<AmetysObject> children = collection.getChildren();
980                
981                for (AmetysObject child : children)
982                {
983                    if (child instanceof Content)
984                    {
985                        Content content = (Content) child;
986                        
987                        _doIndexContent(content, workspaceName, indexAttachments, solrClient);
988                    }
989                }
990            }
991        }
992        finally
993        {
994            // Restore context
995            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
996        }
997    }
998    
999    /**
1000     * Add or update a content into Solr index on all workspaces and commit
1001     * @param contentId The id of the content to index
1002     * @param indexAttachments to index content attachments
1003     * @throws Exception if an error occurs while indexing.
1004     */
1005    public void indexContent(String contentId, boolean indexAttachments) throws Exception
1006    {
1007        String[] workspaceNames = _repository.getWorkspaces();
1008        for (String workspaceName : workspaceNames)
1009        {
1010            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1011            indexContent(contentId, workspaceName, indexAttachments, solrClient);
1012        }
1013    }
1014    
1015    /**
1016     * Add or update a content into Solr index
1017     * @param contentId The id of the content to index
1018     * @param workspaceName the workspace where to index
1019     * @param indexAttachments to index content attachments
1020     * @throws Exception if an error occurs while indexing.
1021     */
1022    public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception
1023    {
1024        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1025        indexContent(contentId, workspaceName, indexAttachments, solrClient);
1026    }
1027    
1028    /**
1029     * Add or update a content into Solr index
1030     * @param contentId The id of the content to index
1031     * @param workspaceName the workspace where to index
1032     * @param indexAttachments to index content attachments
1033     * @param solrClient The solr client to use
1034     * @throws Exception if an error occurs while indexing.
1035     */
1036    public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1037    {
1038        Request request = ContextHelper.getRequest(_context);
1039        
1040        // Retrieve the current workspace.
1041        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1042        
1043        try
1044        {
1045            // Force the workspace.
1046            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1047            
1048            if (_resolver.hasAmetysObjectForId(contentId))
1049            {
1050                Content content = _resolver.resolveById(contentId);
1051                _doIndexContent(content, workspaceName, indexAttachments, solrClient);
1052            }
1053        }
1054        finally
1055        {
1056            // Restore context
1057            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1058        }
1059    }
1060    
1061    private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException
1062    {
1063        try
1064        {
1065            long time_0 = System.currentTimeMillis();
1066            
1067            getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName);
1068            
1069            deleteRepeaterDocs(content.getId(), workspaceName, solrClient);
1070            doIndexContent(content, workspaceName, solrClient);
1071            doIndexContentWorkflow(content, workspaceName, solrClient);
1072            if (indexAttachments)
1073            {
1074                indexContentAttachments(content.getRootAttachments(), content, solrClient);
1075            }
1076            
1077            getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0);
1078        }
1079        catch (Exception e)
1080        {
1081            String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName);
1082            getLogger().error(error, e);
1083            throw new IndexingException(error, e);
1084        }
1085    }
1086    
1087    /**
1088     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1089     * @param contents the collection of contents to index.
1090     * @return the indexation result.
1091     * @throws Exception if an error occurs while indexing.
1092     */
1093    public IndexationResult indexContents(Iterable<Content> contents) throws Exception
1094    {
1095        return indexContents(contents, ProgressionTrackerFactory.createContainerProgressionTracker("Index contents", getLogger()));
1096    }
1097    
1098    
1099    /**
1100     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1101     * @param contents the collection of contents to index.
1102     * @param progressionTracker The progression of the indexation
1103     * @return the indexation result.
1104     * @throws Exception if an error occurs while indexing.
1105     */
1106    public IndexationResult indexContents(Iterable<Content> contents, ContainerProgressionTracker progressionTracker) throws Exception
1107    {
1108        IndexationResult result = new IndexationResult(0, 0);
1109        
1110        String[] workspaceNames = _repository.getWorkspaces();
1111        
1112        for (String workspaceName : workspaceNames)
1113        {
1114            progressionTracker.addSimpleStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_WORKSPACE_STEP_LABEL", List.of(workspaceName)));
1115        }
1116        
1117        for (String workspaceName : workspaceNames)
1118        {
1119            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1120            IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient, progressionTracker.getStep(workspaceName));
1121            
1122            result = new IndexationResult(result.successCount() + wResult.successCount(), result.errorCount() + wResult.errorCount());
1123        }
1124        
1125        return result;
1126    }
1127    
1128    /**
1129     * Send a collection of contents for indexation in the solr server.
1130     * @param contents the collection of contents to index.
1131     * @param workspaceName the workspace where to index
1132     * @param indexAttachments to index content attachments
1133     * @param solrClient The solr client to use
1134     * @return the indexation result.
1135     * @throws Exception if an error occurs while indexing.
1136     */
1137    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1138    {
1139        return indexContents(contents, workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index contents for workspace " + workspaceName, getLogger()));
1140    }
1141    
1142    /**
1143     * Send a collection of contents for indexation in the solr server.
1144     * @param contents the collection of contents to index.
1145     * @param workspaceName the workspace where to index
1146     * @param indexAttachments to index content attachments
1147     * @param solrClient The solr client to use
1148     * @param progressionTracker The progression of the indexation
1149     * @return the indexation result.
1150     * @throws Exception if an error occurs while indexing.
1151     */
1152    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1153    {
1154        Request request = ContextHelper.getRequest(_context);
1155        
1156        // Retrieve the current workspace.
1157        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1158        
1159        try
1160        {
1161            // Force the workspace.
1162            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1163            
1164            getLogger().info("Starting indexation of several contents for workspace {}", workspaceName);
1165            
1166            long start = System.currentTimeMillis();
1167            
1168            List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false)
1169                .map(Content::getId)
1170                .map(this::_resolveSilently)
1171                .filter(Objects::nonNull)
1172                .collect(Collectors.toList());
1173            
1174            IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient, progressionTracker);
1175            
1176            long end = System.currentTimeMillis();
1177            
1178            if (!result.hasErrors())
1179            {
1180                getLogger().info("{} contents indexed without error in {} milliseconds.", result.successCount(), end - start);
1181            }
1182            else
1183            {
1184                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount());
1185            }
1186            
1187            return result;
1188        }
1189        catch (Exception e)
1190        {
1191            String error = String.format("Failed to index several contents in workspace %s", workspaceName);
1192            getLogger().error(error, e);
1193            throw new IndexingException(error, e);
1194        }
1195        finally
1196        {
1197            // Restore context
1198            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1199        }
1200    }
1201    
1202    private Content _resolveSilently(String contentId)
1203    {
1204        try
1205        {
1206            return _resolver.resolveById(contentId);
1207        }
1208        catch (UnknownAmetysObjectException e)
1209        {
1210            return null;
1211        }
1212    }
1213    
1214    /**
1215     * Send some contents for indexation in the solr server.
1216     * @param contents the contents to index.
1217     * @param workspaceName The workspace name
1218     * @param indexAttachments to index content attachments
1219     * @param solrClient The solr client to use
1220     * @param progressionTracker The progression of the indexation
1221     * @return the indexation result.
1222     * @throws Exception if an error occurs committing the results.
1223     */
1224    protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1225    {
1226        int numberOfContents = IterableUtils.size(contents);
1227        progressionTracker.setSize(numberOfContents);
1228        
1229        // Add callable for each content to index
1230        List<Future<Void>> tasks = new ArrayList<>();
1231        for (Content content : contents)
1232        {
1233            tasks.add(_threadIndexerHelper.submitCallable(new ContentIndexerCallable(content, workspaceName, indexAttachments, solrClient, progressionTracker)));
1234        }
1235        
1236        // Now that everything is submitted, we can iterate and wait for result
1237        return IndexationResult.fromTasks(tasks, getLogger());
1238    }
1239    
1240    /**
1241     * Update the value of a specific system property in a content document.
1242     * @param content The content to update.
1243     * @param propertyId The system property ID.
1244     * @param workspaceName The workspace name
1245     * @throws Exception if an error occurs while indexing.
1246     */
1247    public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception
1248    {
1249        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1250        updateSystemProperty(content, propertyId, workspaceName, solrClient);
1251    }
1252    
1253    /**
1254     * Update the value of a specific system property in a content document.
1255     * @param content The content to update.
1256     * @param propertyId The system property ID.
1257     * @param workspaceName The workspace name
1258     * @param solrClient The solr client to use
1259     * @throws Exception if an error occurs while indexing.
1260     */
1261    public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1262    {
1263        getLogger().debug("Updating the system property '{}' for content {} into Solr.", propertyId, content);
1264        
1265        SolrInputDocument document = new SolrInputDocument();
1266        boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document);
1267        
1268        if (!hasUpdate)
1269        {
1270            getLogger().debug("Did not index '{}' system property for content {} in Solr because no update to apply.", propertyId, content);
1271            return;
1272        }
1273        
1274        int status = _pushSolrDocument(document, workspaceName, solrClient);
1275        if (status != 0)
1276        {
1277            throw new IOException("Indexing of system property '" + propertyId + "': got status code '" + status + "'.");
1278        }
1279        
1280        getLogger().debug("Succesfully indexed '{}' system property for content {} in Solr.", propertyId, content);
1281    }
1282    
1283    /**
1284     * Update the value of a specific property in a content document.
1285     * @param content The content to update.
1286     * @param propertyId The system property ID.
1287     * @param workspaceName The workspace name
1288     * @throws Exception if an error occurs while indexing.
1289     */
1290    public void updateProperty(Content content, String propertyId, String workspaceName) throws Exception
1291    {
1292        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1293        updateProperty(content, propertyId, workspaceName, solrClient);
1294    }
1295    
1296    /**
1297     * Update the value of a specific property in a content document.
1298     * @param content The content to update.
1299     * @param propertyId The system property ID.
1300     * @param workspaceName The workspace name
1301     * @param solrClient The solr client to use
1302     * @throws Exception if an error occurs while indexing.
1303     */
1304    public void updateProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1305    {
1306        getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content);
1307        
1308        SolrInputDocument document = new SolrInputDocument();
1309        boolean hasUpdate = _solrContentIndexer.indexPartialProperty(content, propertyId, document);
1310        
1311        if (!hasUpdate)
1312        {
1313            getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content);
1314            return;
1315        }
1316        
1317        int status = _pushSolrDocument(document, workspaceName, solrClient);
1318        if (status != 0)
1319        {
1320            throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'.");
1321        }
1322        
1323        getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content);
1324    }
1325    
1326    private int _pushSolrDocument(SolrInputDocument document, String workspaceName, SolrClient solrClient) throws Exception
1327    {
1328        String collection = _solrClientProvider.getCollectionName(workspaceName);
1329        UpdateResponse solrResponse = solrClient.add(collection, document);
1330        return solrResponse.getStatus();
1331    }
1332    
1333    /**
1334     * Remove a content from Solr index for all workspaces and commit
1335     * @param contentId The id of content to unindex
1336     * @param unindexAttachments also unindex content attachments
1337     * @throws Exception if an error occurs while indexing.
1338     */
1339    public void unindexContent(String contentId, boolean unindexAttachments) throws Exception
1340    {
1341        String[] workspaceNames = _repository.getWorkspaces();
1342        for (String workspaceName : workspaceNames)
1343        {
1344            unindexContent(contentId, workspaceName, unindexAttachments);
1345        }
1346    }
1347    
1348    /**
1349     * Remove a content from Solr index
1350     * @param contentId The id of content to unindex
1351     * @param workspaceName The workspace where to work in
1352     * @param unindexAttachments also unindex content attachments
1353     * @throws Exception if an error occurs while indexing.
1354     */
1355    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception
1356    {
1357        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1358        unindexContent(contentId, workspaceName, unindexAttachments, solrClient);
1359    }
1360    
1361    /**
1362     * Remove a content from Solr index
1363     * @param contentId The id of content to unindex
1364     * @param workspaceName The workspace where to work in
1365     * @param unindexAttachments also unindex content attachments
1366     * @param solrClient The solr client to use
1367     * @throws Exception if an error occurs while indexing.
1368     */
1369    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
1370    {
1371        Request request = ContextHelper.getRequest(_context);
1372        
1373        // Retrieve the current workspace.
1374        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1375        
1376        try
1377        {
1378            // Force the workspace.
1379            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1380            
1381            getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName);
1382            
1383            deleteRepeaterDocs(contentId, workspaceName, solrClient);
1384            doUnindexDocument(contentId, workspaceName, solrClient);
1385            if (unindexAttachments)
1386            {
1387                doUnindexContentAttachments(contentId, workspaceName, solrClient);
1388            }
1389            _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1390            
1391            getLogger().debug("Succesfully deleted content {} from Solr.", contentId);
1392        }
1393        catch (Exception e)
1394        {
1395            String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName);
1396            getLogger().error(error, e);
1397            throw new IndexingException(error, e);
1398        }
1399        finally
1400        {
1401            // Restore workspace
1402            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1403        }
1404    }
1405    
1406    /**
1407     * Remove a content from Solr index for all workspaces and commit
1408     * @param contentIds The id of content to unindex
1409     * @throws Exception if an error occurs while indexing.
1410     */
1411    public void unindexContents(Collection<String> contentIds) throws Exception
1412    {
1413        String[] workspaceNames = _repository.getWorkspaces();
1414        for (String workspaceName : workspaceNames)
1415        {
1416            unindexContents(contentIds, workspaceName);
1417        }
1418    }
1419    
1420    /**
1421     * Remove a content from Solr index
1422     * @param contentIds The id of content to unindex
1423     * @param workspaceName The workspace where to work in
1424     * @throws Exception if an error occurs while indexing.
1425     */
1426    public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception
1427    {
1428        Request request = ContextHelper.getRequest(_context);
1429        
1430        // Retrieve the current workspace.
1431        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1432        
1433        try
1434        {
1435            // Force the workspace.
1436            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1437            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1438            
1439            getLogger().debug("Unindexing several contents from Solr.");
1440            
1441            for (String contentId : contentIds)
1442            {
1443                deleteRepeaterDocs(contentId, workspaceName, solrClient);
1444                doUnindexDocument(contentId, workspaceName, solrClient);
1445                _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1446            }
1447            
1448            getLogger().debug("Succesfully unindexed content from Solr.");
1449        }
1450        catch (Exception e)
1451        {
1452            String error = String.format("Failed to unindex several contents in workspace %s", workspaceName);
1453            getLogger().error(error, e);
1454            throw new IndexingException(error, e);
1455        }
1456        finally
1457        {
1458            // Restore workspace
1459            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1460        }
1461    }
1462    
1463    /**
1464     * Add or update a content into Solr index
1465     * @param content The content to index
1466     * @param workspaceName The workspace where to index
1467     * @param solrClient The solr client to use
1468     * @throws Exception if an error occurs while indexing.
1469     */
1470    protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception
1471    {
1472        long time_0 = System.currentTimeMillis();
1473        
1474        SolrInputDocument document = new SolrInputDocument();
1475        List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document);
1476        
1477        long time_1 = System.currentTimeMillis();
1478        getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0);
1479        
1480        indexAclInitValues(content, document);
1481        
1482        long time_2 = System.currentTimeMillis();
1483        getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1);
1484        
1485        List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments);
1486        documents.add(document);
1487        
1488        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents);
1489        int status = solrResponse.getStatus();
1490        
1491        long time_3 = System.currentTimeMillis();
1492        getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2);
1493        
1494        if (status != 0)
1495        {
1496            throw new IOException("Content indexation: got status code '" + status + "'.");
1497        }
1498    }
1499    
1500    /**
1501     * Indexes read-ACl initial values for object
1502     * @param ametysObject The object
1503     * @param document The Solr document
1504     */
1505    public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document)
1506    {
1507        // Indexation of AmetysObject property
1508        document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true);
1509        
1510        // Indexation of initValues for AllowedUsers
1511        AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject);
1512        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed());
1513        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed());
1514        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString);
1515        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString);
1516        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString);
1517        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString);
1518    }
1519    
1520    private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier)
1521    {
1522        if (values == null)
1523        {
1524            return;
1525        }
1526        
1527        for (T value : values)
1528        {
1529            document.addField(fieldName, stringifier.apply(value));
1530        }
1531    }
1532    
1533    /**
1534     * Index the whole workflow of a content.
1535     * @param content The content.
1536     * @param workspaceName The workspace name
1537     * @param solrClient The solr client to use
1538     * @throws Exception if an error occurs while indexing.
1539     */
1540    protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception
1541    {
1542        if (content instanceof WorkflowAwareContent)
1543        {
1544            _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient);
1545        }
1546    }
1547    
1548    /**
1549     * Index content attachments as new entries in the idnex
1550     * @param collection the collection of attachments
1551     * @param content the content whose attachments will be indexed
1552     * @throws Exception if something goes wrong when indexing the attachments of the content
1553     */
1554    public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception
1555    {
1556        Request request = ContextHelper.getRequest(_context);
1557        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1558        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1559        indexContentAttachments(collection, content, solrClient);
1560    }
1561    
1562    /**
1563     * Index content attachments as new entries in the idnex
1564     * @param collection the collection of attachments
1565     * @param content the content whose attachments will be indexed
1566     * @param solrClient The solr client to use
1567     * @throws Exception if something goes wrong when indexing the attachments of the content
1568     */
1569    public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception
1570    {
1571        if (collection == null)
1572        {
1573            return;
1574        }
1575        
1576        try (AmetysObjectIterable<AmetysObject> children = collection.getChildren())
1577        {
1578            for (AmetysObject object : children)
1579            {
1580                if (object instanceof ResourceCollection)
1581                {
1582                    indexContentAttachments((ResourceCollection) object, content, solrClient);
1583                }
1584                else if (object instanceof Resource)
1585                {
1586                    Resource resource = (Resource) object;
1587                    indexContentAttachment(resource, content, solrClient);
1588                }
1589            }
1590        }
1591    }
1592    
1593    /**
1594     * Index a content attachment
1595     * @param resource the content attachment as a {@link Resource}
1596     * @param content the content whose attachment is going to be indexed
1597     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1598     */
1599    public void indexContentAttachment(Resource resource, Content content) throws Exception
1600    {
1601        Request request = ContextHelper.getRequest(_context);
1602        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1603        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1604        indexContentAttachment(resource, content, solrClient);
1605    }
1606    
1607    /**
1608     * Index a content attachment
1609     * @param resource the content attachment as a {@link Resource}
1610     * @param content the content whose attachment is going to be indexed
1611     * @param solrClient The solr client to use
1612     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1613     */
1614    public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception
1615    {
1616        SolrInputDocument document = new SolrInputDocument();
1617        
1618        // Prepare resource doc
1619        _indexContentAttachment(resource, document, content);
1620        
1621        // Indexation of the document
1622        _indexResourceDocument(resource, document, solrClient);
1623    }
1624    
1625    private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception
1626    {
1627        String language = content.getLanguage();
1628        
1629        _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language);
1630        
1631        // Need the id of the content for unindexing attachment during the unindexing of the content
1632        document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId());
1633    }
1634    
1635    /**
1636     * Index a populated solr input document of type Resource.
1637     * @param resource the resource from which the input document is created
1638     * @param document the input document
1639     * @param solrClient The solr client to use
1640     * @throws SolrServerException if there is an error on the server
1641     * @throws IOException if there is a communication error with the server
1642     */
1643    protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException
1644    {
1645        String resourceId = resource.getId();
1646        
1647        // Retrieve appropriate collection name
1648        Request request = ContextHelper.getRequest(_context);
1649        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1650        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
1651        
1652        // Add document
1653        UpdateResponse solrResponse = solrClient.add(collectionName, document);
1654        int status = solrResponse.getStatus();
1655        
1656        if (status != 0)
1657        {
1658            throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId);
1659        }
1660        
1661        getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId);
1662    }
1663    
1664    /**
1665     * Delete repeater documents of a specified content.
1666     * @param workspaceName The workspace name
1667     * @param contentId the content ID.
1668     * @param solrClient The solr client to use
1669     * @throws Exception if an error occurs while indexing.
1670     */
1671    protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception
1672    {
1673        long time_0 = System.currentTimeMillis();
1674        
1675        // _documentType:repeater AND id:content\://xxx/*
1676        StringBuilder query = new StringBuilder();
1677        query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER)
1678             .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*");
1679        
1680        solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString());
1681        
1682        getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0);
1683    }
1684    
1685    /**
1686     * Index all the resources in a given workspace.
1687     * @param workspaceName The workspace where to index
1688     * @param solrClient The solr client to use
1689     * @param progressionTracker The progression of the indexation
1690     * @return The indexation result as a Map.
1691     * @throws Exception if an error occurs while indexing.
1692     */
1693    public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1694    {
1695        Request request = ContextHelper.getRequest(_context);
1696        
1697        // Retrieve the current workspace.
1698        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1699        
1700        try
1701        {
1702            // Force the workspace.
1703            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1704            
1705            getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName);
1706            
1707            long start = System.currentTimeMillis();
1708            
1709            // Delete all resources
1710            _unindexAllResources(workspaceName, solrClient);
1711            
1712            Map<String, Object> results = new HashMap<>();
1713            
1714            try
1715            {
1716                TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources");
1717                AmetysObjectIterable<Resource> resources = resourceRoot.getChildren();
1718                
1719                IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient, progressionTracker);
1720                
1721                long end = System.currentTimeMillis();
1722                
1723                if (!result.hasErrors())
1724                {
1725                    getLogger().info("{} resources indexed without error in {} milliseconds.", result.successCount(), end - start);
1726                }
1727                else
1728                {
1729                    getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount());
1730                }
1731                
1732                results.put("successCount", result.successCount());
1733                if (result.hasErrors())
1734                {
1735                    results.put("errorCount", result.errorCount());
1736                }
1737            }
1738            catch (UnknownAmetysObjectException e)
1739            {
1740                getLogger().info("There is no root for resources in current workspace.");
1741                progressionTracker.setSize(0);
1742            }
1743            
1744            return results;
1745        }
1746        catch (Exception e)
1747        {
1748            String error = String.format("Failed to index all resources in workspace %s", workspaceName);
1749            getLogger().error(error, e);
1750            throw new IndexingException(error, e);
1751        }
1752        finally
1753        {
1754            // Restore context
1755            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1756        }
1757    }
1758    
1759    /**
1760     * Send a collection of contents for indexation in the solr server.
1761     * @param resources the collection of contents to index.
1762     * @param documentType The document type of the resource
1763     * @param workspaceName The workspace where to index
1764     * @return the indexation result.
1765     * @throws Exception if an error occurs while indexing.
1766     */
1767    public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception
1768    {
1769        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1770        return indexResources(resources, documentType, null, workspaceName, solrClient);
1771    }
1772    
1773    /**
1774     * Send a collection of contents for indexation in the solr server.
1775     * @param resources the collection of contents to index.
1776     * @param documentType The document type of the resource
1777     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1778     * @param workspaceName The workspace where to index
1779     * @param solrClient The solr client to use
1780     * @return the indexation result.
1781     * @throws Exception if an error occurs while indexing.
1782     */
1783    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception
1784    {
1785        return indexResources(resources, documentType, resourceRoot, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resources", getLogger()));
1786    }
1787    
1788    /**
1789     * Send a collection of contents for indexation in the solr server.
1790     * @param resources the collection of contents to index.
1791     * @param documentType The document type of the resource
1792     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1793     * @param workspaceName The workspace where to index
1794     * @param solrClient The solr client to use
1795     * @param progressionTracker The progression of the indexation
1796     * @return the indexation result.
1797     * @throws Exception if an error occurs while indexing.
1798     */
1799    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1800    {
1801        Request request = ContextHelper.getRequest(_context);
1802        
1803        // Retrieve the current workspace.
1804        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1805        
1806        try
1807        {
1808            // Force the workspace.
1809            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1810            
1811            getLogger().info("Starting indexation of several resources for workspace {}", workspaceName);
1812            
1813            long start = System.currentTimeMillis();
1814            
1815            IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient, progressionTracker);
1816            
1817            long end = System.currentTimeMillis();
1818            
1819            if (!result.hasErrors())
1820            {
1821                getLogger().info("{} resources indexed without error in {} milliseconds.", result.successCount(), end - start);
1822            }
1823            else
1824            {
1825                getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount());
1826            }
1827            
1828            return result;
1829        }
1830        catch (Exception e)
1831        {
1832            String error = String.format("Failed to index several resources in workspace %s", workspaceName);
1833            getLogger().error(error, e);
1834            throw new IndexingException(error, e);
1835        }
1836        finally
1837        {
1838            // Restore context
1839            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1840        }
1841    }
1842    
1843    /**
1844     * Send some resources for indexation in the solr server.
1845     * @param resources the resources to index.
1846     * @param documentType The document type of the resource
1847     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1848     * @param workspaceName The workspace where to index
1849     * @param solrClient The solr client to use
1850     * @param progressionTracker The progression of the indexation
1851     * @return the indexation result.
1852     * @throws Exception if an error occurs committing the results.
1853     */
1854    protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1855    {
1856        try
1857        {
1858            // Add callable for each content to index
1859            List<Future<Void>> tasks = new ArrayList<>();
1860            for (AmetysObject resource : resources)
1861            {
1862                tasks.addAll(_asyncIndexExplorerItem(resource, documentType, resourceRoot, solrClient));
1863            }
1864            
1865            // Now that everything is submitted, we can iterate and wait for result
1866            return IndexationResult.fromTasks(tasks, getLogger());
1867        }
1868        finally
1869        {
1870            progressionTracker.increment();
1871        }
1872    }
1873    
1874    /**
1875     * Add or update a resource into Solr index
1876     * @param resource The resource to index
1877     * @param documentType The document type of the resource
1878     * @param workspaceName The workspace where to index
1879     * @throws Exception if an error occurs while indexing.
1880     */
1881    public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception
1882    {
1883        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1884        doIndexResources(Collections.singleton(resource), documentType, null, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resource", getLogger()));
1885    }
1886    
1887    private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception
1888    {
1889        getLogger().debug("Unindexing all resources from Solr.");
1890        
1891        String collection = _solrClientProvider.getCollectionName(workspaceName);
1892        solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE);
1893        
1894        getLogger().debug("Succesfully deleted all resource documents from Solr.");
1895    }
1896    
1897    /**
1898     * Delete all resource documents at a given path for all workspaces and commit
1899     * @param rootId The resource root ID, must not be null.
1900     * @param path The resource path relative to the given root, must start with a slash.
1901     * @throws Exception If an error occurs while unindexing.
1902     */
1903    public void unindexResourcesByPath(String rootId, String path) throws Exception
1904    {
1905        String[] workspaceNames = _repository.getWorkspaces();
1906        for (String workspaceName : workspaceNames)
1907        {
1908            unindexResourcesByPath(rootId, path, workspaceName);
1909        }
1910    }
1911    
1912    /**
1913     * Delete all resource documents at a given path.
1914     * @param rootId The resource root ID, must not be null.
1915     * @param path The resource path relative to the given root, must start with a slash.
1916     * @param workspaceName The workspace where to work in
1917     * @throws Exception If an error occurs while unindexing.
1918     */
1919    public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception
1920    {
1921        Request request = ContextHelper.getRequest(_context);
1922        
1923        // Retrieve the current workspace.
1924        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1925        
1926        try
1927        {
1928            // Force the workspace.
1929            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1930            
1931            getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId);
1932            
1933            Query query = new ResourceLocationQuery(rootId, path);
1934            
1935            String collection = _solrClientProvider.getCollectionName(workspaceName);
1936            _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build());
1937            
1938            getLogger().debug("Succesfully deleted resource document from Solr.");
1939        }
1940        catch (Exception e)
1941        {
1942            String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName);
1943            getLogger().error(error, e);
1944            throw new IndexingException(error, e);
1945        }
1946        finally
1947        {
1948            // Restore workspace
1949            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1950        }
1951    }
1952    
1953    /**
1954     * Remove a resource from Solr index for all workspaces and commit
1955     * @param resourceId The id of resource to unindex
1956     * @throws Exception if an error occurs while indexing.
1957     */
1958    public void unindexResource(String resourceId) throws Exception
1959    {
1960        String[] workspaceNames = _repository.getWorkspaces();
1961        for (String workspaceName : workspaceNames)
1962        {
1963            unindexResource(resourceId, workspaceName);
1964        }
1965    }
1966    
1967    /**
1968     * Remove a resource from the Solr index.
1969     * @param resourceId The id of resource to unindex
1970     * @param workspaceName The workspace where to work in
1971     * @throws Exception if an error occurs while unindexing.
1972     */
1973    public void unindexResource(String resourceId, String workspaceName) throws Exception
1974    {
1975        Request request = ContextHelper.getRequest(_context);
1976        
1977        // Retrieve the current workspace.
1978        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1979        
1980        try
1981        {
1982            // Force the workspace.
1983            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1984            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1985            
1986            getLogger().debug("Unindexing resource {} from Solr.", resourceId);
1987            
1988            doUnindexDocument(resourceId, workspaceName, solrClient);
1989            
1990            getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId);
1991        }
1992        catch (Exception e)
1993        {
1994            String error = String.format("Failed to unindex resource  %s in workspace %s", resourceId, workspaceName);
1995            getLogger().error(error, e);
1996            throw new IndexingException(error, e);
1997        }
1998        finally
1999        {
2000            // Restore workspace
2001            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
2002        }
2003    }
2004    
2005    private List<Future<Void>> _asyncIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2006    {
2007        List<Future<Void>> tasks = new ArrayList<>();
2008        
2009        if (node instanceof ResourceCollection resourceCollection)
2010        {
2011            try (AmetysObjectIterable<AmetysObject> children = resourceCollection.getChildren())
2012            {
2013                for (AmetysObject child : children)
2014                {
2015                    tasks.addAll(_asyncIndexExplorerItem(child, documentType, resourceRoot, solrClient));
2016                }
2017            }
2018        }
2019        else if (node instanceof Resource resource)
2020        {
2021            tasks.add(_threadIndexerHelper.submitCallable(new ResourceIndexerCallable(resource, _workspaceSelector.getWorkspace(), documentType, resourceRoot, solrClient)));
2022        }
2023        
2024        return tasks;
2025    }
2026    
2027    /**
2028     * Process a Solr commit operation in all workspaces.
2029     * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient}
2030     * @throws SolrServerException if there is an error on the server
2031     * @throws IOException if there is a communication error with the server
2032     */
2033    public void commit() throws SolrServerException, IOException
2034    {
2035        String[] workspaceNames;
2036        try
2037        {
2038            workspaceNames = _repository.getWorkspaces();
2039            for (String workspaceName : workspaceNames)
2040            {
2041                SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName);
2042                commit(workspaceName, solrClient);
2043            }
2044        }
2045        catch (RepositoryException e)
2046        {
2047            throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e);
2048        }
2049    }
2050    
2051    /**
2052     * Process a Solr commit operation in given workspace.
2053     * @param workspaceName The workspace's name
2054     * @param solrClient The solr client to use
2055     * @throws SolrServerException if there is an error on the server
2056     * @throws IOException if there is a communication error with the server
2057     */
2058    public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2059    {
2060        long time_0 = System.currentTimeMillis();
2061        
2062        // Commit
2063        UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName));
2064        int status = solrResponse.getStatus();
2065        
2066        if (status != 0)
2067        {
2068            throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2069        }
2070        
2071        getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0);
2072    }
2073    
2074    /**
2075     * Launch a solr index optimization.
2076     * @param workspaceName The workspace's name
2077     * @param solrClient The solr client to use
2078     * @throws SolrServerException if there is an error on the server
2079     * @throws IOException if there is a communication error with the server
2080     */
2081    public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2082    {
2083        UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName));
2084        int status = solrResponse.getStatus();
2085        
2086        if (status != 0)
2087        {
2088            throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2089        }
2090        
2091        getLogger().debug("Successful Solr optimize operation during an Ametys indexing process.");
2092    }
2093    
2094    /**
2095     * Delete all documents from the solr index.
2096     * @param workspaceName The workspace name
2097     * @param solrClient The solr client to use
2098     * @throws Exception if an error occurs while unindexing.
2099     */
2100    public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception
2101    {
2102        getLogger().debug("Deleting all documents from Solr.");
2103        
2104        String collection = _solrClientProvider.getCollectionName(workspaceName);
2105        solrClient.deleteByQuery(collection, "*:*");
2106        
2107        getLogger().debug("Successfully deleted all documents from Solr.");
2108    }
2109    
2110    /**
2111     * Delete a document from the Solr server.
2112     * @param id The id of the document to delete from Solr
2113     * @param workspaceName The workspace name
2114     * @param solrClient The solr client to use
2115     * @throws Exception if an error occurs while indexing.
2116     */
2117    protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception
2118    {
2119        UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id);
2120        int status = solrResponse.getStatus();
2121        
2122        if (status != 0)
2123        {
2124            throw new IOException("Deletion of document " + id + ": got status code '" + status + "'.");
2125        }
2126    }
2127    
2128    /**
2129     * Delete content attachments documents of a given content from the Solr server.
2130     * @param contentId The id of the content
2131     * @param workspaceName The workspace name
2132     * @param solrClient The solr client to use
2133     * @throws Exception if an error occurs while indexing.
2134     */
2135    protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception
2136    {
2137        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
2138        
2139        Query query = new ContentAttachmentQuery(contentId);
2140        UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build());
2141        int status = solrResponse.getStatus();
2142        
2143        if (status != 0)
2144        {
2145            throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'.");
2146        }
2147    }
2148    
2149//    /**
2150//     * Get the collection to use.
2151//     * @return The name of the collection to index into.
2152//     */
2153//    protected String getCollection()
2154//    {
2155//        return _solrCorePrefix + _workspaceSelector.getWorkspace();
2156//    }
2157    
2158    private static final class SchemaRequestComparator implements Comparator<SchemaRequest.Update>
2159    {
2160        public int compare(SchemaRequest.Update u1, SchemaRequest.Update u2)
2161        {
2162            return Integer.compare(_getOrder(u1), _getOrder(u2));
2163        }
2164        
2165        private int _getOrder(SchemaRequest.Update update)
2166        {
2167            // Field types are needed first to define fields
2168            if (update instanceof SchemaRequest.AddFieldType)
2169            {
2170                return 1;
2171            }
2172            
2173            // Fields or dynamic fields can be defined at the same time
2174            if (update instanceof SchemaRequest.AddField || update instanceof SchemaRequest.AddDynamicField)
2175            {
2176                return 2;
2177            }
2178            
2179            // Copy fields can be based on fields or dynamic fields
2180            if (update instanceof SchemaRequest.AddCopyField)
2181            {
2182                return 3;
2183            }
2184            
2185            return 0;
2186        }
2187    }
2188    
2189    private class ContentIndexerCallable extends AbstractIndexerCallable<Content>
2190    {
2191        private boolean _indexAttachments;
2192        private SimpleProgressionTracker _tracker;
2193
2194        @SuppressWarnings("synthetic-access")
2195        public ContentIndexerCallable(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker)
2196        {
2197            super(content, workspaceName, solrClient, _manager, _cocoonContext, _resolver, getLogger());
2198            this._indexAttachments = indexAttachments;
2199            this._tracker = progressionTracker;
2200        }
2201
2202        @Override
2203        protected void process(Content content) throws Exception
2204        {
2205            try
2206            {
2207                doIndexContent(content, _workspaceName, _solrClient);
2208                doIndexContentWorkflow(content, _workspaceName, _solrClient);
2209                if (_indexAttachments)
2210                {
2211                    indexContentAttachments(content.getRootAttachments(), content, _solrClient);
2212                }
2213            }
2214            finally
2215            {
2216                _tracker.increment();
2217            }
2218        }
2219
2220        @Override
2221        protected String getObjectLabel()
2222        {
2223            return "content";
2224        }
2225    }
2226    
2227    private class ResourceIndexerCallable extends AbstractIndexerCallable<Resource>
2228    {
2229        private String _documentType;
2230        private TraversableAmetysObject _resourceRoot;
2231        
2232        @SuppressWarnings("synthetic-access")
2233        public ResourceIndexerCallable(Resource resource, String workspaceName, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient)
2234        {
2235            super(resource, workspaceName, solrClient, _manager, _cocoonContext, _resolver, getLogger());
2236            this._documentType = documentType;
2237            this._resourceRoot = resourceRoot;
2238        }
2239        
2240        @Override
2241        protected void process(Resource resource) throws Exception
2242        {
2243            SolrInputDocument document = new SolrInputDocument();
2244            
2245            _solrResourceIndexer.indexResource(resource, document, _documentType, _resourceRoot);
2246            
2247            UpdateResponse solrResponse = _solrClient.add(_solrClientProvider.getCollectionName(_workspaceName), document);
2248            int status = solrResponse.getStatus();
2249            
2250            if (status != 0)
2251            {
2252                throw new IOException("Resource indexation: got status code '" + status + "'.");
2253            }
2254        }
2255
2256        @Override
2257        protected String getObjectLabel()
2258        {
2259            return "resource";
2260        }
2261    }
2262}