001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.cms.content.indexing.solr;
017
018import java.io.IOException;
019import java.nio.ByteBuffer;
020import java.nio.CharBuffer;
021import java.nio.charset.CharsetDecoder;
022import java.nio.charset.CodingErrorAction;
023import java.nio.charset.StandardCharsets;
024import java.text.SimpleDateFormat;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collection;
028import java.util.Comparator;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.List;
032import java.util.Map;
033import java.util.Objects;
034import java.util.Set;
035import java.util.function.Function;
036import java.util.stream.Collectors;
037import java.util.stream.StreamSupport;
038
039import javax.jcr.RepositoryException;
040
041import org.apache.avalon.framework.activity.Initializable;
042import org.apache.avalon.framework.component.Component;
043import org.apache.avalon.framework.context.Context;
044import org.apache.avalon.framework.context.ContextException;
045import org.apache.avalon.framework.context.Contextualizable;
046import org.apache.avalon.framework.service.ServiceException;
047import org.apache.avalon.framework.service.ServiceManager;
048import org.apache.avalon.framework.service.Serviceable;
049import org.apache.cocoon.components.ContextHelper;
050import org.apache.cocoon.environment.Request;
051import org.apache.commons.collections4.IterableUtils;
052import org.apache.commons.lang3.ObjectUtils;
053import org.apache.commons.lang3.StringUtils;
054import org.apache.solr.client.solrj.SolrClient;
055import org.apache.solr.client.solrj.SolrQuery;
056import org.apache.solr.client.solrj.SolrResponse;
057import org.apache.solr.client.solrj.SolrServerException;
058import org.apache.solr.client.solrj.request.CoreAdminRequest;
059import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
060import org.apache.solr.client.solrj.request.schema.SchemaRequest;
061import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update;
062import org.apache.solr.client.solrj.response.CoreAdminResponse;
063import org.apache.solr.client.solrj.response.QueryResponse;
064import org.apache.solr.client.solrj.response.SolrResponseBase;
065import org.apache.solr.client.solrj.response.UpdateResponse;
066import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
067import org.apache.solr.client.solrj.response.schema.SchemaResponse;
068import org.apache.solr.client.solrj.util.ClientUtils;
069import org.apache.solr.common.SolrInputDocument;
070import org.apache.solr.common.util.NamedList;
071import org.slf4j.Logger;
072
073import org.ametys.cms.indexing.IndexingException;
074import org.ametys.cms.indexing.solr.ReloadAclCacheRequest;
075import org.ametys.cms.indexing.solr.UpdateAclCacheRequest;
076import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest;
077import org.ametys.cms.repository.Content;
078import org.ametys.cms.repository.ContentQueryHelper;
079import org.ametys.cms.repository.WorkflowAwareContent;
080import org.ametys.cms.rights.solrchecking.ReadAccessHelper;
081import org.ametys.cms.search.query.ContentAttachmentQuery;
082import org.ametys.cms.search.query.DocumentTypeQuery;
083import org.ametys.cms.search.query.OrQuery;
084import org.ametys.cms.search.query.Query;
085import org.ametys.cms.search.query.ResourceLocationQuery;
086import org.ametys.cms.search.solr.NoAutoCommitUpdateClient;
087import org.ametys.cms.search.solr.SolrClientProvider;
088import org.ametys.cms.search.solr.schema.SchemaDefinition;
089import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider;
090import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint;
091import org.ametys.cms.search.solr.schema.SchemaFields;
092import org.ametys.cms.search.solr.schema.SchemaHelper;
093import org.ametys.core.group.GroupIdentity;
094import org.ametys.core.right.AllowedUsers;
095import org.ametys.core.schedule.progression.ContainerProgressionTracker;
096import org.ametys.core.schedule.progression.ProgressionTrackerFactory;
097import org.ametys.core.schedule.progression.SimpleProgressionTracker;
098import org.ametys.core.user.UserIdentity;
099import org.ametys.core.util.DateUtils;
100import org.ametys.plugins.explorer.resources.Resource;
101import org.ametys.plugins.explorer.resources.ResourceCollection;
102import org.ametys.plugins.repository.AmetysObject;
103import org.ametys.plugins.repository.AmetysObjectIterable;
104import org.ametys.plugins.repository.AmetysObjectResolver;
105import org.ametys.plugins.repository.RepositoryConstants;
106import org.ametys.plugins.repository.TraversableAmetysObject;
107import org.ametys.plugins.repository.UnknownAmetysObjectException;
108import org.ametys.plugins.repository.collection.AmetysObjectCollection;
109import org.ametys.plugins.repository.provider.AbstractRepository;
110import org.ametys.plugins.repository.provider.JackrabbitRepository;
111import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector;
112import org.ametys.plugins.repository.provider.WorkspaceSelector;
113import org.ametys.runtime.config.Config;
114import org.ametys.runtime.i18n.I18nizableText;
115import org.ametys.runtime.plugin.component.AbstractLogEnabled;
116
117/**
118 * Solr indexer.
119 */
120public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable
121{
122    /** The component role. */
123    public static final String ROLE = SolrIndexer.class.getName();
124    
125    private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>();
126    
127    private static final String _CONFIGSET_NAME_PREFIX = "configset-";
128    
129    private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_");
130    private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general");
131    
132    private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766;
133    
134    /** The ametys object resolver. */
135    protected AmetysObjectResolver _resolver;
136    /** The schema definition provider extension point. */
137    protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP;
138    /** The schema helper. */
139    protected SchemaHelper _schemaHelper;
140    /** Solr Ametys contents indexer */
141    protected SolrContentIndexer _solrContentIndexer;
142    /** Solr workflow indexer. */
143    protected SolrWorkflowIndexer _solrWorkflowIndexer;
144    /** Solr resource indexer. */
145    protected SolrResourceIndexer _solrResourceIndexer;
146    
147    /** The Solr client provider */
148    protected SolrClientProvider _solrClientProvider;
149    
150    /** The solr core prefix. */
151    protected String _solrCorePrefix;
152    /** The Ametys internal URL used by Solr to query Ametys */
153    protected String _ametysInternalUrl;
154    
155    /** The workspace selector. */
156    protected WorkspaceSelector _workspaceSelector;
157    /** The JCR repository */
158    protected JackrabbitRepository _repository;
159    /** The helper for read access */
160    protected ReadAccessHelper _readAccessHelper;
161    
162    /** The avalon context */
163    protected Context _context;
164
165    /**
166     * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone
167     * @return The date format for indexing dates
168     * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead
169     */
170    @Deprecated
171    public static SimpleDateFormat dateFormat()
172    {
173        if (__DATE_FORMAT.get() == null)
174        {
175            __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
176        }
177        return __DATE_FORMAT.get();
178    }
179    
180    /**
181     * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr.
182     * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept.
183     * @param value The string value to index
184     * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log.
185     * @param documentId The id of the document being indexed. Can be null if you do not want to log.
186     * @param fieldName The name of the field being indexed. Can be null if you do not want to log.
187     * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes)
188     */
189    public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName)
190    {
191        if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT)
192        {
193            // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length())
194            // As a result, value.getBytes().length <= limit
195            return value;
196        }
197        
198        // There is a doubt, the string may need to be truncated (or not)
199        byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
200        int bytesLength = valueBytes.length;
201        if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT)
202        {
203            return value;
204        }
205        
206        if (ObjectUtils.allNotNull(logger, documentId, fieldName))
207        {
208            logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT);
209        }
210        
211        // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974)
212        CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT);
213        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
214                                                       .onMalformedInput(CodingErrorAction.IGNORE);
215        decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true);
216        decoder.flush(charBuffer);
217        return new String(charBuffer.array(), 0, charBuffer.position());
218    }
219    
220    @Override
221    public void service(ServiceManager serviceManager) throws ServiceException
222    {
223        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
224        _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE);
225        _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE);
226        _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE);
227        _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE);
228        _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE);
229        _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE);
230        _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE);
231        _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE);
232        _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE);
233    }
234    
235    @Override
236    public void initialize() throws Exception
237    {
238        Config config = Config.getInstance();
239        _solrCorePrefix = config.getValue("cms.solr.core.prefix");
240        
241        _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url");
242        if (StringUtils.isBlank(_ametysInternalUrl))
243        {
244            // fallback to CMS URL as internal URL is an optional parameter
245            _ametysInternalUrl = config.getValue("cms.url");
246        }
247    }
248    
249    @Override
250    public void contextualize(Context context) throws ContextException
251    {
252        _context = context;
253    }
254    
255    /**
256     * Gets the 'autocommit' Solr client
257     * @param workspaceName The name of the workspace
258     * @return the Solr client
259     */
260    protected SolrClient _getAutoCommitSolrClient(String workspaceName)
261    {
262        return _solrClientProvider.getUpdateClient(workspaceName, true);
263    }
264    
265    /**
266     * Gets the 'no autocommit' Solr client
267     * @param workspaceName The name of the workspace
268     * @return the Solr client
269     */
270    protected SolrClient _getNoAutoCommitSolrClient(String workspaceName)
271    {
272        return _solrClientProvider.getUpdateClient(workspaceName, false);
273    }
274    
275    // for admin operations
276    private SolrClient _defaultSolrClient()
277    {
278        return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE);
279    }
280    
281    /**
282     * Get the names of the Solr cores.
283     * @return The names of the Solr cores.
284     * @throws IOException If an I/O error occurs.
285     * @throws SolrServerException If a Solr error occurs.
286     */
287    @SuppressWarnings("unchecked")
288    public Set<String> getCoreNames() throws IOException, SolrServerException
289    {
290        Set<String> coreNames = new HashSet<>();
291        
292        getLogger().debug("Getting core list.");
293        
294        SolrQuery query = new SolrQuery();
295        query.setRequestHandler("/admin/cores");
296        query.setParam("action", "STATUS");
297        
298        QueryResponse response = _defaultSolrClient().query(query);
299        
300        NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status");
301        for (Map.Entry<String, NamedList<?>> core : status)
302        {
303            String fullName = (String) core.getValue().get("name");
304            if (fullName.startsWith(_solrCorePrefix))
305            {
306                coreNames.add(fullName.substring(_solrCorePrefix.length()));
307            }
308        }
309        
310        return coreNames;
311    }
312    
313    /**
314     * Get the names of the Solr cores.
315     * @return The names of the Solr cores.
316     * @throws IOException If an I/O error occurs.
317     * @throws SolrServerException If a Solr error occurs.
318     */
319    protected Set<String> getRealCoreNames() throws IOException, SolrServerException
320    {
321        Set<String> coreNames = new HashSet<>();
322        
323        getLogger().debug("Getting core list.");
324        
325        CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient());
326        
327        NamedList<NamedList<Object>> status = response.getCoreStatus();
328        for (Map.Entry<String, NamedList<Object>> core : status)
329        {
330            String fullName = (String) core.getValue().get("name");
331            if (fullName.startsWith(_solrCorePrefix))
332            {
333                coreNames.add(fullName.substring(_solrCorePrefix.length()));
334            }
335        }
336        
337        return coreNames;
338    }
339    
340    /**
341     * Create a Solr core.
342     * @param name The name of the core to create.
343     * @throws IOException If an I/O error occurs.
344     * @throws SolrServerException If a Solr error occurs.
345     */
346    public void createCore(String name) throws IOException, SolrServerException
347    {
348        Set<String> cores = getCoreNames();
349        if (!cores.contains(name))
350        {
351            String fullName = _solrCorePrefix + name;
352            String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix);
353            _createConfigset(configsetName);
354            
355            getLogger().info("Creating core '{}' (full name: '{}').", name, fullName);
356            
357            SolrQuery query = new SolrQuery();
358            query.setRequestHandler("/admin/cores");
359            query.setParam("action", "CREATE");
360            query.setParam("name", fullName);
361            query.setParam("configSet", configsetName);
362            query.setParam("property.ametys.url", _ametysInternalUrl);
363            
364            QueryResponse response = _defaultSolrClient().query(query);
365            NamedList<?> results = response.getResponse();
366            
367            NamedList<?> error = (NamedList<?>) results.get("error");
368            if (error != null)
369            {
370                throw new IOException("Error creating the core: " + error.get("msg"));
371            }
372        }
373        else
374        {
375            if (getLogger().isDebugEnabled())
376            {
377                getLogger().debug("Core '" + name + "' already exists, skipping it.");
378            }
379        }
380    }
381    
382    /**
383     * Updates the ametys.url property of the Solr cores.
384     */
385    public void updateAmetysUrlCoreProperty()
386    {
387        Set<String> coreNames;
388        try
389        {
390            coreNames = getCoreNames();
391        }
392        catch (SolrServerException | IOException e)
393        {
394            getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e);
395            return;
396        }
397        
398        for (String coreName : coreNames)
399        {
400            String collection = _solrClientProvider.getCollectionName(coreName);
401            SolrResponseBase response;
402            try
403            {
404                response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection);
405            }
406            catch (SolrServerException | IOException e)
407            {
408                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e);
409                continue;
410            }
411            
412            NamedList<Object> responseParams = response.getResponse();
413            if ("ok".equals(responseParams.get("result")))
414            {
415                Boolean valueChanged = responseParams.getBooleanArg("valueChanged");
416                if (valueChanged)
417                {
418                    getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName);
419                }
420                else
421                {
422                    getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName);
423                }
424            }
425            else
426            {
427                getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName);
428            }
429        }
430    }
431    
432    private void _createConfigset(String name) throws IOException, SolrServerException
433    {
434        getLogger().info("Creating (if necessary) configset '{}'", name);
435        
436        // This request handler will check if configset exists. If not it will be created.
437        SolrQuery query = new SolrQuery();
438        query.setRequestHandler("/admin/cores");
439        query.setParam("action", "createConfigset");
440        query.setParam("name", name);
441        
442        QueryResponse response = _defaultSolrClient().query(query);
443        NamedList<?> results = response.getResponse();
444        
445        NamedList<?> error = (NamedList<?>) results.get("error");
446        if (error != null)
447        {
448            throw new IOException("Error creating the core: " + error.get("msg"));
449        }
450    }
451    
452    /**
453     * Delete a Solr core.
454     * @param name The name of the core to delete.
455     * @throws IOException If an I/O error occurs.
456     * @throws SolrServerException If a Solr error occurs.
457     */
458    public void deleteCore(String name) throws IOException, SolrServerException
459    {
460        String fullName = _solrCorePrefix + name;
461        
462        getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName);
463        
464        SolrQuery query = new SolrQuery();
465        query.setRequestHandler("/admin/cores");
466        query.setParam("action", "UNLOAD");
467        query.setParam("core", fullName);
468        query.setParam("deleteInstanceDir", "true");
469        
470        QueryResponse response = _defaultSolrClient().query(query);
471        NamedList<?> results = response.getResponse();
472        
473        NamedList<?> error = (NamedList<?>) results.get("error");
474        if (error != null)
475        {
476            throw new IOException("Error deleting core" + name + ": " + error.get("msg"));
477        }
478    }
479    
480    /**
481     * Send the schema.
482     * @throws IOException If a communication error occurs.
483     * @throws SolrServerException If a solr error occurs.
484     */
485    public void sendSchema() throws IOException, SolrServerException
486    {
487        getLogger().info("Computing and sending the schema to the solr server.");
488        
489        String workspaceName = _workspaceSelector.getWorkspace();
490        String collection = _solrClientProvider.getCollectionName(workspaceName);
491        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
492        
493//        SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema();
494        SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml");
495        
496        // TODO Clear the schema except fields marked ametysReadOnly="true".
497        
498        // Clear the current schema.
499        clearSchema(solrClient, collection);
500        
501        SchemaRequest schemaRequest = new SchemaRequest();
502        SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
503        
504        // The cleared schema contains only the basic fields which can't be deleted.
505        SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation();
506        SchemaFields schemaFields = new SchemaFields(clearedSchema);
507        
508        getLogger().debug("Schema after clear: \n{}", schemaFields.toString());
509        
510        // Add the static schema types and fields.
511        List<SchemaRequest.Update> updates = new ArrayList<>();
512        
513        // Set "add field" definitions from the static schema to the update list.
514        addStaticSchemaUpdates(updates, staticSchema, schemaFields);
515        
516        getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString());
517        
518        // Set "add field" definitions from the static schema to the update list.
519        addCustomUpdates(updates, schemaFields);
520        
521        updates.sort(new SchemaRequestComparator());
522        
523        SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates);
524        SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
525        
526        getLogger().debug("Send schema response: {}", updateResponse.toString());
527        Object errors = updateResponse.getResponse().get("errors");
528        if (errors != null && errors instanceof List && !((List) errors).isEmpty())
529        {
530            String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString();
531            throw new SolrServerException(msg);
532        }
533    
534        getLogger().info("Schema sent to the solr server.");
535        
536        reloadCores();
537    }
538    
539    /**
540     * Compute the list of {@link Update} directives from the static schema.
541     * @param updates The list of {@link Update} directives to fill.
542     * @param staticSchema The static schema representation.
543     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
544     */
545    protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields)
546    {
547        List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes();
548        for (FieldTypeDefinition fieldType : fieldTypes)
549        {
550            String name = (String) fieldType.getAttributes().get("name");
551            if (!schemaFields.hasFieldType(name))
552            {
553                updates.add(new SchemaRequest.AddFieldType(fieldType));
554                schemaFields.addFieldType(name);
555            }
556        }
557        for (Map<String, Object> field : staticSchema.getFields())
558        {
559            String name = (String) field.get("name");
560            if (!schemaFields.hasField(name))
561            {
562                updates.add(new SchemaRequest.AddField(field));
563                schemaFields.addField(name);
564            }
565        }
566        for (Map<String, Object> field : staticSchema.getDynamicFields())
567        {
568            String name = (String) field.get("name");
569            if (!schemaFields.hasDynamicField(name))
570            {
571                updates.add(new SchemaRequest.AddDynamicField(field));
572                schemaFields.addDynamicField(name);
573            }
574        }
575        for (Map<String, Object> field : staticSchema.getCopyFields())
576        {
577            String source = (String) field.get("source");
578            String dest = (String) field.get("dest");
579            if (!schemaFields.hasCopyField(source, dest))
580            {
581                updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest)));
582                schemaFields.addCopyField(source, dest);
583            }
584        }
585    }
586    
587    /**
588     * Compute the list of custom {@link Update} directives.
589     * @param updates The list of {@link Update} directives to fill.
590     * @param schemaFields The current schema fields, used to track the existing fields (to be filled).
591     */
592    protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields)
593    {
594        // Add all our property-managed fields.
595        for (String providerId : _schemaDefProviderEP.getExtensionsIds())
596        {
597            SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId);
598            
599            for (SchemaDefinition definition : definitionProvider.getDefinitions())
600            {
601                if (!definition.exists(schemaFields))
602                {
603                    SchemaRequest.Update update = definition.getSchemaUpdate();
604                    if (update != null)
605                    {
606                        updates.add(update);
607                    }
608                }
609            }
610        }
611    }
612    
613    /**
614     * Delete all the fields of the existing schema in the given collection.
615     * @param solrClient The Solr client
616     * @param collection The collection.
617     * @throws IOException If a communication error occurs.
618     * @throws SolrServerException If a solr error occurs.
619     */
620    protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException
621    {
622        try
623        {
624            getLogger().info("Clearing the existing schema on the solr server.");
625            
626            SchemaRequest schemaRequest = new SchemaRequest();
627            SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
628            
629            SchemaRepresentation schema = schemaResponse.getSchemaRepresentation();
630            
631            List<SchemaRequest.Update> deletions = new ArrayList<>();
632            
633            // First the copy fields, then dynamic and simple fields, and field types in the end.
634            for (Map<String, Object> field : schema.getCopyFields())
635            {
636                String source = (String) field.get("source");
637                String dest = (String) field.get("dest");
638                deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest)));
639            }
640            for (Map<String, Object> field : schema.getDynamicFields())
641            {
642                String name = (String) field.get("name");
643                deletions.add(new SchemaRequest.DeleteDynamicField(name));
644            }
645            for (Map<String, Object> field : schema.getFields())
646            {
647                String name = (String) field.get("name");
648                if (!_READ_ONLY_FIELDS.contains(name))
649                {
650                    deletions.add(new SchemaRequest.DeleteField(name));
651                }
652            }
653            for (FieldTypeDefinition fieldType : schema.getFieldTypes())
654            {
655                String name = (String) fieldType.getAttributes().get("name");
656                if (!_READ_ONLY_FIELDTYPES.contains(name))
657                {
658                    deletions.add(new SchemaRequest.DeleteFieldType(name));
659                }
660            }
661            
662            SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions);
663            SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection);
664            
665            Object errors = updateResponse.getResponse().get("errors");
666            if (errors != null && errors instanceof List && !((List) errors).isEmpty())
667            {
668                String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString();
669                throw new SolrServerException(msg);
670            }
671            else
672            {
673                getLogger().debug("Clear schema response: {}", updateResponse.toString());
674            }
675            
676            getLogger().info("Solr schema cleared.");
677        }
678        catch (SolrServerException | IOException e)
679        {
680            getLogger().error("Error clearing schema in collection " + collection, e);
681            throw e;
682        }
683    }
684    
685    /**
686     * Reload the solr cores.
687     * @throws IOException If a communication error occurs.
688     * @throws SolrServerException If a solr error occurs.
689     */
690    protected void reloadCores() throws IOException, SolrServerException
691    {
692        getLogger().info("Reloading solr cores.");
693        
694        for (String coreName : getCoreNames())
695        {
696            String fullName = _solrCorePrefix + coreName;
697            
698            CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient());
699            
700            getLogger().debug("Reload core response: {}", reloadResponse.toString());
701        }
702        
703        getLogger().info("All cores reloaded.");
704    }
705    
706    /**
707     * Reloads the ACL Solr cache for all users
708     * @throws IOException If an I/O error occurs.
709     * @throws SolrServerException If a Solr error occurs.
710     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
711     */
712    public void reloadAclCache() throws IOException, SolrServerException, RepositoryException
713    {
714        String[] workspaceNames = _repository.getWorkspaces();
715        for (String workspaceName : workspaceNames)
716        {
717            reloadAclCache(workspaceName);
718        }
719    }
720    
721    /**
722     * Reloads the ACL Solr cache for all users
723     * @param workspaceName The workspace name
724     * @throws IOException If an I/O error occurs.
725     * @throws SolrServerException If a Solr error occurs.
726     */
727    public void reloadAclCache(String workspaceName) throws IOException, SolrServerException
728    {
729        reloadAclCache(workspaceName, false);
730    }
731    
732    /**
733     * Reloads the ACL Solr cache for all users
734     * @param workspaceName The workspace name
735     * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache)
736     * @throws IOException If an I/O error occurs.
737     * @throws SolrServerException If a Solr error occurs.
738     */
739    public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException
740    {
741        getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
742        
743        String collection = _solrClientProvider.getCollectionName(workspaceName);
744        SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection);
745        NamedList<Object> responseObj = responseBase.getResponse();
746        
747        if ("ok".equals(responseObj.get("result")))
748        {
749            getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary);
750        }
751        else
752        {
753            Object error = responseObj.get("error");
754            getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error);
755        }
756    }
757    
758    /**
759     * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces.
760     * @param objects the {@link AmetysObject}s to update.
761     * @throws IOException If an I/O error occurs.
762     * @throws SolrServerException If a Solr error occurs.
763     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
764     */
765    public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException
766    {
767        String[] workspaceNames = _repository.getWorkspaces();
768        for (String workspaceName : workspaceNames)
769        {
770            updateAclCache(objects, workspaceName);
771        }
772    }
773    
774    /**
775     * Updates the ACL Solr cache for some {@link AmetysObject}s.
776     * @param objects the {@link AmetysObject}s to update.
777     * @param workspaceName The workspace name
778     * @throws IOException If an I/O error occurs.
779     * @throws SolrServerException If a Solr error occurs.
780     * @throws RepositoryException If a repository exception occurs when retrieving workspaces.
781     */
782    public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException
783    {
784        Map<String, Map<String, Object>> solrParams = new HashMap<>();
785        
786        for (AmetysObject object : objects)
787        {
788            AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object);
789            
790            solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(),
791                                                  "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(),
792                                                  "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
793                                                  "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()),
794                                                  "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()),
795                                                  "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList())));
796        }
797        
798        String collection = _solrClientProvider.getCollectionName(workspaceName);
799        SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection);
800        NamedList<Object> responseObj = response.getResponse();
801        
802        if ("ok".equals(responseObj.get("result")))
803        {
804            if (getLogger().isInfoEnabled())
805            {
806                getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet());
807            }
808        }
809        else
810        {
811            @SuppressWarnings("unchecked")
812            List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects");
813            getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects);
814        }
815    }
816    
817    /**
818     * Index all the contents in a given workspace.
819     * @param workspaceName the workspace where to index
820     * @param indexAttachments to index content attachments
821     * @param solrClient The solr client to use
822     * @return The indexation result as a Map.
823     * @throws Exception if an error occurs while indexing.
824     */
825    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
826    {
827        return indexAllContents(workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index all contents", getLogger()));
828    }
829    
830    /**
831     * Index all the contents in a given workspace.
832     * @param workspaceName the workspace where to index
833     * @param indexAttachments to index content attachments
834     * @param solrClient The solr client to use
835     * @param progressionTracker The progression of the indexation
836     * @return The indexation result as a Map.
837     * @throws Exception if an error occurs while indexing.
838     */
839    public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
840    {
841        Request request = ContextHelper.getRequest(_context);
842        
843        // Retrieve the current workspace.
844        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
845        
846        try
847        {
848            // Force the workspace.
849            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
850            
851            getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName);
852            
853            long start = System.currentTimeMillis();
854            
855            // Delete all contents
856            unindexAllContents(workspaceName, indexAttachments, solrClient);
857
858            String query = ContentQueryHelper.getContentXPathQuery(null);
859            AmetysObjectIterable<Content> contents = _resolver.query(query);
860            
861            IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient, progressionTracker);
862            
863            long end = System.currentTimeMillis();
864            
865            if (!result.hasErrors())
866            {
867                getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
868            }
869            else
870            {
871                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
872            }
873            
874            Map<String, Object> results = new HashMap<>();
875            results.put("successCount", result.getSuccessCount());
876            if (result.hasErrors())
877            {
878                results.put("errorCount", result.getErrorCount());
879            }
880            
881            return results;
882        }
883        catch (Exception e)
884        {
885            String error = String.format("Failed to index all contents in workspace %s", workspaceName);
886            getLogger().error(error, e);
887            throw new IndexingException(error, e);
888        }
889        finally
890        {
891            // Restore context
892            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
893        }
894    }
895    /**
896     * Unindex all content documents.
897     * @param workspaceName The workspace name
898     * @param unindexAttachments also unindex content attachments
899     * @param solrClient The solr client to use
900     * @throws Exception if an error occurs while unindexing.
901     */
902    protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
903    {
904        String collection = _solrClientProvider.getCollectionName(workspaceName);
905        
906        Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT);
907        Query query;
908        if (unindexAttachments)
909        {
910            Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE);
911            Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE);
912            query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents);
913        }
914        else
915        {
916            query = contents;
917        }
918        solrClient.deleteByQuery(collection, query.build());
919    }
920    
921    /**
922     * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit
923     * @param collectionId The id of collection
924     * @param indexAttachments to index content attachments
925     * @throws Exception if an error occurs while indexing.
926     */
927    public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception
928    {
929        String[] workspaceNames = _repository.getWorkspaces();
930        for (String workspaceName : workspaceNames)
931        {
932            indexSubcontents(collectionId, workspaceName, indexAttachments);
933        }
934    }
935    
936    /**
937     * Index the child contents of a {@link AmetysObjectCollection}
938     * @param collectionId The id of collection
939     * @param workspaceName the workspace where to index
940     * @param indexAttachments to index content attachments
941     * @throws Exception if an error occurs while unindexing.
942     */
943    public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception
944    {
945        Request request = ContextHelper.getRequest(_context);
946        
947        // Retrieve the current workspace.
948        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
949        
950        try
951        {
952            // Force the workspace.
953            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
954            
955            if (_resolver.hasAmetysObjectForId(collectionId))
956            {
957                SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
958                AmetysObjectCollection collection = _resolver.resolveById(collectionId);
959                AmetysObjectIterable<AmetysObject> children = collection.getChildren();
960                
961                for (AmetysObject child : children)
962                {
963                    if (child instanceof Content)
964                    {
965                        Content content = (Content) child;
966                        
967                        _doIndexContent(content, workspaceName, indexAttachments, solrClient);
968                    }
969                }
970            }
971        }
972        finally
973        {
974            // Restore context
975            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
976        }
977    }
978    
979    /**
980     * Add or update a content into Solr index on all workspaces and commit
981     * @param contentId The id of the content to index
982     * @param indexAttachments to index content attachments
983     * @throws Exception if an error occurs while indexing.
984     */
985    public void indexContent(String contentId, boolean indexAttachments) throws Exception
986    {
987        String[] workspaceNames = _repository.getWorkspaces();
988        for (String workspaceName : workspaceNames)
989        {
990            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
991            indexContent(contentId, workspaceName, indexAttachments, solrClient);
992        }
993    }
994    
995    /**
996     * Add or update a content into Solr index
997     * @param contentId The id of the content to index
998     * @param workspaceName the workspace where to index
999     * @param indexAttachments to index content attachments
1000     * @throws Exception if an error occurs while indexing.
1001     */
1002    public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception
1003    {
1004        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1005        indexContent(contentId, workspaceName, indexAttachments, solrClient);
1006    }
1007    
1008    /**
1009     * Add or update a content into Solr index
1010     * @param contentId The id of the content to index
1011     * @param workspaceName the workspace where to index
1012     * @param indexAttachments to index content attachments
1013     * @param solrClient The solr client to use
1014     * @throws Exception if an error occurs while indexing.
1015     */
1016    public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1017    {
1018        Request request = ContextHelper.getRequest(_context);
1019        
1020        // Retrieve the current workspace.
1021        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1022        
1023        try
1024        {
1025            // Force the workspace.
1026            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1027            
1028            if (_resolver.hasAmetysObjectForId(contentId))
1029            {
1030                Content content = _resolver.resolveById(contentId);
1031                _doIndexContent(content, workspaceName, indexAttachments, solrClient);
1032            }
1033        }
1034        finally
1035        {
1036            // Restore context
1037            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1038        }
1039    }
1040    
1041    private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException
1042    {
1043        try
1044        {
1045            long time_0 = System.currentTimeMillis();
1046            
1047            getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName);
1048            
1049            deleteRepeaterDocs(content.getId(), workspaceName, solrClient);
1050            doIndexContent(content, workspaceName, solrClient);
1051            doIndexContentWorkflow(content, workspaceName, solrClient);
1052            if (indexAttachments)
1053            {
1054                indexContentAttachments(content.getRootAttachments(), content, solrClient);
1055            }
1056            
1057            getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0);
1058        }
1059        catch (Exception e)
1060        {
1061            String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName);
1062            getLogger().error(error, e);
1063            throw new IndexingException(error, e);
1064        }
1065    }
1066    
1067    /**
1068     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1069     * @param contents the collection of contents to index.
1070     * @return the indexation result.
1071     * @throws Exception if an error occurs while indexing.
1072     */
1073    public IndexationResult indexContents(Iterable<Content> contents) throws Exception
1074    {
1075        return indexContents(contents, ProgressionTrackerFactory.createContainerProgressionTracker("Index contents", getLogger()));
1076    }
1077    
1078    
1079    /**
1080     * Send a collection of contents for indexation in the solr server on all workspaces and commit
1081     * @param contents the collection of contents to index.
1082     * @param progressionTracker The progression of the indexation
1083     * @return the indexation result.
1084     * @throws Exception if an error occurs while indexing.
1085     */
1086    public IndexationResult indexContents(Iterable<Content> contents, ContainerProgressionTracker progressionTracker) throws Exception
1087    {
1088        
1089        IndexationResult result = new IndexationResult();
1090        String[] workspaceNames = _repository.getWorkspaces();
1091        
1092        for (String workspaceName : workspaceNames)
1093        {
1094            progressionTracker.addSimpleStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_WORKSPACE_STEP_LABEL", List.of(workspaceName)));
1095        }
1096        
1097        for (String workspaceName : workspaceNames)
1098        {
1099            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1100            IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient, progressionTracker.getStep(workspaceName));
1101            
1102            result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount());
1103        }
1104        
1105        return result;
1106    }
1107    
1108    /**
1109     * Send a collection of contents for indexation in the solr server.
1110     * @param contents the collection of contents to index.
1111     * @param workspaceName the workspace where to index
1112     * @param indexAttachments to index content attachments
1113     * @param solrClient The solr client to use
1114     * @return the indexation result.
1115     * @throws Exception if an error occurs while indexing.
1116     */
1117    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception
1118    {
1119        return indexContents(contents, workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index contents for workspace " + workspaceName, getLogger()));
1120    }
1121    
1122    /**
1123     * Send a collection of contents for indexation in the solr server.
1124     * @param contents the collection of contents to index.
1125     * @param workspaceName the workspace where to index
1126     * @param indexAttachments to index content attachments
1127     * @param solrClient The solr client to use
1128     * @param progressionTracker The progression of the indexation
1129     * @return the indexation result.
1130     * @throws Exception if an error occurs while indexing.
1131     */
1132    public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1133    {
1134        Request request = ContextHelper.getRequest(_context);
1135        
1136        // Retrieve the current workspace.
1137        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1138        
1139        try
1140        {
1141            // Force the workspace.
1142            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1143            
1144            getLogger().info("Starting indexation of several contents for workspace {}", workspaceName);
1145            
1146            long start = System.currentTimeMillis();
1147            
1148            List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false)
1149                .map(Content::getId)
1150                .map(this::_resolveSilently)
1151                .filter(Objects::nonNull)
1152                .collect(Collectors.toList());
1153            
1154            IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient, progressionTracker);
1155            
1156            long end = System.currentTimeMillis();
1157            
1158            if (!result.hasErrors())
1159            {
1160                getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1161            }
1162            else
1163            {
1164                getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1165            }
1166            
1167            return result;
1168        }
1169        catch (Exception e)
1170        {
1171            String error = String.format("Failed to index several contents in workspace %s", workspaceName);
1172            getLogger().error(error, e);
1173            throw new IndexingException(error, e);
1174        }
1175        finally
1176        {
1177            // Restore context
1178            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1179        }
1180    }
1181    
1182    private Content _resolveSilently(String contentId)
1183    {
1184        try
1185        {
1186            return _resolver.resolveById(contentId);
1187        }
1188        catch (UnknownAmetysObjectException e)
1189        {
1190            return null;
1191        }
1192    }
1193    
1194    /**
1195     * Send some contents for indexation in the solr server.
1196     * @param contents the contents to index.
1197     * @param workspaceName The workspace name
1198     * @param indexAttachments to index content attachments
1199     * @param solrClient The solr client to use
1200     * @param progressionTracker The progression of the indexation
1201     * @return the indexation result.
1202     * @throws Exception if an error occurs committing the results.
1203     */
1204    protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1205    {
1206        int successCount = 0;
1207        int errorCount = 0;
1208        int numberOfContents = IterableUtils.size(contents);
1209        
1210        progressionTracker.setSize(numberOfContents);
1211        
1212        for (Content content : contents)
1213        {
1214            try
1215            {
1216                doIndexContent(content, workspaceName, solrClient);
1217                doIndexContentWorkflow(content, workspaceName, solrClient);
1218                if (indexAttachments)
1219                {
1220                    indexContentAttachments(content.getRootAttachments(), content, solrClient);
1221                }
1222                successCount++;
1223            }
1224            catch (Exception e)
1225            {
1226                getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e);
1227                errorCount++;
1228            }
1229            
1230            progressionTracker.increment();
1231        }
1232        
1233        return new IndexationResult(successCount, errorCount);
1234    }
1235    
1236    /**
1237     * Update the value of a specific system property in a content document.
1238     * @param content The content to update.
1239     * @param propertyId The system property ID.
1240     * @param workspaceName The workspace name
1241     * @throws Exception if an error occurs while indexing.
1242     */
1243    public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception
1244    {
1245        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1246        updateSystemProperty(content, propertyId, workspaceName, solrClient);
1247    }
1248    
1249    /**
1250     * Update the value of a specific system property in a content document.
1251     * @param content The content to update.
1252     * @param propertyId The system property ID.
1253     * @param workspaceName The workspace name
1254     * @param solrClient The solr client to use
1255     * @throws Exception if an error occurs while indexing.
1256     */
1257    public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1258    {
1259        getLogger().debug("Updating the system property '{}' for content {} into Solr.", propertyId, content);
1260        
1261        SolrInputDocument document = new SolrInputDocument();
1262        boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document);
1263        
1264        if (!hasUpdate)
1265        {
1266            getLogger().debug("Did not index '{}' system property for content {} in Solr because no update to apply.", propertyId, content);
1267            return;
1268        }
1269        
1270        int status = _pushSolrDocument(document, workspaceName, solrClient);
1271        if (status != 0)
1272        {
1273            throw new IOException("Indexing of system property '" + propertyId + "': got status code '" + status + "'.");
1274        }
1275        
1276        getLogger().debug("Succesfully indexed '{}' system property for content {} in Solr.", propertyId, content);
1277    }
1278    
1279    /**
1280     * Update the value of a specific property in a content document.
1281     * @param content The content to update.
1282     * @param propertyId The system property ID.
1283     * @param workspaceName The workspace name
1284     * @throws Exception if an error occurs while indexing.
1285     */
1286    public void updateProperty(Content content, String propertyId, String workspaceName) throws Exception
1287    {
1288        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1289        updateProperty(content, propertyId, workspaceName, solrClient);
1290    }
1291    
1292    /**
1293     * Update the value of a specific property in a content document.
1294     * @param content The content to update.
1295     * @param propertyId The system property ID.
1296     * @param workspaceName The workspace name
1297     * @param solrClient The solr client to use
1298     * @throws Exception if an error occurs while indexing.
1299     */
1300    public void updateProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception
1301    {
1302        getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content);
1303        
1304        SolrInputDocument document = new SolrInputDocument();
1305        boolean hasUpdate = _solrContentIndexer.indexPartialProperty(content, propertyId, document);
1306        
1307        if (!hasUpdate)
1308        {
1309            getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content);
1310            return;
1311        }
1312        
1313        int status = _pushSolrDocument(document, workspaceName, solrClient);
1314        if (status != 0)
1315        {
1316            throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'.");
1317        }
1318        
1319        getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content);
1320    }
1321    
1322    private int _pushSolrDocument(SolrInputDocument document, String workspaceName, SolrClient solrClient) throws Exception
1323    {
1324        String collection = _solrClientProvider.getCollectionName(workspaceName);
1325        UpdateResponse solrResponse = solrClient.add(collection, document);
1326        return solrResponse.getStatus();
1327    }
1328    
1329    /**
1330     * Remove a content from Solr index for all workspaces and commit
1331     * @param contentId The id of content to unindex
1332     * @param unindexAttachments also unindex content attachments
1333     * @throws Exception if an error occurs while indexing.
1334     */
1335    public void unindexContent(String contentId, boolean unindexAttachments) throws Exception
1336    {
1337        String[] workspaceNames = _repository.getWorkspaces();
1338        for (String workspaceName : workspaceNames)
1339        {
1340            unindexContent(contentId, workspaceName, unindexAttachments);
1341        }
1342    }
1343    
1344    /**
1345     * Remove a content from Solr index
1346     * @param contentId The id of content to unindex
1347     * @param workspaceName The workspace where to work in
1348     * @param unindexAttachments also unindex content attachments
1349     * @throws Exception if an error occurs while indexing.
1350     */
1351    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception
1352    {
1353        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1354        unindexContent(contentId, workspaceName, unindexAttachments, solrClient);
1355    }
1356    
1357    /**
1358     * Remove a content from Solr index
1359     * @param contentId The id of content to unindex
1360     * @param workspaceName The workspace where to work in
1361     * @param unindexAttachments also unindex content attachments
1362     * @param solrClient The solr client to use
1363     * @throws Exception if an error occurs while indexing.
1364     */
1365    public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception
1366    {
1367        Request request = ContextHelper.getRequest(_context);
1368        
1369        // Retrieve the current workspace.
1370        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1371        
1372        try
1373        {
1374            // Force the workspace.
1375            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1376            
1377            getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName);
1378            
1379            deleteRepeaterDocs(contentId, workspaceName, solrClient);
1380            doUnindexDocument(contentId, workspaceName, solrClient);
1381            if (unindexAttachments)
1382            {
1383                doUnindexContentAttachments(contentId, workspaceName, solrClient);
1384            }
1385            _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1386            
1387            getLogger().debug("Succesfully deleted content {} from Solr.", contentId);
1388        }
1389        catch (Exception e)
1390        {
1391            String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName);
1392            getLogger().error(error, e);
1393            throw new IndexingException(error, e);
1394        }
1395        finally
1396        {
1397            // Restore workspace
1398            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1399        }
1400    }
1401    
1402    /**
1403     * Remove a content from Solr index for all workspaces and commit
1404     * @param contentIds The id of content to unindex
1405     * @throws Exception if an error occurs while indexing.
1406     */
1407    public void unindexContents(Collection<String> contentIds) throws Exception
1408    {
1409        String[] workspaceNames = _repository.getWorkspaces();
1410        for (String workspaceName : workspaceNames)
1411        {
1412            unindexContents(contentIds, workspaceName);
1413        }
1414    }
1415    
1416    /**
1417     * Remove a content from Solr index
1418     * @param contentIds The id of content to unindex
1419     * @param workspaceName The workspace where to work in
1420     * @throws Exception if an error occurs while indexing.
1421     */
1422    public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception
1423    {
1424        Request request = ContextHelper.getRequest(_context);
1425        
1426        // Retrieve the current workspace.
1427        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1428        
1429        try
1430        {
1431            // Force the workspace.
1432            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1433            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1434            
1435            getLogger().debug("Unindexing several contents from Solr.");
1436            
1437            for (String contentId : contentIds)
1438            {
1439                deleteRepeaterDocs(contentId, workspaceName, solrClient);
1440                doUnindexDocument(contentId, workspaceName, solrClient);
1441                _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient);
1442            }
1443            
1444            getLogger().debug("Succesfully unindexed content from Solr.");
1445        }
1446        catch (Exception e)
1447        {
1448            String error = String.format("Failed to unindex several contents in workspace %s", workspaceName);
1449            getLogger().error(error, e);
1450            throw new IndexingException(error, e);
1451        }
1452        finally
1453        {
1454            // Restore workspace
1455            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1456        }
1457    }
1458    
1459    /**
1460     * Add or update a content into Solr index
1461     * @param content The content to index
1462     * @param workspaceName The workspace where to index
1463     * @param solrClient The solr client to use
1464     * @throws Exception if an error occurs while indexing.
1465     */
1466    protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception
1467    {
1468        long time_0 = System.currentTimeMillis();
1469        
1470        SolrInputDocument document = new SolrInputDocument();
1471        List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document);
1472        
1473        long time_1 = System.currentTimeMillis();
1474        getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0);
1475        
1476        indexAclInitValues(content, document);
1477        
1478        long time_2 = System.currentTimeMillis();
1479        getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1);
1480        
1481        List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments);
1482        documents.add(document);
1483        
1484        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents);
1485        int status = solrResponse.getStatus();
1486        
1487        long time_3 = System.currentTimeMillis();
1488        getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2);
1489        
1490        if (status != 0)
1491        {
1492            throw new IOException("Content indexation: got status code '" + status + "'.");
1493        }
1494    }
1495    
1496    /**
1497     * Indexes read-ACl initial values for object
1498     * @param ametysObject The object
1499     * @param document The Solr document
1500     */
1501    public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document)
1502    {
1503        // Indexation of AmetysObject property
1504        document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true);
1505        
1506        // Indexation of initValues for AllowedUsers
1507        AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject);
1508        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed());
1509        document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed());
1510        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString);
1511        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString);
1512        _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString);
1513        _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString);
1514    }
1515    
1516    private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier)
1517    {
1518        if (values == null)
1519        {
1520            return;
1521        }
1522        
1523        for (T value : values)
1524        {
1525            document.addField(fieldName, stringifier.apply(value));
1526        }
1527    }
1528    
1529    /**
1530     * Index the whole workflow of a content.
1531     * @param content The content.
1532     * @param workspaceName The workspace name
1533     * @param solrClient The solr client to use
1534     * @throws Exception if an error occurs while indexing.
1535     */
1536    protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception
1537    {
1538        if (content instanceof WorkflowAwareContent)
1539        {
1540            _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient);
1541        }
1542    }
1543    
1544    /**
1545     * Index content attachments as new entries in the idnex
1546     * @param collection the collection of attachments
1547     * @param content the content whose attachments will be indexed
1548     * @throws Exception if something goes wrong when indexing the attachments of the content
1549     */
1550    public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception
1551    {
1552        Request request = ContextHelper.getRequest(_context);
1553        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1554        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1555        indexContentAttachments(collection, content, solrClient);
1556    }
1557    
1558    /**
1559     * Index content attachments as new entries in the idnex
1560     * @param collection the collection of attachments
1561     * @param content the content whose attachments will be indexed
1562     * @param solrClient The solr client to use
1563     * @throws Exception if something goes wrong when indexing the attachments of the content
1564     */
1565    public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception
1566    {
1567        if (collection == null)
1568        {
1569            return;
1570        }
1571        
1572        try (AmetysObjectIterable<AmetysObject> children = collection.getChildren())
1573        {
1574            for (AmetysObject object : children)
1575            {
1576                if (object instanceof ResourceCollection)
1577                {
1578                    indexContentAttachments((ResourceCollection) object, content, solrClient);
1579                }
1580                else if (object instanceof Resource)
1581                {
1582                    Resource resource = (Resource) object;
1583                    indexContentAttachment(resource, content, solrClient);
1584                }
1585            }
1586        }
1587    }
1588    
1589    /**
1590     * Index a content attachment
1591     * @param resource the content attachment as a {@link Resource}
1592     * @param content the content whose attachment is going to be indexed
1593     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1594     */
1595    public void indexContentAttachment(Resource resource, Content content) throws Exception
1596    {
1597        Request request = ContextHelper.getRequest(_context);
1598        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1599        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1600        indexContentAttachment(resource, content, solrClient);
1601    }
1602    
1603    /**
1604     * Index a content attachment
1605     * @param resource the content attachment as a {@link Resource}
1606     * @param content the content whose attachment is going to be indexed
1607     * @param solrClient The solr client to use
1608     * @throws Exception if something goes wrong when processing the indexation of the content attachment
1609     */
1610    public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception
1611    {
1612        SolrInputDocument document = new SolrInputDocument();
1613        
1614        // Prepare resource doc
1615        _indexContentAttachment(resource, document, content);
1616        
1617        // Indexation of the document
1618        _indexResourceDocument(resource, document, solrClient);
1619    }
1620    
1621    private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception
1622    {
1623        String language = content.getLanguage();
1624        
1625        _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language);
1626        
1627        // Need the id of the content for unindexing attachment during the unindexing of the content
1628        document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId());
1629    }
1630    
1631    /**
1632     * Index a populated solr input document of type Resource.
1633     * @param resource the resource from which the input document is created
1634     * @param document the input document
1635     * @param solrClient The solr client to use
1636     * @throws SolrServerException if there is an error on the server
1637     * @throws IOException if there is a communication error with the server
1638     */
1639    protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException
1640    {
1641        String resourceId = resource.getId();
1642        
1643        // Retrieve appropriate collection name
1644        Request request = ContextHelper.getRequest(_context);
1645        String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1646        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
1647        
1648        // Add document
1649        UpdateResponse solrResponse = solrClient.add(collectionName, document);
1650        int status = solrResponse.getStatus();
1651        
1652        if (status != 0)
1653        {
1654            throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId);
1655        }
1656        
1657        getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId);
1658    }
1659    
1660    /**
1661     * Delete repeater documents of a specified content.
1662     * @param workspaceName The workspace name
1663     * @param contentId the content ID.
1664     * @param solrClient The solr client to use
1665     * @throws Exception if an error occurs while indexing.
1666     */
1667    protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception
1668    {
1669        long time_0 = System.currentTimeMillis();
1670        
1671        // _documentType:repeater AND id:content\://xxx/*
1672        StringBuilder query = new StringBuilder();
1673        query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER)
1674             .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*");
1675        
1676        solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString());
1677        
1678        getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0);
1679    }
1680    
1681    /**
1682     * Index all the resources in a given workspace.
1683     * @param workspaceName The workspace where to index
1684     * @param solrClient The solr client to use
1685     * @param progressionTracker The progression of the indexation
1686     * @return The indexation result as a Map.
1687     * @throws Exception if an error occurs while indexing.
1688     */
1689    public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1690    {
1691        Request request = ContextHelper.getRequest(_context);
1692        
1693        // Retrieve the current workspace.
1694        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1695        
1696        try
1697        {
1698            // Force the workspace.
1699            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1700            
1701            getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName);
1702            
1703            long start = System.currentTimeMillis();
1704            
1705            // Delete all resources
1706            _unindexAllResources(workspaceName, solrClient);
1707            
1708            Map<String, Object> results = new HashMap<>();
1709            
1710            try
1711            {
1712                TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources");
1713                AmetysObjectIterable<Resource> resources = resourceRoot.getChildren();
1714                
1715                IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient, progressionTracker);
1716                
1717                long end = System.currentTimeMillis();
1718                
1719                if (!result.hasErrors())
1720                {
1721                    getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1722                }
1723                else
1724                {
1725                    getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1726                }
1727                
1728                results.put("successCount", result.getSuccessCount());
1729                if (result.hasErrors())
1730                {
1731                    results.put("errorCount", result.getErrorCount());
1732                }
1733            }
1734            catch (UnknownAmetysObjectException e)
1735            {
1736                getLogger().info("There is no root for resources in current workspace.");
1737                progressionTracker.setSize(0);
1738            }
1739            
1740            return results;
1741        }
1742        catch (Exception e)
1743        {
1744            String error = String.format("Failed to index all resources in workspace %s", workspaceName);
1745            getLogger().error(error, e);
1746            throw new IndexingException(error, e);
1747        }
1748        finally
1749        {
1750            // Restore context
1751            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1752        }
1753    }
1754    
1755    /**
1756     * Send a collection of contents for indexation in the solr server.
1757     * @param resources the collection of contents to index.
1758     * @param documentType The document type of the resource
1759     * @param workspaceName The workspace where to index
1760     * @return the indexation result.
1761     * @throws Exception if an error occurs while indexing.
1762     */
1763    public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception
1764    {
1765        SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1766        return indexResources(resources, documentType, null, workspaceName, solrClient);
1767    }
1768    
1769    /**
1770     * Send a collection of contents for indexation in the solr server.
1771     * @param resources the collection of contents to index.
1772     * @param documentType The document type of the resource
1773     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1774     * @param workspaceName The workspace where to index
1775     * @param solrClient The solr client to use
1776     * @return the indexation result.
1777     * @throws Exception if an error occurs while indexing.
1778     */
1779    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception
1780    {
1781        return indexResources(resources, documentType, resourceRoot, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resources", getLogger()));
1782    }
1783    
1784    /**
1785     * Send a collection of contents for indexation in the solr server.
1786     * @param resources the collection of contents to index.
1787     * @param documentType The document type of the resource
1788     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1789     * @param workspaceName The workspace where to index
1790     * @param solrClient The solr client to use
1791     * @param progressionTracker The progression of the indexation
1792     * @return the indexation result.
1793     * @throws Exception if an error occurs while indexing.
1794     */
1795    public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1796    {
1797        Request request = ContextHelper.getRequest(_context);
1798        
1799        // Retrieve the current workspace.
1800        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1801        
1802        try
1803        {
1804            // Force the workspace.
1805            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1806            
1807            getLogger().info("Starting indexation of several resources for workspace {}", workspaceName);
1808            
1809            long start = System.currentTimeMillis();
1810            
1811            IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient, progressionTracker);
1812            
1813            long end = System.currentTimeMillis();
1814            
1815            if (!result.hasErrors())
1816            {
1817                getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start);
1818            }
1819            else
1820            {
1821                getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount());
1822            }
1823            
1824            return result;
1825        }
1826        catch (Exception e)
1827        {
1828            String error = String.format("Failed to index several resources in workspace %s", workspaceName);
1829            getLogger().error(error, e);
1830            throw new IndexingException(error, e);
1831        }
1832        finally
1833        {
1834            // Restore context
1835            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1836        }
1837    }
1838    
1839    /**
1840     * Send some resources for indexation in the solr server.
1841     * @param resources the resources to index.
1842     * @param documentType The document type of the resource
1843     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
1844     * @param workspaceName The workspace where to index
1845     * @param solrClient The solr client to use
1846     * @param progressionTracker The progression of the indexation
1847     * @return the indexation result.
1848     * @throws Exception if an error occurs committing the results.
1849     */
1850    protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
1851    {
1852        int successCount = 0;
1853        int errorCount = 0;
1854        int numberOfResources = IterableUtils.size(resources);
1855
1856        progressionTracker.setSize(numberOfResources);
1857
1858        for (AmetysObject resource : resources)
1859        {
1860            try
1861            {
1862                doIndexExplorerItem(resource, documentType, resourceRoot, solrClient);
1863                successCount++;
1864            }
1865            catch (Exception e)
1866            {
1867                getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e);
1868                errorCount++;
1869            }
1870            
1871            progressionTracker.increment();
1872        }
1873        
1874        return new IndexationResult(successCount, errorCount);
1875    }
1876    
1877    /**
1878     * Add or update a resource into Solr index
1879     * @param resource The resource to index
1880     * @param documentType The document type of the resource
1881     * @param workspaceName The workspace where to index
1882     * @throws Exception if an error occurs while indexing.
1883     */
1884    public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception
1885    {
1886        Request request = ContextHelper.getRequest(_context);
1887        
1888        // Retrieve the current workspace.
1889        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1890        
1891        try
1892        {
1893            // Force the workspace.
1894            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1895            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
1896            
1897            getLogger().debug("Indexing resource {} into Solr.", resource.getId());
1898            
1899            doIndexResource(resource, documentType, null, solrClient);
1900            
1901            getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId());
1902        }
1903        catch (Exception e)
1904        {
1905            String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName);
1906            getLogger().error(error, e);
1907            throw new IndexingException(error, e);
1908        }
1909        finally
1910        {
1911            // Restore context
1912            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1913        }
1914    }
1915    
1916    private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception
1917    {
1918        getLogger().debug("Unindexing all resources from Solr.");
1919        
1920        String collection = _solrClientProvider.getCollectionName(workspaceName);
1921        solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE);
1922        
1923        getLogger().debug("Succesfully deleted all resource documents from Solr.");
1924    }
1925    
1926    /**
1927     * Delete all resource documents at a given path for all workspaces and commit
1928     * @param rootId The resource root ID, must not be null.
1929     * @param path The resource path relative to the given root, must start with a slash.
1930     * @throws Exception If an error occurs while unindexing.
1931     */
1932    public void unindexResourcesByPath(String rootId, String path) throws Exception
1933    {
1934        String[] workspaceNames = _repository.getWorkspaces();
1935        for (String workspaceName : workspaceNames)
1936        {
1937            unindexResourcesByPath(rootId, path, workspaceName);
1938        }
1939    }
1940    
1941    /**
1942     * Delete all resource documents at a given path.
1943     * @param rootId The resource root ID, must not be null.
1944     * @param path The resource path relative to the given root, must start with a slash.
1945     * @param workspaceName The workspace where to work in
1946     * @throws Exception If an error occurs while unindexing.
1947     */
1948    public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception
1949    {
1950        Request request = ContextHelper.getRequest(_context);
1951        
1952        // Retrieve the current workspace.
1953        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
1954        
1955        try
1956        {
1957            // Force the workspace.
1958            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
1959            
1960            getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId);
1961            
1962            Query query = new ResourceLocationQuery(rootId, path);
1963            
1964            String collection = _solrClientProvider.getCollectionName(workspaceName);
1965            _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build());
1966            
1967            getLogger().debug("Succesfully deleted resource document from Solr.");
1968        }
1969        catch (Exception e)
1970        {
1971            String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName);
1972            getLogger().error(error, e);
1973            throw new IndexingException(error, e);
1974        }
1975        finally
1976        {
1977            // Restore workspace
1978            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
1979        }
1980    }
1981    
1982    /**
1983     * Remove a resource from Solr index for all workspaces and commit
1984     * @param resourceId The id of resource to unindex
1985     * @throws Exception if an error occurs while indexing.
1986     */
1987    public void unindexResource(String resourceId) throws Exception
1988    {
1989        String[] workspaceNames = _repository.getWorkspaces();
1990        for (String workspaceName : workspaceNames)
1991        {
1992            unindexResource(resourceId, workspaceName);
1993        }
1994    }
1995    
1996    /**
1997     * Remove a resource from the Solr index.
1998     * @param resourceId The id of resource to unindex
1999     * @param workspaceName The workspace where to work in
2000     * @throws Exception if an error occurs while unindexing.
2001     */
2002    public void unindexResource(String resourceId, String workspaceName) throws Exception
2003    {
2004        Request request = ContextHelper.getRequest(_context);
2005        
2006        // Retrieve the current workspace.
2007        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
2008        
2009        try
2010        {
2011            // Force the workspace.
2012            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
2013            SolrClient solrClient = _getAutoCommitSolrClient(workspaceName);
2014            
2015            getLogger().debug("Unindexing resource {} from Solr.", resourceId);
2016            
2017            doUnindexDocument(resourceId, workspaceName, solrClient);
2018            
2019            getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId);
2020        }
2021        catch (Exception e)
2022        {
2023            String error = String.format("Failed to unindex resource  %s in workspace %s", resourceId, workspaceName);
2024            getLogger().error(error, e);
2025            throw new IndexingException(error, e);
2026        }
2027        finally
2028        {
2029            // Restore workspace
2030            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
2031        }
2032    }
2033    
2034    /**
2035     * Add or update a resource into Solr index
2036     * @param node The resource to index
2037     * @param documentType The document type of the resource
2038     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
2039     * @param solrClient The solr client to use
2040     * @throws Exception if an error occurs while indexing.
2041     */
2042    protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2043    {
2044        if (node instanceof ResourceCollection)
2045        {
2046            try (AmetysObjectIterable<AmetysObject> children = ((ResourceCollection) node).getChildren())
2047            {
2048                for (AmetysObject child : children)
2049                {
2050                    doIndexExplorerItem(child, documentType, resourceRoot, solrClient);
2051                }
2052            }
2053        }
2054        else if (node instanceof Resource)
2055        {
2056            doIndexResource((Resource) node, documentType, resourceRoot, solrClient);
2057        }
2058    }
2059    
2060    /**
2061     * Add or update a resource into Solr index
2062     * @param resource The resource to index
2063     * @param documentType The document type of the resource
2064     * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource.
2065     * @param solrClient The solr client to use
2066     * @throws Exception if an error occurs while indexing.
2067     */
2068    protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception
2069    {
2070        SolrInputDocument document = new SolrInputDocument();
2071        
2072        _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot);
2073        
2074        String workspaceName = _workspaceSelector.getWorkspace();
2075        UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document);
2076        int status = solrResponse.getStatus();
2077        
2078        if (status != 0)
2079        {
2080            throw new IOException("Resource indexation: got status code '" + status + "'.");
2081        }
2082    }
2083    
2084    /**
2085     * Process a Solr commit operation in all workspaces.
2086     * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient}
2087     * @throws SolrServerException if there is an error on the server
2088     * @throws IOException if there is a communication error with the server
2089     */
2090    public void commit() throws SolrServerException, IOException
2091    {
2092        String[] workspaceNames;
2093        try
2094        {
2095            workspaceNames = _repository.getWorkspaces();
2096            for (String workspaceName : workspaceNames)
2097            {
2098                SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName);
2099                commit(workspaceName, solrClient);
2100            }
2101        }
2102        catch (RepositoryException e)
2103        {
2104            throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e);
2105        }
2106    }
2107    
2108    /**
2109     * Process a Solr commit operation in given workspace.
2110     * @param workspaceName The workspace's name
2111     * @param solrClient The solr client to use
2112     * @throws SolrServerException if there is an error on the server
2113     * @throws IOException if there is a communication error with the server
2114     */
2115    public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2116    {
2117        long time_0 = System.currentTimeMillis();
2118        
2119        // Commit
2120        UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName));
2121        int status = solrResponse.getStatus();
2122        
2123        if (status != 0)
2124        {
2125            throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2126        }
2127        
2128        getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0);
2129    }
2130    
2131    /**
2132     * Launch a solr index optimization.
2133     * @param workspaceName The workspace's name
2134     * @param solrClient The solr client to use
2135     * @throws SolrServerException if there is an error on the server
2136     * @throws IOException if there is a communication error with the server
2137     */
2138    public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException
2139    {
2140        UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName));
2141        int status = solrResponse.getStatus();
2142        
2143        if (status != 0)
2144        {
2145            throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'.");
2146        }
2147        
2148        getLogger().debug("Successful Solr optimize operation during an Ametys indexing process.");
2149    }
2150    
2151    /**
2152     * Delete all documents from the solr index.
2153     * @param workspaceName The workspace name
2154     * @param solrClient The solr client to use
2155     * @throws Exception if an error occurs while unindexing.
2156     */
2157    public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception
2158    {
2159        getLogger().debug("Deleting all documents from Solr.");
2160        
2161        String collection = _solrClientProvider.getCollectionName(workspaceName);
2162        solrClient.deleteByQuery(collection, "*:*");
2163        
2164        getLogger().debug("Successfully deleted all documents from Solr.");
2165    }
2166    
2167    /**
2168     * Delete a document from the Solr server.
2169     * @param id The id of the document to delete from Solr
2170     * @param workspaceName The workspace name
2171     * @param solrClient The solr client to use
2172     * @throws Exception if an error occurs while indexing.
2173     */
2174    protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception
2175    {
2176        UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id);
2177        int status = solrResponse.getStatus();
2178        
2179        if (status != 0)
2180        {
2181            throw new IOException("Deletion of document " + id + ": got status code '" + status + "'.");
2182        }
2183    }
2184    
2185    /**
2186     * Delete content attachments documents of a given content from the Solr server.
2187     * @param contentId The id of the content
2188     * @param workspaceName The workspace name
2189     * @param solrClient The solr client to use
2190     * @throws Exception if an error occurs while indexing.
2191     */
2192    protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception
2193    {
2194        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
2195        
2196        Query query = new ContentAttachmentQuery(contentId);
2197        UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build());
2198        int status = solrResponse.getStatus();
2199        
2200        if (status != 0)
2201        {
2202            throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'.");
2203        }
2204    }
2205    
2206//    /**
2207//     * Get the collection to use.
2208//     * @return The name of the collection to index into.
2209//     */
2210//    protected String getCollection()
2211//    {
2212//        return _solrCorePrefix + _workspaceSelector.getWorkspace();
2213//    }
2214    
2215    class IndexationResult
2216    {
2217        protected int _successCount;
2218        
2219        protected int _errorCount;
2220        
2221        /**
2222         * Constructor
2223         */
2224        public IndexationResult()
2225        {
2226            this(0, 0);
2227        }
2228        
2229        /**
2230         * Constructor
2231         * @param successCount The success count.
2232         * @param errorCount The error count.
2233         */
2234        public IndexationResult(int successCount, int errorCount)
2235        {
2236            this._successCount = successCount;
2237            this._errorCount = errorCount;
2238        }
2239        
2240        /**
2241         * Get the successCount.
2242         * @return the successCount
2243         */
2244        public int getSuccessCount()
2245        {
2246            return _successCount;
2247        }
2248        
2249        /**
2250         * Set the successCount.
2251         * @param successCount the successCount to set
2252         */
2253        public void setSuccessCount(int successCount)
2254        {
2255            this._successCount = successCount;
2256        }
2257        
2258        /**
2259         * Test if the indexation had errors.
2260         * @return true if the indexation had errors, false otherwise.
2261         */
2262        public boolean hasErrors()
2263        {
2264            return _errorCount > 0;
2265        }
2266        
2267        /**
2268         * Get the errorCount.
2269         * @return the errorCount
2270         */
2271        public int getErrorCount()
2272        {
2273            return _errorCount;
2274        }
2275        
2276        /**
2277         * Set the errorCount.
2278         * @param errorCount the errorCount to set
2279         */
2280        public void setErrorCount(int errorCount)
2281        {
2282            this._errorCount = errorCount;
2283        }
2284    }
2285    
2286    private static final class SchemaRequestComparator implements Comparator<SchemaRequest.Update>
2287    {
2288        public int compare(SchemaRequest.Update u1, SchemaRequest.Update u2)
2289        {
2290            return Integer.compare(_getOrder(u1), _getOrder(u2));
2291        }
2292        
2293        private int _getOrder(SchemaRequest.Update update)
2294        {
2295            // Field types are needed first to define fields
2296            if (update instanceof SchemaRequest.AddFieldType)
2297            {
2298                return 1;
2299            }
2300            
2301            // Fields or dynamic fields can be defined at the same time
2302            if (update instanceof SchemaRequest.AddField || update instanceof SchemaRequest.AddDynamicField)
2303            {
2304                return 2;
2305            }
2306            
2307            // Copy fields can be based on fields or dynamic fields
2308            if (update instanceof SchemaRequest.AddCopyField)
2309            {
2310                return 3;
2311            }
2312            
2313            return 0;
2314        }
2315    }
2316}