Source code

001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.cms.indexing.solr;
017
018import java.io.IOException;
019import java.util.List;
020import java.util.Map;
021import java.util.Set;
022
023import javax.jcr.Repository;
024import javax.jcr.RepositoryException;
025import javax.jcr.Session;
026
027import org.apache.avalon.framework.component.Component;
028import org.apache.avalon.framework.context.Context;
029import org.apache.avalon.framework.context.ContextException;
030import org.apache.avalon.framework.context.Contextualizable;
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.avalon.framework.service.Serviceable;
034import org.apache.cocoon.components.ContextHelper;
035import org.apache.cocoon.environment.Request;
036import org.apache.solr.client.solrj.SolrClient;
037import org.apache.solr.client.solrj.SolrServerException;
038
039import org.ametys.cms.content.indexing.solr.SolrIndexer;
040import org.ametys.cms.indexing.IndexingException;
041import org.ametys.cms.indexing.WorkspaceIndexer;
042import org.ametys.cms.search.solr.SolrClientProvider;
043import org.ametys.core.schedule.progression.ContainerProgressionTracker;
044import org.ametys.core.schedule.progression.ProgressionTrackerFactory;
045import org.ametys.core.schedule.progression.SimpleProgressionTracker;
046import org.ametys.plugins.repository.RepositoryConstants;
047import org.ametys.plugins.repository.provider.AbstractRepository;
048import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector;
049import org.ametys.plugins.repository.provider.WorkspaceSelector;
050import org.ametys.runtime.i18n.I18nizableText;
051import org.ametys.runtime.plugin.component.AbstractLogEnabled;
052
053/**
054 * Component indexing a workspace in a Solr server.
055 */
056public class SolrWorkspaceIndexer extends AbstractLogEnabled implements WorkspaceIndexer, Component, Serviceable, Contextualizable
057{
058    /** The repository. */
059    protected Repository _repository;
060    
061    /** The solr indexer. */
062    protected SolrIndexer _solrIndexer;
063    
064    /** The workspace selector. */
065    protected WorkspaceSelector _workspaceSelector;
066    
067    /** Additional documents provider extension point. */
068    protected DocumentProviderExtensionPoint _docProviderEP;
069    
070    /** The Solr client provider */
071    protected SolrClientProvider _solrClientProvider;
072    
073    /** The component context. */
074    protected Context _context;
075
076    @Override
077    public void service(ServiceManager manager) throws ServiceException
078    {
079        _repository = (Repository) manager.lookup(AbstractRepository.ROLE);
080        _solrIndexer = (SolrIndexer) manager.lookup(SolrIndexer.ROLE);
081        _workspaceSelector = (WorkspaceSelector) manager.lookup(WorkspaceSelector.ROLE);
082        _docProviderEP = (DocumentProviderExtensionPoint) manager.lookup(DocumentProviderExtensionPoint.ROLE);
083        _solrClientProvider = (SolrClientProvider) manager.lookup(SolrClientProvider.ROLE);
084    }
085    
086    @Override
087    public void contextualize(Context context) throws ContextException
088    {
089        _context = context;
090    }
091    
092    @Override
093    public String[] getWorkspacesNames() throws RepositoryException
094    {
095        Session session = _repository.login();
096        return session.getWorkspace().getAccessibleWorkspaceNames();
097    }
098    
099    @Override
100    public void indexAllWorkspaces() throws IndexingException
101    {
102        indexAllWorkspaces(ProgressionTrackerFactory.createContainerProgressionTracker("Index all workspaces", getLogger()));
103    }
104    
105    @Override
106    public void indexAllWorkspaces(ContainerProgressionTracker progressionTracker) throws IndexingException
107    {
108        String[] workspaceNames;
109        
110        try
111        {
112            // Get all the workspaces and reindex them.
113            workspaceNames = getWorkspacesNames();
114        }
115        catch (RepositoryException e)
116        {
117            getLogger().error("Error while indexing the workspaces.", e);
118            throw new IndexingException("Error while indexing the workspaces.", e);
119        }
120        
121        SimpleProgressionTracker schemaStep = progressionTracker.addSimpleStep("schema", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_PREPARING_STEP_LABEL"));
122        schemaStep.setSize(2);
123        for (String workspaceName : workspaceNames)
124        {
125            progressionTracker.addContainerStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_WORKSPACES_STEPS_LABEL", List.of(workspaceName)));
126        }
127        
128        // Before sending schema, all cores need to exist
129        ensureCoresExists(workspaceNames);
130        schemaStep.increment();
131
132        _sendSchema();
133        schemaStep.increment();
134        
135        for (String workspaceName : workspaceNames)
136        {
137            _index(workspaceName, false, false, progressionTracker.getStep(workspaceName)); // no need to ensure core exists, nor sending schema, as it was done just before
138        }
139    }
140
141    // Send schema (using default update Solr client)
142    private void _sendSchema() throws IndexingException
143    {
144        Request request = ContextHelper.getRequest(_context);
145        // Retrieve the current workspace.
146        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
147        try
148        {
149            // Force the default workspace.
150            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, RepositoryConstants.DEFAULT_WORKSPACE);
151            _solrIndexer.sendSchema();
152        }
153        catch (IOException | SolrServerException e)
154        {
155            getLogger().error("Error while sending schema.", e);
156            throw new IndexingException("Error while sending schema.", e);
157        }
158        finally
159        {
160            // Restore context
161            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
162        }
163    }
164    
165    @Override
166    public void index(String workspaceName) throws IndexingException
167    {
168        index(workspaceName, ProgressionTrackerFactory.createContainerProgressionTracker("Index workspace " + workspaceName, getLogger()));
169    }
170    
171    @Override
172    public void index(String workspaceName, ContainerProgressionTracker progressionTracker) throws IndexingException
173    {
174        _index(workspaceName, true, true, progressionTracker);
175    }
176    
177    private void _index(String workspaceName, boolean ensureCoreExists, boolean sendSchema, ContainerProgressionTracker progressionTracker) throws IndexingException
178    {
179        if (ensureCoreExists)
180        {
181            // Create the core corresponding to the workspace if it doesn't exist.
182            ensureCoreExists(workspaceName);
183        }
184        
185        getLogger().info("Start indexing workspace {}...", workspaceName);
186        
187        _forceWorkspaceAndDoIndex(workspaceName, sendSchema, progressionTracker);
188        
189        getLogger().info("Successfully indexed workspace {}", workspaceName);
190    }
191    
192    private void _forceWorkspaceAndDoIndex(String workspaceName, boolean sendSchema, ContainerProgressionTracker progressionTracker) throws IndexingException
193    {
194        Request request = ContextHelper.getRequest(_context);
195        
196        // Retrieve the current workspace.
197        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
198        
199        try
200        {
201            // Force the workspace.
202            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
203            
204            if (sendSchema)
205            {
206                try
207                {
208                    _solrIndexer.sendSchema();
209                }
210                catch (IOException | SolrServerException e)
211                {
212                    getLogger().error("Error while sending schema.", e);
213                    throw new IndexingException("Error while sending schema.", e);
214                }
215            }
216            
217            doIndex(workspaceName, progressionTracker);
218        }
219        finally
220        {
221            // Restore context
222            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
223        }
224    }
225    
226    /**
227     * Index the given workspace.
228     * @param workspaceName The workspace name.
229     * @throws IndexingException If an error occurs indexing the workspace.
230     */
231    protected void doIndex(String workspaceName) throws IndexingException
232    {
233        doIndex(workspaceName, ProgressionTrackerFactory.createContainerProgressionTracker("Indexation of " + workspaceName, getLogger()));
234    }
235    
236    private void _createProgressionTrackerStepsForDoIndex(ContainerProgressionTracker progressionTracker)
237    {
238        progressionTracker.addSimpleStep("unindexing", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_UNINDEXING_DOCUMENT_SUB_STEP_LABEL"));
239
240        progressionTracker.addSimpleStep("contents", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_STEP_LABEL"));
241        
242        progressionTracker.addSimpleStep("resources", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_RESOURCES_STEP_LABEL"));
243         
244        _createAdditionalDocumentsProgressionTrackerSteps(progressionTracker);
245        
246        progressionTracker.addSimpleStep("commitoptimize", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_SAVING_COMMIT_AND_OPTIMIZE_SUB_STEP_LABEL"));
247    }
248    
249    /**
250     * Index the given workspace.
251     * @param workspaceName The workspace name.
252     * @param progressionTracker The progression of the indexation
253     * @throws IndexingException If an error occurs indexing the workspace.
254     */
255    protected void doIndex(String workspaceName, ContainerProgressionTracker progressionTracker) throws IndexingException
256    {
257        _createProgressionTrackerStepsForDoIndex(progressionTracker);
258        
259        try
260        {
261            SolrClient solrClient = _solrClientProvider.getUpdateClient(workspaceName, false);
262            
263            // First, unindex all documents.
264            _solrIndexer.unindexAllDocuments(workspaceName, solrClient);
265            ((SimpleProgressionTracker) progressionTracker.getStep("unindexing")).increment();
266            
267            // Index all contents.
268            _solrIndexer.indexAllContents(workspaceName, true, solrClient, progressionTracker.getStep("contents"));
269            
270            // Index all resources.
271            _solrIndexer.indexAllResources(workspaceName, solrClient, progressionTracker.getStep("resources"));
272            
273            // Eventually index additional documents.
274            indexAdditionalDocuments(workspaceName, solrClient, progressionTracker);
275            
276            // Commit changes
277            _solrIndexer.commit(workspaceName, solrClient);
278
279            // When done indexing, optimize.
280            _solrIndexer.optimize(workspaceName, solrClient);
281            ((SimpleProgressionTracker) progressionTracker.getStep("commitoptimize")).increment();
282        }
283        catch (Exception e)
284        {
285            getLogger().error("Error indexing the workspace '" + workspaceName + "'.", e);
286            throw new IndexingException("Error indexing the workspace '" + workspaceName + "'.", e);
287        }
288    }
289    
290    /**
291     * Index additional documents provided by the extensions.
292     * @param workspaceName The workspace name.
293     * @param solrClient The solr client to use
294     * @throws IndexingException If an error occurs while indexing.
295     */
296    protected void indexAdditionalDocuments(String workspaceName, SolrClient solrClient) throws IndexingException
297    {
298        indexAdditionalDocuments(workspaceName, solrClient, ProgressionTrackerFactory.createContainerProgressionTracker("Index additional documents for workspace " + workspaceName, getLogger()));
299    }
300    
301    /**
302     * Create the progression tracker steps for additional documents
303     * @param progressionTracker The progression tracker
304     */
305    protected void _createAdditionalDocumentsProgressionTrackerSteps(ContainerProgressionTracker progressionTracker)
306    {
307        Set<String> extensionsIds = _docProviderEP.getExtensionsIds();
308        
309        for (String id : extensionsIds)
310        {
311            DocumentProvider extension = _docProviderEP.getExtension(id);
312            progressionTracker.addContainerStep("documents-" + id, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_DOCUMENTS_PROVIDER_STEPS_LABEL", Map.of("0", extension.getLabel())));
313        }
314    }
315    
316    /**
317     * Index additional documents provided by the extensions.
318     * @param workspaceName The workspace name.
319     * @param solrClient The solr client to use
320     * @param progressionTracker The progression of the indexation
321     * @throws IndexingException If an error occurs while indexing.
322     */
323    protected void indexAdditionalDocuments(String workspaceName, SolrClient solrClient, ContainerProgressionTracker progressionTracker) throws IndexingException
324    {
325        Set<String> extensionsIds = _docProviderEP.getExtensionsIds();
326        
327        for (String id : extensionsIds)
328        {
329            DocumentProvider docProvider = _docProviderEP.getExtension(id);
330            docProvider.indexDocuments(workspaceName, solrClient, progressionTracker.getStep("documents-" + id));
331        }
332    }
333    
334    /**
335     * Create the given cores if they do not exist.
336     * @param coreNames The core names.
337     * @throws IndexingException If an error occurs while checking if cores exist.
338     */
339    protected void ensureCoresExists(String[] coreNames) throws IndexingException
340    {
341        for (String coreName : coreNames)
342        {
343            ensureCoreExists(coreName);
344        }
345    }
346    
347    /**
348     * Create the given core if it doesn't exist.
349     * @param coreName The core name.
350     * @throws IndexingException If an error occurs while checking if core exists.
351     */
352    protected void ensureCoreExists(String coreName) throws IndexingException
353    {
354        try
355        {
356            Set<String> coreNames = _solrIndexer.getCoreNames();
357            if (!coreNames.contains(coreName))
358            {
359                _solrIndexer.createCore(coreName);
360            }
361        }
362        catch (IOException | SolrServerException e)
363        {
364            String msg = String.format("Error while checking if core '%s' exists.", coreName);
365            getLogger().error(msg, e);
366            throw new IndexingException(msg, e);
367        }
368    }
369}