Source code

001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.indexing.solr;
017
018import java.io.IOException;
019import java.util.ArrayList;
020import java.util.Collection;
021import java.util.List;
022import java.util.Map;
023
024import org.apache.avalon.framework.context.Context;
025import org.apache.avalon.framework.context.ContextException;
026import org.apache.avalon.framework.context.Contextualizable;
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.cocoon.components.ContextHelper;
031import org.apache.cocoon.environment.Request;
032import org.apache.solr.client.solrj.SolrClient;
033import org.apache.solr.client.solrj.SolrServerException;
034import org.apache.solr.client.solrj.response.UpdateResponse;
035
036import org.ametys.cms.content.archive.ArchiveConstants;
037import org.ametys.cms.content.indexing.solr.SolrFieldNames;
038import org.ametys.cms.content.indexing.solr.SolrIndexer;
039import org.ametys.cms.indexing.IndexingException;
040import org.ametys.cms.indexing.solr.AdditionalDataIndexer;
041import org.ametys.cms.indexing.solr.AdditionalDataIndexerExtensionPoint;
042import org.ametys.cms.search.query.AndQuery;
043import org.ametys.cms.search.query.OrQuery;
044import org.ametys.cms.search.query.Query;
045import org.ametys.cms.search.solr.SolrClientProvider;
046import org.ametys.core.schedule.progression.ContainerProgressionTracker;
047import org.ametys.core.schedule.progression.ProgressionTrackerFactory;
048import org.ametys.core.schedule.progression.SimpleProgressionTracker;
049import org.ametys.plugins.repository.AmetysObjectIterable;
050import org.ametys.plugins.repository.AmetysRepositoryException;
051import org.ametys.plugins.repository.RepositoryConstants;
052import org.ametys.plugins.repository.UnknownAmetysObjectException;
053import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector;
054import org.ametys.runtime.i18n.I18nizableText;
055import org.ametys.runtime.plugin.component.AbstractLogEnabled;
056import org.ametys.web.WebConstants;
057import org.ametys.web.indexing.SiteIndexer;
058import org.ametys.web.repository.page.Page;
059import org.ametys.web.repository.site.Site;
060import org.ametys.web.repository.site.SiteManager;
061import org.ametys.web.repository.sitemap.Sitemap;
062import org.ametys.web.search.query.SiteQuery;
063import org.ametys.web.search.query.SitemapQuery;
064
065/**
066 * Solr implementation of {@link SiteIndexer}.
067 */
068public class SolrSiteIndexer extends AbstractLogEnabled implements SiteIndexer, Serviceable, Contextualizable
069{
070    /** The site manager. */
071    protected SiteManager _siteManager;
072    /** The solr indexer. */
073    protected SolrIndexer _solrIndexer;
074    /** The solr page indexer. */
075    protected SolrPageIndexer _solrPageIndexer;
076    /** The site document provider handler. */
077    protected SiteDocumentProviderExtensionPoint _siteDocProviderEP;
078    /** The Solr client provider */
079    protected SolrClientProvider _solrClientProvider;
080    /** The additional data indexer ep */
081    protected AdditionalDataIndexerExtensionPoint _additionalDataIndexerExtensionPoint;
082    
083    private Context _context;
084    
085    @Override
086    public void contextualize(Context context) throws ContextException
087    {
088        _context = context;
089    }
090    
091    @Override
092    public void service(ServiceManager manager) throws ServiceException
093    {
094        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
095        _solrIndexer = (SolrIndexer) manager.lookup(SolrIndexer.ROLE);
096        _solrPageIndexer = (SolrPageIndexer) manager.lookup(SolrPageIndexer.ROLE);
097        _siteDocProviderEP = (SiteDocumentProviderExtensionPoint) manager.lookup(SiteDocumentProviderExtensionPoint.ROLE);
098        _solrClientProvider = (SolrClientProvider) manager.lookup(SolrClientProvider.ROLE);
099        _additionalDataIndexerExtensionPoint = (AdditionalDataIndexerExtensionPoint) manager.lookup(AdditionalDataIndexerExtensionPoint.ROLE);
100    }
101    
102    @Override
103    public void indexSite(Site site) throws IndexingException
104    {
105        indexSite(site, ProgressionTrackerFactory.createContainerProgressionTracker("Index site '" + site.getName() + "'", getLogger()));
106    }
107    
108    @Override
109    public void indexSite(Site site, ContainerProgressionTracker progressionTracker) throws IndexingException
110    {
111        indexSite(site.getName(), progressionTracker);
112    }
113    
114    @Override
115    public void indexSite(String siteName) throws IndexingException
116    {
117        indexSite(siteName, ProgressionTrackerFactory.createContainerProgressionTracker("Index site '" + siteName + "'", getLogger()));
118    }
119    
120    /**
121     * Index a site in all workspaces
122     * @param siteName  the name of the site to index.
123     * @param progressionTracker The progression of the indexation
124     * @throws IndexingException If an error occurs while indexing the site.
125     */
126    public void indexSite(String siteName, ContainerProgressionTracker progressionTracker) throws IndexingException
127    {
128        progressionTracker.addContainerStep(RepositoryConstants.DEFAULT_WORKSPACE, new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_SITE_INDEXATION_STEPS_LABEL", List.of(RepositoryConstants.DEFAULT_WORKSPACE)));
129        progressionTracker.addContainerStep(WebConstants.LIVE_WORKSPACE, new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_SITE_INDEXATION_STEPS_LABEL", List.of(WebConstants.LIVE_WORKSPACE)));
130        progressionTracker.addContainerStep(ArchiveConstants.ARCHIVE_WORKSPACE, new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_SITE_INDEXATION_STEPS_LABEL", List.of(ArchiveConstants.ARCHIVE_WORKSPACE)));
131        
132        indexSite(siteName, RepositoryConstants.DEFAULT_WORKSPACE, (ContainerProgressionTracker) progressionTracker.getStep(RepositoryConstants.DEFAULT_WORKSPACE));
133        indexSite(siteName, WebConstants.LIVE_WORKSPACE, (ContainerProgressionTracker) progressionTracker.getStep(WebConstants.LIVE_WORKSPACE));
134        indexSite(siteName, ArchiveConstants.ARCHIVE_WORKSPACE, (ContainerProgressionTracker) progressionTracker.getStep(ArchiveConstants.ARCHIVE_WORKSPACE));
135    }
136    
137    @Override
138    public void indexSite(String siteName, String workspaceName) throws IndexingException
139    {
140        indexSite(siteName, workspaceName, ProgressionTrackerFactory.createContainerProgressionTracker("Index site '" + siteName + "' for workspace " + workspaceName, getLogger()));
141    }
142    
143    @Override
144    public void indexSite(String siteName, String workspaceName, ContainerProgressionTracker progressionTracker) throws IndexingException
145    {
146        SolrClient solrClient = _solrClientProvider.getUpdateClient(workspaceName, false);
147        _indexSite(siteName, workspaceName, solrClient, true, progressionTracker);
148    }
149    
150    @Override
151    public void indexSite(String siteName, String workspaceName, SolrClient solrClient, ContainerProgressionTracker progressionTracker) throws IndexingException
152    {
153        // Pass false for commit as caller provided a SolrClient and thus will take care of commit operation (if it is a NoAutoCommit Solr client)
154        boolean commit = false;
155        _indexSite(siteName, workspaceName, solrClient, commit, progressionTracker);
156    }
157    
158    private void _createProgressionTrackerStepsForSubIndexSite(ContainerProgressionTracker progressionTracker, boolean commit) throws AmetysRepositoryException
159    {
160        progressionTracker.addSimpleStep("unindexing", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_UNINDEXING_DOCUMENTS_STEP_LABEL"));
161
162        progressionTracker.addSimpleStep("contents", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_STEP_LABEL"));
163        
164        progressionTracker.addSimpleStep("sitemaps", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_SITE_INDEXATION_SITEMAPS_STEP_LABEL"));
165        
166        progressionTracker.addSimpleStep("resources", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_RESOURCES_STEP_LABEL"));
167        
168        for (String docProviderId : _siteDocProviderEP.getExtensionsIds())
169        {
170            SiteDocumentProvider extension = _siteDocProviderEP.getExtension(docProviderId);
171            progressionTracker.addContainerStep("documents-" + docProviderId, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_DOCUMENTS_PROVIDER_STEPS_LABEL", Map.of("0", extension.getLabel())));
172        }
173
174        if (commit)
175        {
176            progressionTracker.addSimpleStep("commit", new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_SAVING_STEP_LABEL"));
177        }
178    }
179    
180    private void _indexSite(String siteName, String workspaceName, SolrClient solrClient, boolean commit, ContainerProgressionTracker progressionTracker) throws IndexingException
181    {
182        Request request = ContextHelper.getRequest(_context);
183        
184        // Retrieve the current workspace.
185        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
186        // Retrieve the current site name.
187        String currentSiteName = (String) request.getAttribute("siteName");
188        
189        try
190        {
191            // Force the workspace.
192            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
193            
194            // Get the site in the given workspace.
195            Site site = null;
196            try
197            {
198                site = _siteManager.getSite(siteName);
199            }
200            catch (UnknownAmetysObjectException e)
201            {
202                // Site might not exist in the desired workspace (archive for example)
203                return;
204            }
205            
206            _createProgressionTrackerStepsForSubIndexSite(progressionTracker, commit);
207            
208            // Set the site name in the request.
209            request.setAttribute("siteName", siteName);
210
211            getLogger().info("Indexing site '{}' in workspace '{}'", siteName, workspaceName);
212            
213            // Delete all documents from this site in current workspace
214            _unindexSiteDocuments(siteName, workspaceName, solrClient, progressionTracker.getStep("unindexing"));
215            
216            // Index the site's contents in current workspace
217            _indexContents(site, workspaceName, solrClient, progressionTracker.getStep("contents"));
218            
219            // Index the site's sitemaps and pages in current workspace
220            _indexSitemaps(site, workspaceName, solrClient, progressionTracker.getStep("sitemaps"));
221            
222            // Index the site's resources in current workspace
223            _indexResources(site, workspaceName, solrClient, progressionTracker.getStep("resources"));
224            
225            // Add additional site documents
226            for (String docProviderId : _siteDocProviderEP.getExtensionsIds())
227            {
228                SiteDocumentProvider docProvider = _siteDocProviderEP.getExtension(docProviderId);
229                
230                getLogger().debug("Indexing additional documents for site {} with provider {}", siteName, docProvider);
231                
232                docProvider.indexSiteDocuments(site, workspaceName, solrClient, progressionTracker.getStep("documents-" + docProviderId));
233            }
234            
235            if (commit)
236            {
237                _solrIndexer.commit(workspaceName, solrClient);
238                ((SimpleProgressionTracker) progressionTracker.getStep("commit")).increment();
239            }
240        }
241        catch (Exception e)
242        {
243            String error = String.format("Failed to index site %s in workspace %s", siteName, workspaceName);
244            getLogger().error(error, e);
245            throw new IndexingException(error, e);
246        }
247        finally
248        {
249            // Restore the site name.
250            request.setAttribute("siteName", currentSiteName);
251            // Restore context
252            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
253        }
254    }
255    
256    @Override
257    public void indexSitemap(Sitemap sitemap) throws IndexingException
258    {
259        indexSitemap(sitemap.getSiteName(), sitemap.getName());
260    }
261    
262    @Override
263    public void indexSitemap(String siteName, String sitemapName) throws IndexingException
264    {
265        indexSitemap(siteName, sitemapName, RepositoryConstants.DEFAULT_WORKSPACE);
266        indexSitemap(siteName, sitemapName, WebConstants.LIVE_WORKSPACE);
267    }
268    
269    @Override
270    public void indexSitemap(String siteName, String sitemapName, String workspaceName) throws IndexingException
271    {
272        Request request = ContextHelper.getRequest(_context);
273        
274        // Retrieve the current workspace.
275        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
276        
277        try
278        {
279            // Force the workspace.
280            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
281            request.setAttribute("siteName", siteName);
282            SolrClient solrClient = _solrClientProvider.getUpdateClient(workspaceName, false);
283            
284            getLogger().info("Indexing sitemap '{}' of site '{}' in workspace '{}'", sitemapName, siteName, workspaceName);
285            
286            _indexSitemap(siteName, sitemapName, workspaceName, solrClient);
287            
288            _solrIndexer.commit(workspaceName, solrClient);
289        }
290        catch (Exception e)
291        {
292            String error = String.format("Failed to index sitemap '%s' for site '%s' and workspace '%s'", sitemapName, siteName, workspaceName);
293            getLogger().error(error, e);
294            throw new IndexingException(error, e);
295        }
296        finally
297        {
298            // Restore context
299            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
300        }
301    }
302    
303    @Override
304    public void unindexSite(String siteName) throws IndexingException
305    {
306        unindexSite(siteName, RepositoryConstants.DEFAULT_WORKSPACE);
307        unindexSite(siteName, WebConstants.LIVE_WORKSPACE);
308        unindexSite(siteName, ArchiveConstants.ARCHIVE_WORKSPACE);
309    }
310   
311    @Override
312    public void unindexSite(String siteName, String workspaceName) throws IndexingException
313    {
314        Request request = ContextHelper.getRequest(_context);
315        
316        // Retrieve the current workspace.
317        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
318        
319        try
320        {
321            // Force the workspace.
322            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
323            SolrClient solrClient = _solrClientProvider.getUpdateClient(workspaceName, true);
324            
325            getLogger().info("Unindexing site '{}' in workspace {}", siteName, workspaceName);
326            
327            _unindexSiteDocuments(siteName, workspaceName, solrClient);
328        }
329        catch (Exception e)
330        {
331            String error = String.format("Failed to unindex the site %s for workspace %s", siteName, workspaceName);
332            getLogger().error(error, e);
333            throw new IndexingException(error, e);
334        }
335        finally
336        {
337            // Restore context
338            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
339        }
340    }
341    
342    @Override
343    public void unindexSitemap(String siteName, String sitemapName) throws IndexingException
344    {
345        unindexSitemap(siteName, sitemapName, RepositoryConstants.DEFAULT_WORKSPACE);
346        unindexSitemap(siteName, sitemapName, WebConstants.LIVE_WORKSPACE);
347    }
348
349    @Override
350    public void unindexSitemap(String siteName, String sitemapName, String workspaceName) throws IndexingException
351    {
352        Request request = ContextHelper.getRequest(_context);
353        
354        // Retrieve the current workspace.
355        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
356        
357        try
358        {
359            // Force the workspace.
360            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName);
361            SolrClient solrClient = _solrClientProvider.getUpdateClient(workspaceName);
362            
363            getLogger().info("Unindexing sitemap '{}' of site '{}' in workspace '{}'", sitemapName, siteName, workspaceName);
364            
365            _unindexSitemapDocuments(siteName, sitemapName, workspaceName, solrClient);
366        }
367        catch (Exception e)
368        {
369            String error = String.format("Failed to unindex sitemap '%s' for site '%s' and workspace '%s'", sitemapName, siteName, workspaceName);
370            getLogger().error(error, e);
371            throw new IndexingException(error, e);
372        }
373        finally
374        {
375            // Restore context
376            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
377        }
378    }
379    
380    private void _indexSitemaps(Site site, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
381    {
382        // The sitemap node may not exist if site was created but not yet configured
383        if (site.getNode().hasNode(RepositoryConstants.NAMESPACE_PREFIX_INTERNAL + ":sitemaps"))
384        {
385            
386            AmetysObjectIterable<Sitemap> sitemaps = site.getSitemaps();
387            long nbOfSitemaps = sitemaps.getSize();
388            
389            progressionTracker.setSize(nbOfSitemaps);
390
391            for (Sitemap sitemap : sitemaps)
392            {
393                getLogger().info("Indexing sitemap {} started", sitemap.getName());
394                
395                long start = System.currentTimeMillis();
396                
397                _indexSitemap(site.getName(), sitemap.getName(), workspaceName, solrClient);
398                
399                long end = System.currentTimeMillis();
400                
401                getLogger().info("Indexing sitemap {} ended in {} ms", sitemap.getName(), end - start);
402                
403                progressionTracker.increment();
404            }
405        }
406    }
407    
408    private void _indexSitemap(String siteName, String sitemapName, String workspaceName, SolrClient solrClient) throws Exception
409    {
410        // Get the sitemap in the given workspace.
411        Sitemap sitemap = _siteManager.getSite(siteName).getSitemap(sitemapName);
412        
413        // First delete the directory if exists
414        _unindexSitemapDocuments(siteName, sitemapName, workspaceName, solrClient);
415
416        // Index pages of this sitemap
417        AmetysObjectIterable<? extends Page> children = sitemap.getChildrenPages();
418        for (Page page : children)
419        {
420            // Index page recursively, without committing.
421            _solrPageIndexer.indexPage(page.getId(), workspaceName, true, true, solrClient);
422        }
423        
424        // Add additional sitemap documents
425        for (String docProviderId : _siteDocProviderEP.getExtensionsIds())
426        {
427            SiteDocumentProvider docProvider = _siteDocProviderEP.getExtension(docProviderId);
428            
429            getLogger().debug("Indexing additional documents for sitemap {} with provider {}", sitemapName, docProvider);
430            
431            docProvider.indexSitemapDocuments(sitemap, workspaceName, solrClient);
432        }
433    }
434    
435    private void _unindexSiteDocuments(String siteName, String workspaceName, SolrClient solrClient) throws SolrServerException, IOException, Exception
436    {
437        _unindexSiteDocuments(siteName, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Unindex site document for site '" + siteName + "' for workspace " + workspaceName, getLogger()));
438    }
439    
440    private void _unindexSiteDocuments(String siteName, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws SolrServerException, IOException, Exception
441    {
442        List<Query> queries = new ArrayList<>();
443        
444        queries.add(new SiteQuery(siteName));
445        
446        // Calling additional for "content"...
447        // Should also call for "page", but currently not indexing with it
448        Collection<AdditionalDataIndexer> indexers = _additionalDataIndexerExtensionPoint.getIndexers("content");
449        for (AdditionalDataIndexer indexer : indexers)
450        {
451            if (indexer instanceof AdditionalWebDataIndexer awdIndexer)
452            {
453                queries.add(awdIndexer.getUnindexSiteQuery("content", siteName));
454            }
455        }
456        
457        Query query = new OrQuery(queries);
458
459        // delete
460        String collectionName = _solrClientProvider.getCollectionName(workspaceName);
461        UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build());
462        int status = solrResponse.getStatus();
463        
464        if (status != 0)
465        {
466            throw new IOException("Ametys Site de-indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Site name : " + siteName);
467        }
468        
469        getLogger().debug("Successful site de-indexing. Site identifier : {}", siteName);
470        
471        progressionTracker.increment();
472    }
473    
474    private void _unindexSitemapDocuments(String siteName, String sitemapName, String workspace, SolrClient solrClient) throws SolrServerException, IOException, Exception
475    {
476        List<Query> queries = new ArrayList<>();
477        
478        queries.add(new AndQuery(new SiteQuery(siteName), new SitemapQuery(sitemapName)));
479        
480        Query query = new OrQuery(queries);
481        
482        // delete
483        String collectionName = _solrClientProvider.getCollectionName(workspace);
484        UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build());
485        int status = solrResponse.getStatus();
486        
487        if (status != 0)
488        {
489            throw new IOException("Ametys sitemap de-indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Sitemap name : " + siteName + "/" + sitemapName);
490        }
491        
492        getLogger().debug("Successful sitemap de-indexing. Sitemap name : {}/{}", siteName, sitemapName);
493    }
494    
495    /**
496     * Index the contents of a site.
497     * @param site The site to index.
498     * @param workspaceName The workspace name
499     * @param solrClient The solr client to use
500     * @param progressionTracker The progression of the indexation
501     * @throws Exception If an error occurs indexing the contents.
502     */
503    protected void _indexContents(Site site, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
504    {
505        _solrIndexer.indexContents(site.getContents(), workspaceName, true, solrClient, progressionTracker);
506    }
507    
508    /**
509     * Index the resources of a site.
510     * @param site The site to index.
511     * @param workspaceName The workspace name
512     * @param solrClient The solr client to use
513     * @param progressionTracker The progression of the indexation
514     * @throws Exception If an error occurs indexing the resources.
515     */
516    protected void _indexResources(Site site, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception
517    {
518        try
519        {
520            _solrIndexer.indexResources(site.getResources(), SolrFieldNames.TYPE_RESOURCE, site.getRootResources(), workspaceName, solrClient, progressionTracker);
521        }
522        catch (UnknownAmetysObjectException e)
523        {
524            // Ignore if the resource root is not present.
525            progressionTracker.increment();
526        }
527    }
528}