001/*
002 *  Copyright 2013 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016
017package org.ametys.web.live;
018
019import java.io.PrintWriter;
020import java.io.StringWriter;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.List;
024import java.util.Map;
025import java.util.Set;
026
027import javax.jcr.Repository;
028import javax.jcr.Session;
029
030import org.apache.avalon.framework.component.Component;
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.avalon.framework.service.Serviceable;
034import org.apache.commons.lang.StringUtils;
035import org.apache.solr.client.solrj.SolrClient;
036
037import org.ametys.cms.content.indexing.solr.SolrIndexer;
038import org.ametys.cms.indexing.IndexingException;
039import org.ametys.cms.indexing.WorkspaceIndexer;
040import org.ametys.cms.repository.Content;
041import org.ametys.cms.search.solr.SolrClientProvider;
042import org.ametys.core.cache.AbstractCacheManager;
043import org.ametys.core.schedule.progression.ContainerProgressionTracker;
044import org.ametys.core.schedule.progression.SimpleProgressionTracker;
045import org.ametys.plugins.repository.AmetysObjectIterable;
046import org.ametys.plugins.repository.AmetysObjectResolver;
047import org.ametys.plugins.repository.ChainedAmetysObjectIterable;
048import org.ametys.plugins.repository.CollectionIterable;
049import org.ametys.plugins.repository.collection.AmetysObjectCollection;
050import org.ametys.runtime.i18n.I18nizableText;
051import org.ametys.runtime.plugin.component.AbstractLogEnabled;
052import org.ametys.web.WebConstants;
053import org.ametys.web.cache.CacheHelper;
054import org.ametys.web.indexing.SiteIndexer;
055import org.ametys.web.repository.site.Site;
056import org.ametys.web.repository.site.SiteManager;
057import org.ametys.web.skin.Skin;
058import org.ametys.web.skin.SkinsManager;
059
060/**
061 * Component for rebuild the live workspace, reindex all sitemaps and reset cache.
062 * Provide a way to rebuild the full live workspace, or only the live of a site.
063 */
064public class RebuildLiveComponent extends AbstractLogEnabled implements Component, Serviceable
065{
066    /** Avalon Role. */
067    public static final String ROLE = RebuildLiveComponent.class.getName();
068    
069    private AbstractCacheManager _cacheManager;
070    private Repository _repository;
071    private LivePopulatorExtensionPoint _livePopulatorExtensionPoint;
072    private SitePopulator _sitePopulator;
073    private SiteIndexer _siteIndexer;
074    private WorkspaceIndexer _workspaceIndexer;
075    private SiteManager _siteManager;
076    private SkinsManager _skinsManager;
077    private SolrIndexer _solrIndexer;
078    private SolrClientProvider _solrClientProvider;
079    private AmetysObjectResolver _resolver;
080    
081    @Override
082    public void service(ServiceManager manager) throws ServiceException
083    {
084        _cacheManager = (AbstractCacheManager) manager.lookup(AbstractCacheManager.ROLE);
085        _siteIndexer = (SiteIndexer) manager.lookup(SiteIndexer.ROLE);
086        _repository = (Repository) manager.lookup(Repository.class.getName());
087        _livePopulatorExtensionPoint = (LivePopulatorExtensionPoint)  manager.lookup(LivePopulatorExtensionPoint.ROLE);
088        _sitePopulator = (SitePopulator) manager.lookup(SitePopulator.ROLE);
089        _workspaceIndexer = (WorkspaceIndexer) manager.lookup(WorkspaceIndexer.ROLE);
090        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
091        _skinsManager = (SkinsManager) manager.lookup(SkinsManager.ROLE);
092        _solrIndexer = (SolrIndexer) manager.lookup(SolrIndexer.ROLE);
093        _solrClientProvider = (SolrClientProvider) manager.lookup(SolrClientProvider.ROLE);
094        _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
095    }
096    
097    /**
098     * Rebuild live workspace, index all sitemaps and reset cache.
099     * @param progressionTracker The tracker of the progression
100     * @throws Exception if an error occurs.
101     */
102    public void rebuildLiveWorkspace(ContainerProgressionTracker progressionTracker) throws Exception
103    {
104        progressionTracker.addContainerStep("populate", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATE_LIVE_WORKSPACES_STEP_LABEL"));
105        progressionTracker.addContainerStep("index", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_INDEXATION_LIVE_STEP_LABEL"));
106        progressionTracker.addSimpleStep("cache", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_CLEAR_SITE_CACHE_STEP_LABEL"));
107        
108        List<String> errors = new ArrayList<>();
109        
110        try
111        {
112            getLogger().info("Rebuilding live workspace started");
113
114            long t0 = System.currentTimeMillis();
115            
116            errors.addAll(_populateLiveWorkspace(progressionTracker.getStep("populate")));
117            
118            errors.addAll(_reindexLiveWorkspace(progressionTracker.getStep("index")));
119            
120            errors.addAll(_clearAllSitesCache(progressionTracker.getStep("cache")));
121            
122            _clearApplicationCache();
123            
124            getLogger().info("Rebuilding live workspace ended in {} ms with {} error(s)", System.currentTimeMillis() - t0, errors.size());
125        }
126        catch (Exception e)
127        {
128            throw new Exception("Failed to rebuild live workspace", e);
129        }
130        
131        if (errors.size() > 0)
132        {
133            throw new Exception("Failed to rebuild live workspace with " + errors.size() + " error(s)\n\n--------------------------------------\n" + StringUtils.join(errors, "\n\n") + "\n--------------------------------------");
134        }
135    }
136    
137    private List<String> _populateLiveWorkspace(ContainerProgressionTracker progressionTracker)
138    {
139        return _populate(_livePopulatorExtensionPoint.getExtensionsIds(), progressionTracker);
140    }
141
142    private void _createProgressionTrackerStepsForPopulate(Set<String> populatorIds, ContainerProgressionTracker progressionTracker)
143    {
144        for (String populatorId : populatorIds)
145        {
146            LivePopulator populator = _livePopulatorExtensionPoint.getExtension(populatorId);
147            progressionTracker.addContainerStep(populatorId, new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATOR_STEP_LABEL", Map.of("0", populator.getLabel())));
148        }
149    }
150    
151    private List<String> _populate(Set<String> populatorIds, ContainerProgressionTracker progressionTracker)
152    {
153        _createProgressionTrackerStepsForPopulate(populatorIds, progressionTracker);
154        
155        List<String> errors = new ArrayList<>();
156
157        Session session = null;
158        Session liveSession = null;
159        
160        try
161        {
162            getLogger().info("Populating live workspace started");
163            
164            long t0 = System.currentTimeMillis();
165            
166            session = _repository.login();
167            liveSession = _repository.login(WebConstants.LIVE_WORKSPACE);
168
169            // Synchronize other data
170            for (String id : populatorIds)
171            {
172                _populate(id, session, liveSession, errors, progressionTracker.getStep(id));
173            }
174            
175            getLogger().info("Populating live workspace ended in {} ms", System.currentTimeMillis() - t0);
176        }
177        catch (Throwable t)
178        {
179            StringWriter sw = new StringWriter();
180            t.printStackTrace(new PrintWriter(sw));
181            
182            errors.add("Failed to populate live workspace\n" + sw.toString());
183            getLogger().error("Failed to populate live workspace", t);
184        }
185        finally
186        {
187            if (session != null)
188            {
189                session.logout();
190            }
191            
192            if (liveSession != null)
193            {
194                liveSession.logout();
195            }
196        }
197        
198        return errors;
199    }
200
201    private void _populate(String populatorId, Session session, Session liveSession, List<String> errors, ContainerProgressionTracker progressionTracker)
202    {
203        getLogger().info("Populating live workspace using LivePopulator {} started", populatorId);
204          
205        try
206        {
207            LivePopulator populator = _livePopulatorExtensionPoint.getExtension(populatorId);
208            
209            errors.addAll(populator.populate(session, liveSession, progressionTracker));
210            
211            if (liveSession.hasPendingChanges())
212            {
213                liveSession.save();
214            }
215        }
216        catch (Throwable t)
217        {
218            errors.add("Failed to populate live workspace with populator '" + populatorId + "'\n" + t.getMessage());
219            getLogger().error("Failed to populate live workspace with populator '" + populatorId + "'", t);
220        }
221        
222        getLogger().info("Populating live workspace using LivePopulator {} ended", populatorId);
223    }
224    
225    private List<String> _reindexLiveWorkspace(ContainerProgressionTracker progressionTracker)
226    {
227        List<String> errors = new ArrayList<>();
228        
229        try
230        {
231            getLogger().info("Indexing Live workspace started");
232            
233            long t0 = System.currentTimeMillis();
234            
235            // Index the whole live workspace and not just every site.
236            _workspaceIndexer.index(WebConstants.LIVE_WORKSPACE, progressionTracker);
237            
238            getLogger().info("Indexing live workspace ended in {} ms", System.currentTimeMillis() - t0);
239        }
240        catch (IndexingException e)
241        {
242            errors.add("Failed to index live workspace\n" + e.getMessage());
243            getLogger().error("Failed to index live workspace", e);
244        }
245        
246        return errors;
247    }
248    
249    private List<String> _clearAllSitesCache(SimpleProgressionTracker progressionTracker)
250    {
251        AmetysObjectIterable<Site> sites = _siteManager.getSites();
252        
253        progressionTracker.setSize(sites.getSize());
254        
255        List<String> errors = new ArrayList<>();
256        try (AmetysObjectIterable<Site> siteIterable = sites)
257        {
258            for (Site site : siteIterable)
259            {
260                try
261                {
262                    // TODO Clear only the live workspace?
263                    _clearSiteCache(site);
264                }
265                catch (Throwable t)
266                {
267                    errors.add("Failed to clear cache of site '" + site.getName() + "'\n" + t.getMessage());
268                    getLogger().error("Failed to clear cache of site '" + site.getName() + "'", t);
269                }
270                
271                progressionTracker.increment();
272            }
273        }
274        
275        return errors;
276    }
277
278    private void _clearSiteCache(Site site) throws Exception
279    {
280        String siteName = site.getName();
281        
282        getLogger().info("Clearing cache for site {} started", siteName);
283        
284        long t0 = System.currentTimeMillis();
285        
286        CacheHelper.invalidateCache(site, getLogger());
287        
288        getLogger().info("Clearing cache for site '{}' ended in {} ms", siteName, System.currentTimeMillis() - t0);
289    }
290    
291    private void _clearApplicationCache()
292    {
293        _cacheManager.resetAllMemoryCaches();
294    }
295
296    private void _createProgressionTrackerStepsForRebuildLive(String siteName, ContainerProgressionTracker progressionTracker)
297    {
298        progressionTracker.addContainerStep("site", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATE_SITE_STEP_LABEL", Map.of("siteName", new I18nizableText(siteName))));
299        progressionTracker.addContainerStep("indexation", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_INDEXATION_LIVE_STEP_LABEL"));
300        progressionTracker.addSimpleStep("sitecache", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_CLEAR_SITE_CACHE_STEP_LABEL"));
301    }
302    
303    /**
304     * Rebuild the live of a site, index all sitemaps and reset cache.
305     * @param site The site to be rebuilt
306     * @param progressionTracker A progression tracker
307     * @throws Exception if an error occurs.
308     */
309    public void rebuildLive(Site site, ContainerProgressionTracker progressionTracker) throws Exception
310    {
311        String siteName = "";
312        try
313        {
314            siteName = site.getName();
315            _createProgressionTrackerStepsForRebuildLive(siteName, progressionTracker);
316
317            Skin skin = _skinsManager.getSkin(site.getSkinId());
318            assert skin != null;
319            
320            getLogger().info("Rebuilding site {} started", siteName);
321
322            long t0 = System.currentTimeMillis();
323            
324            _populateSite(site, skin, progressionTracker.getStep("site"));
325            _reindexSite(site, progressionTracker.getStep("indexation"));
326            
327            _clearSiteCache(site);
328            ((SimpleProgressionTracker) progressionTracker.getStep("sitecache")).increment();
329            
330            _clearApplicationCache();
331            
332            getLogger().info("Rebuilding site {} ended in {} ms", siteName, System.currentTimeMillis() - t0);
333        }
334        catch (Exception e)
335        {
336            throw new Exception("Failed to rebuild live workspace for site '" + siteName + "'", e);
337        }
338    }
339    
340    private void _populateSite(Site site, Skin skin, ContainerProgressionTracker progressionTracker) throws Exception
341    {
342        String siteName = site.getName();
343
344        getLogger().info("Populating site {} started", siteName);
345        
346        long t0 = System.currentTimeMillis();
347        
348        _sitePopulator.populate(site, skin, progressionTracker);
349        
350        getLogger().info("Populating site {} ended in {} ms", siteName, System.currentTimeMillis() - t0);
351    }
352    
353    private void _reindexSite(Site site, ContainerProgressionTracker progressionTracker) throws Exception
354    {
355        String siteName = site.getName();
356        
357        getLogger().info("Indexing site '{}' started", siteName);
358        
359        long t0 = System.currentTimeMillis();
360        
361        // Reindex the site in the live workspace.
362        _siteIndexer.indexSite(site.getName(), WebConstants.LIVE_WORKSPACE, progressionTracker);
363        
364        getLogger().info("Indexing of site '{}' ended in {} ms", siteName, System.currentTimeMillis() - t0);
365    }
366    
367    private void _createProgressionTrackerStepsForRebuildContentsWithoutSiteWorkspace(ContainerProgressionTracker progressionTracker)
368    {
369        progressionTracker.addContainerStep("populate", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATE_LIVE_WORKSPACES_STEP_LABEL"));
370        progressionTracker.addContainerStep("index", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_REINDEX_CONTENTS_WITHOUT_SITES_STEP_LABEL"));
371        progressionTracker.addSimpleStep("cache", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_CLEAR_SITE_CACHE_STEP_LABEL"));
372    }
373    
374    /**
375     * Rebuild the live workspace for contents without site.
376     * 
377     * More precisely, this method will synchronize and reindex all contents located
378     * at '/ametys:root/ametys:contents' and '/ametys:root/ametys:plugins/&lt;pluginName>/ametys:contents'
379     * @param progressionTracker A progression tracker
380     * @throws Exception if an error occurred
381     */
382    public void rebuildContentsWithoutSiteWorkspace(ContainerProgressionTracker progressionTracker) throws Exception
383    {
384        _createProgressionTrackerStepsForRebuildContentsWithoutSiteWorkspace(progressionTracker);
385        
386        List<String> errors = new ArrayList<>();
387        
388        try
389        {
390            getLogger().info("Rebuilding live workspace started");
391
392            long t0 = System.currentTimeMillis();
393
394            // call ContentsLivePopulator and PluginsLivePopulator
395            errors.addAll(_populate(Set.of(ContentsLivePopulator.class.getName(), PluginsLivePopulator.class.getName()), progressionTracker.getStep("populate")));
396            // reindex impacted contents
397            errors.addAll(_reindexContentWithoutSiteLiveWorkspace(progressionTracker.getStep("index")));
398            
399            // Clear every site's cache.
400            errors.addAll(_clearAllSitesCache(progressionTracker.getStep("cache")));
401            
402            // Clear application cache to make sur that no outdated info is kept in cache
403            _clearApplicationCache();
404            
405            getLogger().info("Rebuilding live workspace ended in {} ms with {} error(s)", System.currentTimeMillis() - t0, errors.size());
406        }
407        catch (Exception e)
408        {
409            throw new Exception("Failed to rebuild live workspace", e);
410        }
411        
412        if (errors.size() > 0)
413        {
414            throw new Exception("Failed to rebuild live workspace with " + errors.size() + " error(s)\n\n--------------------------------------\n" + StringUtils.join(errors, "\n\n") + "\n--------------------------------------");
415        }
416    }
417
418    private void _createProgressionTrackerStepsForReindexContentWithoutSiteLiveWorkspace(ContainerProgressionTracker progressionTracker)
419    {
420        boolean createNullTracker = true;
421        try (AmetysObjectIterable<AmetysObjectCollection> roots = _getContentsWithoutSiteRootNodes())
422        {
423            if (roots.getSize() != 0)
424            {
425                createNullTracker = false;
426            }
427
428            for (AmetysObjectCollection root : roots)
429            {
430                progressionTracker.addSimpleStep(root.getPath(), new I18nizableText(root.getPath()));
431            }
432        }
433        
434        if (createNullTracker)
435        {
436            // Create a sub step just to set the progression of the parent to 100%
437            progressionTracker.addSimpleStep("-", new I18nizableText("")).setSize(0);
438        }
439    }
440    
441    private Collection< ? extends String> _reindexContentWithoutSiteLiveWorkspace(ContainerProgressionTracker progressionTracker) throws Exception
442    {
443        _createProgressionTrackerStepsForReindexContentWithoutSiteLiveWorkspace(progressionTracker);
444
445        List<String> errors = new ArrayList<>();
446        
447        try
448        {
449            getLogger().info("Indexing contents without site live workspace started");
450            
451            long t0 = System.currentTimeMillis();
452
453            // Do not close the client, this is the provider responsibility
454            SolrClient solrClient = _solrClientProvider.getUpdateClient(WebConstants.LIVE_WORKSPACE, false);
455            
456            try (AmetysObjectIterable<AmetysObjectCollection> roots = _getContentsWithoutSiteRootNodes())
457            {
458                for (AmetysObjectCollection root : roots)
459                {
460                    getLogger().info("Indexing live contents at path {}", root.getPath());
461                    
462                    try (AmetysObjectIterable<Content> contents = root.getChildren())
463                    {
464                        _solrIndexer.indexContents(contents, WebConstants.LIVE_WORKSPACE, true, solrClient, progressionTracker.getStep(root.getPath()));
465                        _solrIndexer.commit(WebConstants.LIVE_WORKSPACE, solrClient);
466                    }
467                    
468                    getLogger().info("Indexing live contents at path {} ended", root.getPath());
469                }
470            }
471            
472            getLogger().info("Indexing contents witout site live workspace ended in {} ms", System.currentTimeMillis() - t0);
473        }
474        catch (IndexingException e)
475        {
476            errors.add("Failed to index contents without site live workspace\n" + e.getMessage());
477            getLogger().error("Failed to index contents without site live workspace", e);
478        }
479        
480        return errors;
481    }
482
483    private AmetysObjectIterable<AmetysObjectCollection> _getContentsWithoutSiteRootNodes()
484    {
485        // contents nodes in plugins
486        AmetysObjectIterable<AmetysObjectCollection> roots = _resolver.<AmetysObjectCollection>query("/jcr:root/ametys:root/ametys:plugins/*/ametys:contents");
487        // contents node at ametys root
488        AmetysObjectIterable<AmetysObjectCollection> rootIterable = new CollectionIterable<>(List.of(_resolver.resolveByPath("/ametys:contents")));
489        
490        // Return an iterable so that the consumer is responsible for closing
491        return new ChainedAmetysObjectIterable<>(List.of(rootIterable, roots));
492    }
493}