001/*
002 *  Copyright 2013 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016
017package org.ametys.web.live;
018
019import java.io.PrintWriter;
020import java.io.StringWriter;
021import java.util.ArrayList;
022import java.util.Collection;
023import java.util.List;
024import java.util.Map;
025import java.util.Set;
026
027import javax.jcr.Repository;
028import javax.jcr.Session;
029
030import org.apache.avalon.framework.component.Component;
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.avalon.framework.service.Serviceable;
034import org.apache.commons.lang3.StringUtils;
035import org.apache.solr.client.solrj.SolrClient;
036
037import org.ametys.cms.content.indexing.solr.SolrIndexer;
038import org.ametys.cms.indexing.IndexingException;
039import org.ametys.cms.indexing.WorkspaceIndexer;
040import org.ametys.cms.repository.Content;
041import org.ametys.cms.search.solr.SolrClientProvider;
042import org.ametys.core.cache.AbstractCacheManager;
043import org.ametys.core.schedule.progression.ContainerProgressionTracker;
044import org.ametys.core.schedule.progression.SimpleProgressionTracker;
045import org.ametys.plugins.repository.AmetysObjectIterable;
046import org.ametys.plugins.repository.AmetysObjectResolver;
047import org.ametys.plugins.repository.ChainedAmetysObjectIterable;
048import org.ametys.plugins.repository.CollectionIterable;
049import org.ametys.plugins.repository.collection.AmetysObjectCollection;
050import org.ametys.runtime.i18n.I18nizableText;
051import org.ametys.runtime.plugin.component.AbstractLogEnabled;
052import org.ametys.web.WebConstants;
053import org.ametys.web.cache.FOCommHelper;
054import org.ametys.web.indexing.SiteIndexer;
055import org.ametys.web.repository.site.Site;
056import org.ametys.web.repository.site.SiteManager;
057import org.ametys.web.skin.Skin;
058import org.ametys.web.skin.SkinsManager;
059
060/**
061 * Component for rebuild the live workspace, reindex all sitemaps and reset cache.
062 * Provide a way to rebuild the full live workspace, or only the live of a site.
063 */
064public class RebuildLiveComponent extends AbstractLogEnabled implements Component, Serviceable
065{
066    /** Avalon Role. */
067    public static final String ROLE = RebuildLiveComponent.class.getName();
068    
069    private AbstractCacheManager _cacheManager;
070    private Repository _repository;
071    private LivePopulatorExtensionPoint _livePopulatorExtensionPoint;
072    private SitePopulator _sitePopulator;
073    private SiteIndexer _siteIndexer;
074    private WorkspaceIndexer _workspaceIndexer;
075    private SiteManager _siteManager;
076    private SkinsManager _skinsManager;
077    private SolrIndexer _solrIndexer;
078    private SolrClientProvider _solrClientProvider;
079    private AmetysObjectResolver _resolver;
080    private FOCommHelper _foCommHelper;
081    
082    @Override
083    public void service(ServiceManager manager) throws ServiceException
084    {
085        _cacheManager = (AbstractCacheManager) manager.lookup(AbstractCacheManager.ROLE);
086        _siteIndexer = (SiteIndexer) manager.lookup(SiteIndexer.ROLE);
087        _repository = (Repository) manager.lookup(Repository.class.getName());
088        _livePopulatorExtensionPoint = (LivePopulatorExtensionPoint)  manager.lookup(LivePopulatorExtensionPoint.ROLE);
089        _sitePopulator = (SitePopulator) manager.lookup(SitePopulator.ROLE);
090        _workspaceIndexer = (WorkspaceIndexer) manager.lookup(WorkspaceIndexer.ROLE);
091        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
092        _skinsManager = (SkinsManager) manager.lookup(SkinsManager.ROLE);
093        _solrIndexer = (SolrIndexer) manager.lookup(SolrIndexer.ROLE);
094        _solrClientProvider = (SolrClientProvider) manager.lookup(SolrClientProvider.ROLE);
095        _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
096        _foCommHelper = (FOCommHelper) manager.lookup(FOCommHelper.ROLE);
097    }
098    
099    /**
100     * Rebuild live workspace, index all sitemaps and reset cache.
101     * @param reindex <code>true</code> to force reindexation at the end of the rebuild
102     * @param progressionTracker The tracker of the progression
103     * @throws Exception if an error occurs.
104     */
105    public void rebuildLiveWorkspace(boolean reindex, ContainerProgressionTracker progressionTracker) throws Exception
106    {
107        progressionTracker.addContainerStep("populate", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATE_LIVE_WORKSPACES_STEP_LABEL"));
108        if (reindex)
109        {
110            progressionTracker.addContainerStep("index", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_INDEXATION_LIVE_STEP_LABEL"));
111        }
112        progressionTracker.addSimpleStep("cache", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_CLEAR_SITE_CACHE_STEP_LABEL"));
113        
114        List<String> errors = new ArrayList<>();
115        
116        try
117        {
118            getLogger().info("Rebuilding live workspace started");
119
120            long t0 = System.currentTimeMillis();
121            
122            errors.addAll(_populateLiveWorkspace(progressionTracker.getStep("populate")));
123            
124            if (reindex)
125            {
126                errors.addAll(_reindexLiveWorkspace(progressionTracker.getStep("index")));
127            }
128            
129            errors.addAll(_clearAllSitesCache(progressionTracker.getStep("cache")));
130            
131            _clearApplicationCache();
132            
133            getLogger().info("Rebuilding live workspace ended in {} ms with {} error(s)", System.currentTimeMillis() - t0, errors.size());
134        }
135        catch (Exception e)
136        {
137            throw new Exception("Failed to rebuild live workspace", e);
138        }
139        
140        if (errors.size() > 0)
141        {
142            throw new Exception("Failed to rebuild live workspace with " + errors.size() + " error(s)\n\n--------------------------------------\n" + StringUtils.join(errors, "\n\n") + "\n--------------------------------------");
143        }
144    }
145    
146    private List<String> _populateLiveWorkspace(ContainerProgressionTracker progressionTracker)
147    {
148        return _populate(_livePopulatorExtensionPoint.getExtensionsIds(), progressionTracker);
149    }
150
151    private void _createProgressionTrackerStepsForPopulate(Set<String> populatorIds, ContainerProgressionTracker progressionTracker)
152    {
153        for (String populatorId : populatorIds)
154        {
155            LivePopulator populator = _livePopulatorExtensionPoint.getExtension(populatorId);
156            progressionTracker.addContainerStep(populatorId, new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATOR_STEP_LABEL", Map.of("0", populator.getLabel())));
157        }
158    }
159    
160    private List<String> _populate(Set<String> populatorIds, ContainerProgressionTracker progressionTracker)
161    {
162        _createProgressionTrackerStepsForPopulate(populatorIds, progressionTracker);
163        
164        List<String> errors = new ArrayList<>();
165
166        Session session = null;
167        Session liveSession = null;
168        
169        try
170        {
171            getLogger().info("Populating live workspace started");
172            
173            long t0 = System.currentTimeMillis();
174            
175            session = _repository.login();
176            liveSession = _repository.login(WebConstants.LIVE_WORKSPACE);
177
178            // Synchronize other data
179            for (String id : populatorIds)
180            {
181                _populate(id, session, liveSession, errors, progressionTracker.getStep(id));
182            }
183            
184            getLogger().info("Populating live workspace ended in {} ms", System.currentTimeMillis() - t0);
185        }
186        catch (Throwable t)
187        {
188            StringWriter sw = new StringWriter();
189            t.printStackTrace(new PrintWriter(sw));
190            
191            errors.add("Failed to populate live workspace\n" + sw.toString());
192            getLogger().error("Failed to populate live workspace", t);
193        }
194        finally
195        {
196            if (session != null)
197            {
198                session.logout();
199            }
200            
201            if (liveSession != null)
202            {
203                liveSession.logout();
204            }
205        }
206        
207        return errors;
208    }
209
210    private void _populate(String populatorId, Session session, Session liveSession, List<String> errors, ContainerProgressionTracker progressionTracker)
211    {
212        getLogger().info("Populating live workspace using LivePopulator {} started", populatorId);
213          
214        try
215        {
216            LivePopulator populator = _livePopulatorExtensionPoint.getExtension(populatorId);
217            
218            errors.addAll(populator.populate(session, liveSession, progressionTracker));
219            
220            if (liveSession.hasPendingChanges())
221            {
222                liveSession.save();
223            }
224        }
225        catch (Throwable t)
226        {
227            errors.add("Failed to populate live workspace with populator '" + populatorId + "'\n" + t.getMessage());
228            getLogger().error("Failed to populate live workspace with populator '" + populatorId + "'", t);
229        }
230        
231        getLogger().info("Populating live workspace using LivePopulator {} ended", populatorId);
232    }
233    
234    private List<String> _reindexLiveWorkspace(ContainerProgressionTracker progressionTracker)
235    {
236        List<String> errors = new ArrayList<>();
237        
238        try
239        {
240            getLogger().info("Indexing Live workspace started");
241            
242            long t0 = System.currentTimeMillis();
243            
244            // Index the whole live workspace and not just every site.
245            _workspaceIndexer.index(WebConstants.LIVE_WORKSPACE, progressionTracker);
246            
247            getLogger().info("Indexing live workspace ended in {} ms", System.currentTimeMillis() - t0);
248        }
249        catch (IndexingException e)
250        {
251            errors.add("Failed to index live workspace\n" + e.getMessage());
252            getLogger().error("Failed to index live workspace", e);
253        }
254        
255        return errors;
256    }
257    
258    private List<String> _clearAllSitesCache(SimpleProgressionTracker progressionTracker)
259    {
260        AmetysObjectIterable<Site> sites = _siteManager.getSites();
261        
262        progressionTracker.setSize(sites.getSize());
263        
264        List<String> errors = new ArrayList<>();
265        try (AmetysObjectIterable<Site> siteIterable = sites)
266        {
267            for (Site site : siteIterable)
268            {
269                try
270                {
271                    // TODO Clear only the live workspace?
272                    _clearSiteCache(site);
273                }
274                catch (Throwable t)
275                {
276                    errors.add("Failed to clear cache of site '" + site.getName() + "'\n" + t.getMessage());
277                    getLogger().error("Failed to clear cache of site '" + site.getName() + "'", t);
278                }
279                
280                progressionTracker.increment();
281            }
282        }
283        
284        return errors;
285    }
286
287    private void _clearSiteCache(Site site) throws Exception
288    {
289        String siteName = site.getName();
290        
291        getLogger().info("Clearing cache for site {} started", siteName);
292        
293        long t0 = System.currentTimeMillis();
294        
295        _foCommHelper.invalidateFOCache(site);
296        
297        getLogger().info("Clearing cache for site '{}' ended in {} ms", siteName, System.currentTimeMillis() - t0);
298    }
299    
300    private void _clearApplicationCache()
301    {
302        _cacheManager.resetAllMemoryCaches();
303    }
304
305    private void _createProgressionTrackerStepsForRebuildLive(String siteName, boolean reindex, ContainerProgressionTracker progressionTracker)
306    {
307        progressionTracker.addContainerStep("site", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATE_SITE_STEP_LABEL", Map.of("siteName", new I18nizableText(siteName))));
308        if (reindex)
309        {
310            progressionTracker.addContainerStep("indexation", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_INDEXATION_LIVE_STEP_LABEL"));
311        }
312        progressionTracker.addSimpleStep("sitecache", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_CLEAR_SITE_CACHE_STEP_LABEL"));
313    }
314    
315    /**
316     * Rebuild the live of a site, index all sitemaps and reset cache.
317     * @param site The site to be rebuilt
318     * @param reindex <code>true</code> to force reindexation at the end of the rebuild
319     * @param progressionTracker A progression tracker
320     * @throws Exception if an error occurs.
321     */
322    public void rebuildLive(Site site, boolean reindex, ContainerProgressionTracker progressionTracker) throws Exception
323    {
324        String siteName = "";
325        try
326        {
327            siteName = site.getName();
328            _createProgressionTrackerStepsForRebuildLive(siteName, reindex, progressionTracker);
329
330            Skin skin = _skinsManager.getSkin(site.getSkinId());
331            assert skin != null;
332            
333            getLogger().info("Rebuilding site {} started", siteName);
334
335            long t0 = System.currentTimeMillis();
336            
337            _populateSite(site, skin, progressionTracker.getStep("site"));
338            
339            if (reindex)
340            {
341                _reindexSite(site, progressionTracker.getStep("indexation"));
342            }
343            
344            _clearSiteCache(site);
345            ((SimpleProgressionTracker) progressionTracker.getStep("sitecache")).increment();
346            
347            _clearApplicationCache();
348            
349            getLogger().info("Rebuilding site {} ended in {} ms", siteName, System.currentTimeMillis() - t0);
350        }
351        catch (Exception e)
352        {
353            throw new Exception("Failed to rebuild live workspace for site '" + siteName + "'", e);
354        }
355    }
356    
357    private void _populateSite(Site site, Skin skin, ContainerProgressionTracker progressionTracker) throws Exception
358    {
359        String siteName = site.getName();
360
361        getLogger().info("Populating site {} started", siteName);
362        
363        long t0 = System.currentTimeMillis();
364        
365        _sitePopulator.populate(site, skin, progressionTracker);
366        
367        getLogger().info("Populating site {} ended in {} ms", siteName, System.currentTimeMillis() - t0);
368    }
369    
370    private void _reindexSite(Site site, ContainerProgressionTracker progressionTracker) throws Exception
371    {
372        String siteName = site.getName();
373        
374        getLogger().info("Indexing site '{}' started", siteName);
375        
376        long t0 = System.currentTimeMillis();
377        
378        // Reindex the site in the live workspace.
379        _siteIndexer.indexSite(site.getName(), WebConstants.LIVE_WORKSPACE, progressionTracker);
380        
381        getLogger().info("Indexing of site '{}' ended in {} ms", siteName, System.currentTimeMillis() - t0);
382    }
383    
384    private void _createProgressionTrackerStepsForRebuildContentsWithoutSiteWorkspace(boolean reindex, ContainerProgressionTracker progressionTracker)
385    {
386        progressionTracker.addContainerStep("populate", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_POPULATE_LIVE_WORKSPACES_STEP_LABEL"));
387        if (reindex)
388        {
389            progressionTracker.addContainerStep("index", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_REINDEX_CONTENTS_WITHOUT_SITES_STEP_LABEL"));
390        }
391        progressionTracker.addSimpleStep("cache", new I18nizableText("plugin.web", "PLUGINS_WEB_SCHEDULABLE_REBUILD_LIVE_CLEAR_SITE_CACHE_STEP_LABEL"));
392    }
393    
394    /**
395     * Rebuild the live workspace for contents without site.
396     * 
397     * More precisely, this method will synchronize and reindex all contents located
398     * at '/ametys:root/ametys:contents' and '/ametys:root/ametys:plugins/&lt;pluginName>/ametys:contents'
399     * @param reindex <code>true</code> to force reindexation at the end of the rebuild
400     * @param progressionTracker A progression tracker
401     * @throws Exception if an error occurred
402     */
403    public void rebuildContentsWithoutSiteWorkspace(boolean reindex, ContainerProgressionTracker progressionTracker) throws Exception
404    {
405        _createProgressionTrackerStepsForRebuildContentsWithoutSiteWorkspace(reindex, progressionTracker);
406        
407        List<String> errors = new ArrayList<>();
408        
409        try
410        {
411            getLogger().info("Rebuilding live workspace started");
412
413            long t0 = System.currentTimeMillis();
414
415            // call ContentsLivePopulator and PluginsLivePopulator
416            errors.addAll(_populate(Set.of(ContentsLivePopulator.class.getName(), PluginsLivePopulator.class.getName()), progressionTracker.getStep("populate")));
417            
418            if (reindex)
419            {
420                // reindex impacted contents
421                errors.addAll(_reindexContentWithoutSiteLiveWorkspace(progressionTracker.getStep("index")));
422            }
423            
424            // Clear every site's cache.
425            errors.addAll(_clearAllSitesCache(progressionTracker.getStep("cache")));
426            
427            // Clear application cache to make sur that no outdated info is kept in cache
428            _clearApplicationCache();
429            
430            getLogger().info("Rebuilding live workspace ended in {} ms with {} error(s)", System.currentTimeMillis() - t0, errors.size());
431        }
432        catch (Exception e)
433        {
434            throw new Exception("Failed to rebuild live workspace", e);
435        }
436        
437        if (errors.size() > 0)
438        {
439            throw new Exception("Failed to rebuild live workspace with " + errors.size() + " error(s)\n\n--------------------------------------\n" + StringUtils.join(errors, "\n\n") + "\n--------------------------------------");
440        }
441    }
442
443    private void _createProgressionTrackerStepsForReindexContentWithoutSiteLiveWorkspace(ContainerProgressionTracker progressionTracker)
444    {
445        boolean createNullTracker = true;
446        try (AmetysObjectIterable<AmetysObjectCollection> roots = _getContentsWithoutSiteRootNodes())
447        {
448            if (roots.getSize() != 0)
449            {
450                createNullTracker = false;
451            }
452
453            for (AmetysObjectCollection root : roots)
454            {
455                progressionTracker.addSimpleStep(root.getPath(), new I18nizableText(root.getPath()));
456            }
457        }
458        
459        if (createNullTracker)
460        {
461            // Create a sub step just to set the progression of the parent to 100%
462            progressionTracker.addSimpleStep("-", new I18nizableText("")).setSize(0);
463        }
464    }
465    
466    private Collection< ? extends String> _reindexContentWithoutSiteLiveWorkspace(ContainerProgressionTracker progressionTracker) throws Exception
467    {
468        _createProgressionTrackerStepsForReindexContentWithoutSiteLiveWorkspace(progressionTracker);
469
470        List<String> errors = new ArrayList<>();
471        
472        try
473        {
474            getLogger().info("Indexing contents without site live workspace started");
475            
476            long t0 = System.currentTimeMillis();
477
478            // Do not close the client, this is the provider responsibility
479            SolrClient solrClient = _solrClientProvider.getUpdateClient(WebConstants.LIVE_WORKSPACE, false);
480            
481            try (AmetysObjectIterable<AmetysObjectCollection> roots = _getContentsWithoutSiteRootNodes())
482            {
483                for (AmetysObjectCollection root : roots)
484                {
485                    getLogger().info("Indexing live contents at path {}", root.getPath());
486                    
487                    try (AmetysObjectIterable<Content> contents = root.getChildren())
488                    {
489                        _solrIndexer.indexContents(contents, WebConstants.LIVE_WORKSPACE, true, solrClient, progressionTracker.getStep(root.getPath()));
490                        _solrIndexer.commit(WebConstants.LIVE_WORKSPACE, solrClient);
491                    }
492                    
493                    getLogger().info("Indexing live contents at path {} ended", root.getPath());
494                }
495            }
496            
497            getLogger().info("Indexing contents witout site live workspace ended in {} ms", System.currentTimeMillis() - t0);
498        }
499        catch (IndexingException e)
500        {
501            errors.add("Failed to index contents without site live workspace\n" + e.getMessage());
502            getLogger().error("Failed to index contents without site live workspace", e);
503        }
504        
505        return errors;
506    }
507
508    private AmetysObjectIterable<AmetysObjectCollection> _getContentsWithoutSiteRootNodes()
509    {
510        // contents nodes in plugins
511        AmetysObjectIterable<AmetysObjectCollection> roots = _resolver.<AmetysObjectCollection>query("/jcr:root/ametys:root/ametys:plugins/*/ametys:contents");
512        // contents node at ametys root
513        AmetysObjectIterable<AmetysObjectCollection> rootIterable = new CollectionIterable<>(List.of(_resolver.resolveByPath("/ametys:contents")));
514        
515        // Return an iterable so that the consumer is responsible for closing
516        return new ChainedAmetysObjectIterable<>(List.of(rootIterable, roots));
517    }
518}