Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.statistics;
017
018import java.util.ArrayList;
019import java.util.Comparator;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Map;
023import java.util.Map.Entry;
024import java.util.function.Function;
025import java.util.stream.Collectors;
026
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang3.StringUtils;
031
032import org.ametys.cms.repository.Content;
033import org.ametys.core.util.I18nUtils;
034import org.ametys.plugins.explorer.resources.Resource;
035import org.ametys.plugins.explorer.resources.ResourceCollection;
036import org.ametys.plugins.repository.AmetysObject;
037import org.ametys.plugins.repository.AmetysObjectIterable;
038import org.ametys.plugins.repository.AmetysObjectResolver;
039import org.ametys.plugins.repository.TraversableAmetysObject;
040import org.ametys.runtime.config.Config;
041import org.ametys.runtime.i18n.I18nizableText;
042import org.ametys.runtime.plugin.component.AbstractLogEnabled;
043import org.ametys.runtime.plugin.component.PluginAware;
044import org.ametys.runtime.plugins.admin.statistics.Statistics;
045import org.ametys.runtime.plugins.admin.statistics.StatisticsNode;
046import org.ametys.runtime.plugins.admin.statistics.StatisticsProvider;
047import org.ametys.runtime.plugins.admin.statistics.StatisticsValue;
048import org.ametys.web.cache.PageHelper;
049import org.ametys.web.gdpr.GDPRComponentEnumerator;
050import org.ametys.web.repository.content.SharedContent;
051import org.ametys.web.repository.content.WebContent;
052import org.ametys.web.repository.page.Page;
053import org.ametys.web.repository.page.SitemapElement;
054import org.ametys.web.repository.page.Zone;
055import org.ametys.web.repository.page.ZoneItem;
056import org.ametys.web.repository.site.Site;
057import org.ametys.web.repository.site.SiteManager;
058import org.ametys.web.repository.sitemap.Sitemap;
059import org.ametys.web.service.Service;
060import org.ametys.web.service.ServiceExtensionPoint;
061import org.ametys.web.site.SiteConfigurationManager;
062import org.ametys.web.skin.SkinsManager;
063
064/**
065 * Web statistics
066 */
067public class WebStatisticsProvider extends AbstractLogEnabled implements StatisticsProvider, Serviceable, PluginAware
068{
069    private String _id;
070    private SiteConfigurationManager _siteConfigurationManager;
071    private SiteManager _siteManager;
072    private PageHelper _pageHelper;
073    private ServiceExtensionPoint _serviceEP;
074    private I18nUtils _i18nUtils;
075    private AmetysObjectResolver _ametysResolver;
076    private SkinsManager _skinsManager;
077    private GDPRComponentEnumerator _gDPRComponentEnumerator;
078
079    public void service(ServiceManager manager) throws ServiceException
080    {
081        _siteConfigurationManager = (SiteConfigurationManager) manager.lookup(SiteConfigurationManager.ROLE);
082        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
083        _pageHelper = (PageHelper) manager.lookup(PageHelper.ROLE);
084        _serviceEP = (ServiceExtensionPoint) manager.lookup(ServiceExtensionPoint.ROLE);
085        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
086        _ametysResolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
087        _skinsManager = (SkinsManager) manager.lookup(SkinsManager.ROLE);
088        _gDPRComponentEnumerator = (GDPRComponentEnumerator) manager.lookup(GDPRComponentEnumerator.ROLE);
089    }
090    
091    public void setPluginInfo(String pluginName, String featureName, String id)
092    {
093        _id = id;
094    }
095    
096    public Statistics getStatistics()
097    {
098        SitesInfos sitesInfos = _getSitesInfos();
099        List<List<Map<String, Long>>> countPages = sitesInfos.sitesPages();
100        int siteCount = countPages.size();
101        int sitemapCount = countPages.stream().map(List<Map<String, Long>>::size).reduce(0, Integer::sum);
102        List<Long> nbPagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBPAGES")).sorted().toList();
103        long pagesCount = nbPagesPerSitemap.stream().reduce(0L, Long::sum);
104        List<Long> nbContentsRefPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("CONTENTS")).sorted().toList();
105        long contentsRefsCount = nbContentsRefPerSitemap.stream().reduce(0L, Long::sum);
106        List<Long> nbServicesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("SERVICES")).sorted().toList();
107        long servicesCount = nbServicesPerSitemap.stream().reduce(0L, Long::sum);
108        List<Long> nbCacheablePagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBCACHEABLE")).sorted().toList();
109        long cacheablePagesCount = nbCacheablePagesPerSitemap.stream().reduce(0L, Long::sum);
110        
111        Map<String, Long> contentsForSites = _processContentsForSites();
112        
113        List<Statistics> skins = _skinsManager.getResourceSkins()
114                                              .stream()
115                                              .sorted(Comparator.naturalOrder())
116                                              .map(skinName -> new StatisticsValue(skinName,
117                                                                                   new I18nizableText(skinName),
118                                                                                   "ametysicon-puzzle-piece1",
119                                                                                   _skinsManager.getVersion(skinName)))
120                                              .map(Statistics.class::cast)
121                                              .toList();
122
123        
124        return new StatisticsNode(
125            _id,
126            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_LABEL"),
127            "ametysicon-code-html-link",
128            null,
129            List.of(
130                new StatisticsNode(
131                    "sites",
132                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_LABEL"),
133                    "ametysicon-world-earth-black",
134                    siteCount,
135                    List.of(
136                        new StatisticsNode(
137                            "sitemaps",
138                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_LABEL"),
139                            "ametysicon-world-flag",
140                            sitemapCount,
141                            List.of(
142                                new StatisticsNode(
143                                    "pages",
144                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_LABEL"),
145                                    "ametysicon-website38",
146                                    pagesCount,
147                                    List.of(
148                                        new StatisticsValue(
149                                            "max",
150                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MAX_LABEL"),
151                                            "ametysicon-sort51",
152                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() - 1) : 0
153                                        ),
154                                        new StatisticsValue(
155                                            "median",
156                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MEDIAN_LABEL"),
157                                            "ametysicon-maths-window-symbol-x",
158                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() / 2) : 0
159                                        ),
160                                        new StatisticsValue(
161                                            "cacheable",
162                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_CACHEABLE_LABEL"),
163                                            "ametysicon-desktop-archive",
164                                            cacheablePagesCount
165                                        )
166                                    ),
167                                    false
168                                ),
169                                new StatisticsNode(
170                                    "content-references",
171                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_LABEL"),
172                                    "ametysicon-text70",
173                                    contentsRefsCount,
174                                    List.of(
175                                        new StatisticsValue(
176                                            "max",
177                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MAX_LABEL"),
178                                            "ametysicon-sort51",
179                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() - 1) : 0
180                                        ),
181                                        new StatisticsValue(
182                                            "median",
183                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MEDIAN_LABEL"),
184                                            "ametysicon-maths-window-symbol-x",
185                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() / 2) : 0
186                                        )
187                                    ),
188                                    false
189                                ),
190                                new StatisticsNode(
191                                    "service-references",
192                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_LABEL"),
193                                    "ametysicon-system-monitoring",
194                                    servicesCount,
195                                    List.of(
196                                        new StatisticsValue(
197                                            "max",
198                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MAX_LABEL"),
199                                            "ametysicon-sort51",
200                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() - 1) : 0
201                                        ),
202                                        new StatisticsValue(
203                                            "median",
204                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MEDIAN_LABEL"),
205                                            "ametysicon-maths-window-symbol-x",
206                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() / 2) : 0
207                                        )
208                                    ),
209                                    false
210                                )
211                            ),
212                            false
213                        ),
214                        new StatisticsNode(
215                            "contents",
216                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_LABEL"),
217                            "ametysicon-text70",
218                            contentsForSites.get("COUNT"),
219                            List.of(
220                                new StatisticsValue(
221                                    "orphans",
222                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_ORPHANS_LABEL"),
223                                    "ametysicon-sort51",
224                                    contentsForSites.get("ORPHANS")
225                                ),
226                                new StatisticsValue(
227                                    "shared",
228                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_EXTERNAL_LABEL"),
229                                    "ametysicon-maths-window-symbol-x",
230                                    contentsForSites.get("EXTERNAL")
231                                )
232                            ),
233                            false
234                        ),
235                        new StatisticsNode(
236                            "services",
237                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_LABEL"),
238                            "ametysicon-system-monitoring",
239                            null,
240                            _getServices(),
241                            false
242                        ),
243                        _processResources(),
244                        new StatisticsNode(
245                            "skins",
246                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SKINS_LABEL"),
247                            "ametysicon-puzzle-piece1",
248                            null,
249                            skins,
250                            false
251                        ),
252                        new StatisticsNode(
253                                "gdpr",
254                                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_GDPR_LABEL"),
255                                "ametysicon-desktop-clipboard-list",
256                                _getGDPRGlobal(),
257                                _getGDPRBySite(),
258                                false
259                        )
260                    ),
261                    true
262                ),
263                new StatisticsValue(
264                        "unavailable-sites",
265                        new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_UNAVAILABLE_SITES_LABEL"),
266                        "ametysicon-sign-caution",
267                        sitesInfos.unavailableSites()
268                    )
269            ),
270            true
271        );
272    }
273
274    private SitesInfos _getSitesInfos()
275    {
276        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
277        {
278            List<List<Map<String, Long>>> sitesValues = new ArrayList<>();
279            int unavailableSites = 0;
280            for (Site site : sites)
281            {
282                try
283                {
284                    // most operation on site and sitemap element expect a valid site
285                    // ignore invalid site to prevent errors
286                    if (_siteConfigurationManager.isSiteConfigurationValid(site))
287                    {
288                        sitesValues.add(_countPages(site));
289                    }
290                    else
291                    {
292                        unavailableSites++;
293                    }
294                }
295                catch (Exception e)
296                {
297                    getLogger().warn("Failed to compute statistics for site '{}', the site will be considered unavailable", e);
298                    unavailableSites++;
299                }
300            }
301            
302            return new SitesInfos(sitesValues, unavailableSites);
303        }
304    }
305    
306    private record SitesInfos(List<List<Map<String, Long>>> sitesPages, int unavailableSites) { }
307    
308    private List<Map<String, Long>> _countPages(Site site)
309    {
310        List<Map<String, Long>> siteValues = new ArrayList<>();
311
312        for (Sitemap sitemap : site.getSitemaps())
313        {
314            siteValues.add(_countPages(sitemap));
315        }
316        
317        return siteValues;
318    }
319    
320    private Map<String, Long> _countPages(SitemapElement pages)
321    {
322        Map<String, Long> sitemapElementValues = _newCountMap();
323        
324        AmetysObjectIterable<? extends Page> childPages = pages.getChildrenPages();
325        
326        sitemapElementValues.put("NBPAGES", sitemapElementValues.get("NBPAGES") + childPages.getSize());
327        
328        int cacheableCount = 0;
329        
330        for (Page page : childPages)
331        {
332            boolean isCacheable = _pageHelper.isCacheable(page);
333            for (Zone zone : page.getZones())
334            {
335                AmetysObjectIterable<? extends ZoneItem> zoneItems = zone.getZoneItems();
336                for (ZoneItem item : zoneItems)
337                {
338                    switch (item.getType())
339                    {
340                        case CONTENT:
341                            sitemapElementValues.put("CONTENTS", sitemapElementValues.get("CONTENTS") + 1);
342                            break;
343                        case SERVICE:
344                            sitemapElementValues.put("SERVICES", sitemapElementValues.get("SERVICES") + 1);
345                            break;
346                        default:
347                            break;
348                    }
349                }
350            }
351            
352            if (isCacheable)
353            {
354                cacheableCount++;
355            }
356            
357            _sumMaps(sitemapElementValues, _countPages(page));
358        }
359        
360        sitemapElementValues.put("NBCACHEABLE", sitemapElementValues.get("NBCACHEABLE") + cacheableCount);
361        
362        return sitemapElementValues;
363    }
364    
365    
366    private Map<String, Long> _newCountMap()
367    {
368        Map<String, Long> countMap = new HashMap<>();
369        countMap.put("NBPAGES", 0L); // Number of pages
370        countMap.put("NBCACHEABLE", 0L); // Number of cacheable pages
371        countMap.put("SERVICES", 0L); // Number of services
372        countMap.put("CONTENTS", 0L); // Number of contents
373        return countMap;
374    }
375    
376    private void _sumMaps(Map<String, Long> mainMap, Map<String, Long> mapToAdd)
377    {
378        for (String k : mainMap.keySet())
379        {
380            mainMap.put(k, mainMap.get(k) + mapToAdd.get(k));
381        }
382    }
383    
384    private List<Statistics> _getServices()
385    {
386        List<Statistics> stats = new ArrayList<>();
387        
388        for (String serviceId : _serviceEP.getExtensionsIds())
389        {
390            Service service = _serviceEP.getExtension(serviceId);
391            
392            stats.add(new StatisticsValue(
393                serviceId,
394                new I18nizableText(_i18nUtils.translate(service.getLabel())), // Translate now for sorting
395                StringUtils.defaultIfBlank(service.getIconGlyph(), "ametysicon-system-monitoring"),
396                _countAllServices(serviceId)
397            ));
398        }
399        
400        stats.sort(new Comparator<>() {
401            public int compare(Statistics o1, Statistics o2)
402            {
403                return o1.getLabel().getLabel().compareTo(o2.getLabel().getLabel());
404            }
405        });
406        
407        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
408                + _serviceEP.getExtensionsIds().stream().map(id -> " and not(@ametys-internal:service='" + id + "')").collect(Collectors.joining())
409                + "]";
410        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
411        {
412            stats.add(0, new StatisticsValue(
413                "unknown",
414                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_UNKNOWN_LABEL"),
415                "ametysicon-file-extension-generic-unknown",
416                serviceZoneItems.getSize()
417            ));
418        }
419        
420        return stats;
421    }
422
423    private long _countAllServices(String serviceId)
424    {
425        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
426                + (serviceId != null ? " and @ametys-internal:service='" + serviceId + "'" : "")
427                + "]";
428        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
429        {
430            return serviceZoneItems.getSize();
431        }
432    }
433    
434    private Map<String, Long> _processContentsForSites()
435    {
436        Map<String, Long> values = new HashMap<>();
437        values.put("COUNT", 0L);
438        values.put("ORPHANS", 0L);
439        values.put("EXTERNAL", 0L);
440        
441        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
442        {
443            for (Site site : sites)
444            {
445                _processContents (site, values);
446            }
447        }
448        catch (Exception e)
449        {
450            throw new IllegalStateException(e);
451        }
452        
453        return values;
454    }
455    
456    private void _processContents (Site site, Map<String, Long> values)
457    {
458        // Number of contents
459        AmetysObjectIterable<Content> contents = site.getContents();
460        values.put("COUNT", values.get("COUNT") + contents.getSize());
461        
462        int orphans = 0;
463        int external = 0;
464        for (Content content : contents)
465        {
466            if (content instanceof WebContent webContent
467                && webContent.getReferencingPages().size() == 0)
468            {
469                orphans++;
470            }
471            if (content instanceof SharedContent)
472            {
473                external++;
474            }
475        }
476        
477        // Number of orphan contents
478        values.put("ORPHANS", values.get("ORPHANS") + orphans);
479        
480        // Number of shared contents
481        values.put("EXTERNAL", values.get("EXTERNAL") + external);
482    }
483    
484    private Statistics _processResources()
485    {
486        HashMap<String, Long> values = new HashMap<>();
487        values.put("FOLDERCOUNT", 0L);
488        values.put("RESOURCECOUNT", 0L);
489        values.put("TOTALSIZE", 0L);
490        
491        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
492        {
493            for (Site site : sites)
494            {
495                _processResources (site.getRootResources(), values);
496            }
497        }
498        
499        return new StatisticsNode(
500            "explorer",
501            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_LABEL"),
502            "ametysicon-folder249",
503            values.get("TOTALSIZE"),
504            List.of(
505                new StatisticsValue(
506                    "folders",
507                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FOLDER_LABEL"),
508                    "ametysicon-folder250",
509                    values.get("FOLDERCOUNT")
510                ),
511                new StatisticsValue(
512                    "files",
513                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FILES_LABEL"),
514                    "ametysicon-document77",
515                    values.get("RESOURCECOUNT")
516                )
517            ),
518            false
519        );
520    }
521    
522    private void _processResources(TraversableAmetysObject resourceContainer, HashMap<String, Long> values)
523    {
524        AmetysObjectIterable<? extends AmetysObject> objects = resourceContainer.getChildren();
525        
526        for (AmetysObject object : objects)
527        {
528            // Traverse the child nodes if depth < 0 (generate all) or depth > 0 (we're not in the last level).
529            if (object instanceof ResourceCollection)
530            {
531                values.put("FOLDERCOUNT", values.get("FOLDERCOUNT") + 1);
532                
533                _processResources((ResourceCollection) object, values);
534            }
535            else if (object instanceof Resource)
536            {
537                Resource resource = (Resource) object;
538                
539                values.put("RESOURCECOUNT", values.get("RESOURCECOUNT") + 1);
540                values.put("TOTALSIZE", values.get("TOTALSIZE") + resource.getLength());
541            }
542        }
543    }
544
545    private I18nizableText _getGDPRGlobal()
546    {
547        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
548        {
549            return _gDPRComponentEnumerator.getEntry(Config.getInstance().getValue("plugins.web.gdpr.choice"));
550        }
551        catch (Exception e)
552        {
553            throw new IllegalStateException(e);
554        }
555    }
556    
557    private List<Statistics> _getGDPRBySite()
558    {
559        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
560        {
561            List<Statistics> gdpr = new ArrayList<>();
562
563            Map<String, Long> gdprChoiceNumber = sites.stream()
564                    .map(site -> (String) (site.hasValue("gdpr-component-choice") ? site.getValue("gdpr-component-choice") : "_ametys_general_configuration"))
565                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
566
567            gdpr.add(new StatisticsValue("_ametys_general_configuration",
568                    new I18nizableText("plugin.web", "PLUGINS_WEB_SITE_COOKIES_GENERAL_CONFIGURATION"),
569                    "ametysicon-system-monitoring",
570                    gdprChoiceNumber.getOrDefault("_ametys_general_configuration", 0L)));
571            for (Entry<String, I18nizableText> entry : _gDPRComponentEnumerator.getEntries().entrySet())
572            {
573                gdpr.add(new StatisticsValue(entry.getKey(),
574                        entry.getValue(),
575                        "ametysicon-system-monitoring",
576                        gdprChoiceNumber.getOrDefault(entry.getKey(), 0L)));
577            }
578            return gdpr;
579        }
580        catch (Exception e)
581        {
582            throw new IllegalStateException(e);
583        }
584        
585    }
586}