Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.statistics;
017
018import java.util.ArrayList;
019import java.util.Comparator;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Map;
023import java.util.Map.Entry;
024import java.util.function.Function;
025import java.util.stream.Collectors;
026
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang3.StringUtils;
031
032import org.ametys.cms.repository.Content;
033import org.ametys.core.util.I18nUtils;
034import org.ametys.plugins.explorer.resources.Resource;
035import org.ametys.plugins.explorer.resources.ResourceCollection;
036import org.ametys.plugins.repository.AmetysObject;
037import org.ametys.plugins.repository.AmetysObjectIterable;
038import org.ametys.plugins.repository.AmetysObjectResolver;
039import org.ametys.plugins.repository.TraversableAmetysObject;
040import org.ametys.runtime.config.Config;
041import org.ametys.runtime.i18n.I18nizableText;
042import org.ametys.runtime.plugin.component.PluginAware;
043import org.ametys.runtime.plugins.admin.statistics.Statistics;
044import org.ametys.runtime.plugins.admin.statistics.StatisticsNode;
045import org.ametys.runtime.plugins.admin.statistics.StatisticsProvider;
046import org.ametys.runtime.plugins.admin.statistics.StatisticsValue;
047import org.ametys.web.cache.PageHelper;
048import org.ametys.web.gdpr.GDPRComponentEnumerator;
049import org.ametys.web.repository.content.SharedContent;
050import org.ametys.web.repository.content.WebContent;
051import org.ametys.web.repository.page.Page;
052import org.ametys.web.repository.page.SitemapElement;
053import org.ametys.web.repository.page.Zone;
054import org.ametys.web.repository.page.ZoneItem;
055import org.ametys.web.repository.site.Site;
056import org.ametys.web.repository.site.SiteManager;
057import org.ametys.web.repository.sitemap.Sitemap;
058import org.ametys.web.service.Service;
059import org.ametys.web.service.ServiceExtensionPoint;
060import org.ametys.web.site.SiteConfigurationManager;
061import org.ametys.web.skin.SkinsManager;
062
063/**
064 * Web statistics
065 */
066public class WebStatisticsProvider implements StatisticsProvider, Serviceable, PluginAware
067{
068    private String _id;
069    private SiteConfigurationManager _siteConfigurationManager;
070    private SiteManager _siteManager;
071    private PageHelper _pageHelper;
072    private ServiceExtensionPoint _serviceEP;
073    private I18nUtils _i18nUtils;
074    private AmetysObjectResolver _ametysResolver;
075    private SkinsManager _skinsManager;
076    private GDPRComponentEnumerator _gDPRComponentEnumerator;
077
078    public void service(ServiceManager manager) throws ServiceException
079    {
080        _siteConfigurationManager = (SiteConfigurationManager) manager.lookup(SiteConfigurationManager.ROLE);
081        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
082        _pageHelper = (PageHelper) manager.lookup(PageHelper.ROLE);
083        _serviceEP = (ServiceExtensionPoint) manager.lookup(ServiceExtensionPoint.ROLE);
084        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
085        _ametysResolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
086        _skinsManager = (SkinsManager) manager.lookup(SkinsManager.ROLE);
087        _gDPRComponentEnumerator = (GDPRComponentEnumerator) manager.lookup(GDPRComponentEnumerator.ROLE);
088    }
089    
090    public void setPluginInfo(String pluginName, String featureName, String id)
091    {
092        _id = id;
093    }
094    
095    public Statistics getStatistics()
096    {
097        SitesInfos sitesInfos = _getSitesInfos();
098        List<List<Map<String, Long>>> countPages = sitesInfos.sitesPages();
099        int siteCount = countPages.size();
100        int sitemapCount = countPages.stream().map(List<Map<String, Long>>::size).reduce(0, Integer::sum);
101        List<Long> nbPagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBPAGES")).sorted().toList();
102        long pagesCount = nbPagesPerSitemap.stream().reduce(0L, Long::sum);
103        List<Long> nbContentsRefPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("CONTENTS")).sorted().toList();
104        long contentsRefsCount = nbContentsRefPerSitemap.stream().reduce(0L, Long::sum);
105        List<Long> nbServicesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("SERVICES")).sorted().toList();
106        long servicesCount = nbServicesPerSitemap.stream().reduce(0L, Long::sum);
107        List<Long> nbCacheablePagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBCACHEABLE")).sorted().toList();
108        long cacheablePagesCount = nbCacheablePagesPerSitemap.stream().reduce(0L, Long::sum);
109        
110        Map<String, Long> contentsForSites = _processContentsForSites();
111        
112        List<Statistics> skins = _skinsManager.getResourceSkins()
113                                              .stream()
114                                              .sorted(Comparator.naturalOrder())
115                                              .map(skinName -> new StatisticsValue(skinName,
116                                                                                   new I18nizableText(skinName),
117                                                                                   "ametysicon-puzzle-piece1",
118                                                                                   _skinsManager.getVersion(skinName)))
119                                              .map(Statistics.class::cast)
120                                              .toList();
121
122        
123        return new StatisticsNode(
124            _id,
125            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_LABEL"),
126            "ametysicon-code-html-link",
127            null,
128            List.of(
129                new StatisticsNode(
130                    "sites",
131                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_LABEL"),
132                    "ametysicon-world-earth-black",
133                    siteCount,
134                    List.of(
135                        new StatisticsNode(
136                            "sitemaps",
137                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_LABEL"),
138                            "ametysicon-world-flag",
139                            sitemapCount,
140                            List.of(
141                                new StatisticsNode(
142                                    "pages",
143                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_LABEL"),
144                                    "ametysicon-website38",
145                                    pagesCount,
146                                    List.of(
147                                        new StatisticsValue(
148                                            "max",
149                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MAX_LABEL"),
150                                            "ametysicon-sort51",
151                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() - 1) : 0
152                                        ),
153                                        new StatisticsValue(
154                                            "median",
155                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MEDIAN_LABEL"),
156                                            "ametysicon-maths-window-symbol-x",
157                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() / 2) : 0
158                                        ),
159                                        new StatisticsValue(
160                                            "cacheable",
161                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_CACHEABLE_LABEL"),
162                                            "ametysicon-desktop-archive",
163                                            cacheablePagesCount
164                                        )
165                                    ),
166                                    false
167                                ),
168                                new StatisticsNode(
169                                    "content-references",
170                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_LABEL"),
171                                    "ametysicon-text70",
172                                    contentsRefsCount,
173                                    List.of(
174                                        new StatisticsValue(
175                                            "max",
176                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MAX_LABEL"),
177                                            "ametysicon-sort51",
178                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() - 1) : 0
179                                        ),
180                                        new StatisticsValue(
181                                            "median",
182                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MEDIAN_LABEL"),
183                                            "ametysicon-maths-window-symbol-x",
184                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() / 2) : 0
185                                        )
186                                    ),
187                                    false
188                                ),
189                                new StatisticsNode(
190                                    "content-references",
191                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_LABEL"),
192                                    "ametysicon-text70",
193                                    servicesCount,
194                                    List.of(
195                                        new StatisticsValue(
196                                            "max",
197                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MAX_LABEL"),
198                                            "ametysicon-sort51",
199                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() - 1) : 0
200                                        ),
201                                        new StatisticsValue(
202                                            "median",
203                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MEDIAN_LABEL"),
204                                            "ametysicon-maths-window-symbol-x",
205                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() / 2) : 0
206                                        )
207                                    ),
208                                    false
209                                )
210                            ),
211                            false
212                        ),
213                        new StatisticsNode(
214                            "contents",
215                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_LABEL"),
216                            "ametysicon-text70",
217                            contentsForSites.get("COUNT"),
218                            List.of(
219                                new StatisticsValue(
220                                    "orphans",
221                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_ORPHANS_LABEL"),
222                                    "ametysicon-sort51",
223                                    contentsForSites.get("ORPHANS")
224                                ),
225                                new StatisticsValue(
226                                    "shared",
227                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_EXTERNAL_LABEL"),
228                                    "ametysicon-maths-window-symbol-x",
229                                    contentsForSites.get("EXTERNAL")
230                                )
231                            ),
232                            false
233                        ),
234                        new StatisticsNode(
235                            "services",
236                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_LABEL"),
237                            "ametysicon-system-monitoring",
238                            null,
239                            _getServices(),
240                            false
241                        ),
242                        _processResources(),
243                        new StatisticsNode(
244                            "skins",
245                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SKINS_LABEL"),
246                            "ametysicon-puzzle-piece1",
247                            null,
248                            skins,
249                            false
250                        ),
251                        new StatisticsNode(
252                                "gdpr",
253                                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_GDPR_LABEL"),
254                                "ametysicon-desktop-clipboard-list",
255                                _getGDPRGlobal(),
256                                _getGDPRBySite(),
257                                false
258                        )
259                    ),
260                    true
261                ),
262                new StatisticsValue(
263                        "unavailable-sites",
264                        new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_UNAVAILABLE_SITES_LABEL"),
265                        "ametysicon-sign-caution",
266                        sitesInfos.unavailableSites()
267                    )
268            ),
269            true
270        );
271    }
272
273    private SitesInfos _getSitesInfos()
274    {
275        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
276        {
277            List<List<Map<String, Long>>> sitesValues = new ArrayList<>();
278            int unavailableSites = 0;
279            for (Site site : sites)
280            {
281                // most operation on site and sitemap element expect a valid site
282                // ignore invalid site to prevent errors
283                if (_siteConfigurationManager.isSiteConfigurationValid(site))
284                {
285                    sitesValues.add(_countPages(site));
286                }
287                else
288                {
289                    unavailableSites++;
290                }
291            }
292            
293            return new SitesInfos(sitesValues, unavailableSites);
294        }
295    }
296    
297    private record SitesInfos(List<List<Map<String, Long>>> sitesPages, int unavailableSites) { }
298    
299    private List<Map<String, Long>> _countPages(Site site)
300    {
301        List<Map<String, Long>> siteValues = new ArrayList<>();
302
303        for (Sitemap sitemap : site.getSitemaps())
304        {
305            siteValues.add(_countPages(sitemap));
306        }
307        
308        return siteValues;
309    }
310    
311    private Map<String, Long> _countPages(SitemapElement pages)
312    {
313        Map<String, Long> sitemapElementValues = _newCountMap();
314        
315        AmetysObjectIterable<? extends Page> childPages = pages.getChildrenPages();
316        
317        sitemapElementValues.put("NBPAGES", sitemapElementValues.get("NBPAGES") + childPages.getSize());
318        
319        int cacheableCount = 0;
320        
321        for (Page page : childPages)
322        {
323            boolean isCacheable = _pageHelper.isCacheable(page);
324            for (Zone zone : page.getZones())
325            {
326                AmetysObjectIterable<? extends ZoneItem> zoneItems = zone.getZoneItems();
327                for (ZoneItem item : zoneItems)
328                {
329                    switch (item.getType())
330                    {
331                        case CONTENT:
332                            sitemapElementValues.put("CONTENTS", sitemapElementValues.get("CONTENTS") + 1);
333                            break;
334                        case SERVICE:
335                            sitemapElementValues.put("SERVICES", sitemapElementValues.get("SERVICES") + 1);
336                            break;
337                        default:
338                            break;
339                    }
340                }
341            }
342            
343            if (isCacheable)
344            {
345                cacheableCount++;
346            }
347            
348            _sumMaps(sitemapElementValues, _countPages(page));
349        }
350        
351        sitemapElementValues.put("NBCACHEABLE", sitemapElementValues.get("NBCACHEABLE") + cacheableCount);
352        
353        return sitemapElementValues;
354    }
355    
356    
357    private Map<String, Long> _newCountMap()
358    {
359        Map<String, Long> countMap = new HashMap<>();
360        countMap.put("NBPAGES", 0L); // Number of pages
361        countMap.put("NBCACHEABLE", 0L); // Number of cacheable pages
362        countMap.put("SERVICES", 0L); // Number of services
363        countMap.put("CONTENTS", 0L); // Number of contents
364        return countMap;
365    }
366    
367    private void _sumMaps(Map<String, Long> mainMap, Map<String, Long> mapToAdd)
368    {
369        for (String k : mainMap.keySet())
370        {
371            mainMap.put(k, mainMap.get(k) + mapToAdd.get(k));
372        }
373    }
374    
375    private List<Statistics> _getServices()
376    {
377        List<Statistics> stats = new ArrayList<>();
378        
379        for (String serviceId : _serviceEP.getExtensionsIds())
380        {
381            Service service = _serviceEP.getExtension(serviceId);
382            
383            stats.add(new StatisticsValue(
384                serviceId,
385                new I18nizableText(_i18nUtils.translate(service.getLabel())), // Translate now for sorting
386                StringUtils.defaultIfBlank(service.getIconGlyph(), "ametysicon-system-monitoring"),
387                _countAllServices(serviceId)
388            ));
389        }
390        
391        stats.sort(new Comparator<>() {
392            public int compare(Statistics o1, Statistics o2)
393            {
394                return o1.getLabel().getLabel().compareTo(o2.getLabel().getLabel());
395            }
396        });
397        
398        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
399                + _serviceEP.getExtensionsIds().stream().map(id -> " and not(@ametys-internal:service='" + id + "')").collect(Collectors.joining())
400                + "]";
401        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
402        {
403            stats.add(0, new StatisticsValue(
404                "unknown",
405                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_UNKNOWN_LABEL"),
406                "ametysicon-file-extension-generic-unknown",
407                serviceZoneItems.getSize()
408            ));
409        }
410        
411        return stats;
412    }
413
414    private long _countAllServices(String serviceId)
415    {
416        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
417                + (serviceId != null ? " and @ametys-internal:service='" + serviceId + "'" : "")
418                + "]";
419        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
420        {
421            return serviceZoneItems.getSize();
422        }
423    }
424    
425    private Map<String, Long> _processContentsForSites()
426    {
427        Map<String, Long> values = new HashMap<>();
428        values.put("COUNT", 0L);
429        values.put("ORPHANS", 0L);
430        values.put("EXTERNAL", 0L);
431        
432        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
433        {
434            for (Site site : sites)
435            {
436                _processContents (site, values);
437            }
438        }
439        catch (Exception e)
440        {
441            throw new IllegalStateException(e);
442        }
443        
444        return values;
445    }
446    
447    private void _processContents (Site site, Map<String, Long> values)
448    {
449        // Number of contents
450        AmetysObjectIterable<Content> contents = site.getContents();
451        values.put("COUNT", values.get("COUNT") + contents.getSize());
452        
453        int orphans = 0;
454        int external = 0;
455        for (Content content : contents)
456        {
457            if (content instanceof WebContent webContent
458                && webContent.getReferencingPages().size() == 0)
459            {
460                orphans++;
461            }
462            if (content instanceof SharedContent)
463            {
464                external++;
465            }
466        }
467        
468        // Number of orphan contents
469        values.put("ORPHANS", values.get("ORPHANS") + orphans);
470        
471        // Number of shared contents
472        values.put("EXTERNAL", values.get("EXTERNAL") + external);
473    }
474    
475    private Statistics _processResources()
476    {
477        HashMap<String, Long> values = new HashMap<>();
478        values.put("FOLDERCOUNT", 0L);
479        values.put("RESOURCECOUNT", 0L);
480        values.put("TOTALSIZE", 0L);
481        
482        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
483        {
484            for (Site site : sites)
485            {
486                _processResources (site.getRootResources(), values);
487            }
488        }
489        
490        return new StatisticsNode(
491            "explorer",
492            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_LABEL"),
493            "ametysicon-folder249",
494            values.get("TOTALSIZE"),
495            List.of(
496                new StatisticsValue(
497                    "folders",
498                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FOLDER_LABEL"),
499                    "ametysicon-folder250",
500                    values.get("FOLDERCOUNT")
501                ),
502                new StatisticsValue(
503                    "files",
504                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FILES_LABEL"),
505                    "ametysicon-document77",
506                    values.get("RESOURCECOUNT")
507                )
508            ),
509            false
510        );
511    }
512    
513    private void _processResources(TraversableAmetysObject resourceContainer, HashMap<String, Long> values)
514    {
515        AmetysObjectIterable<? extends AmetysObject> objects = resourceContainer.getChildren();
516        
517        for (AmetysObject object : objects)
518        {
519            // Traverse the child nodes if depth < 0 (generate all) or depth > 0 (we're not in the last level).
520            if (object instanceof ResourceCollection)
521            {
522                values.put("FOLDERCOUNT", values.get("FOLDERCOUNT") + 1);
523                
524                _processResources((ResourceCollection) object, values);
525            }
526            else if (object instanceof Resource)
527            {
528                Resource resource = (Resource) object;
529                
530                values.put("RESOURCECOUNT", values.get("RESOURCECOUNT") + 1);
531                values.put("TOTALSIZE", values.get("TOTALSIZE") + resource.getLength());
532            }
533        }
534    }
535
536    private I18nizableText _getGDPRGlobal()
537    {
538        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
539        {
540            return _gDPRComponentEnumerator.getEntry(Config.getInstance().getValue("plugins.web.gdpr.choice"));
541        }
542        catch (Exception e)
543        {
544            throw new IllegalStateException(e);
545        }
546    }
547    
548    private List<Statistics> _getGDPRBySite()
549    {
550        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
551        {
552            List<Statistics> gdpr = new ArrayList<>();
553
554            Map<String, Long> gdprChoiceNumber = sites.stream()
555                    .map(site -> (String) (site.hasValue("gdpr-component-choice") ? site.getValue("gdpr-component-choice") : "_ametys_general_configuration"))
556                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
557
558            gdpr.add(new StatisticsValue("_ametys_general_configuration",
559                    new I18nizableText("plugin.web", "PLUGINS_WEB_SITE_COOKIES_GENERAL_CONFIGURATION"),
560                    "ametysicon-system-monitoring",
561                    gdprChoiceNumber.getOrDefault("_ametys_general_configuration", 0L)));
562            for (Entry<String, I18nizableText> entry : _gDPRComponentEnumerator.getEntries().entrySet())
563            {
564                gdpr.add(new StatisticsValue(entry.getKey(),
565                        entry.getValue(),
566                        "ametysicon-system-monitoring",
567                        gdprChoiceNumber.getOrDefault(entry.getKey(), 0L)));
568            }
569            return gdpr;
570        }
571        catch (Exception e)
572        {
573            throw new IllegalStateException(e);
574        }
575        
576    }
577}