Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.statistics;
017
018import java.util.ArrayList;
019import java.util.Comparator;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Map;
023import java.util.Map.Entry;
024import java.util.function.Function;
025import java.util.stream.Collectors;
026
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang3.StringUtils;
031
032import org.ametys.cms.repository.Content;
033import org.ametys.core.util.I18nUtils;
034import org.ametys.plugins.explorer.resources.Resource;
035import org.ametys.plugins.explorer.resources.ResourceCollection;
036import org.ametys.plugins.repository.AmetysObject;
037import org.ametys.plugins.repository.AmetysObjectIterable;
038import org.ametys.plugins.repository.AmetysObjectResolver;
039import org.ametys.plugins.repository.TraversableAmetysObject;
040import org.ametys.runtime.config.Config;
041import org.ametys.runtime.i18n.I18nizableText;
042import org.ametys.runtime.plugin.component.PluginAware;
043import org.ametys.runtime.plugins.admin.statistics.Statistics;
044import org.ametys.runtime.plugins.admin.statistics.StatisticsNode;
045import org.ametys.runtime.plugins.admin.statistics.StatisticsProvider;
046import org.ametys.runtime.plugins.admin.statistics.StatisticsValue;
047import org.ametys.web.cache.PageHelper;
048import org.ametys.web.gdpr.GDPRComponentEnumerator;
049import org.ametys.web.repository.content.SharedContent;
050import org.ametys.web.repository.content.WebContent;
051import org.ametys.web.repository.page.Page;
052import org.ametys.web.repository.page.SitemapElement;
053import org.ametys.web.repository.page.Zone;
054import org.ametys.web.repository.page.ZoneItem;
055import org.ametys.web.repository.site.Site;
056import org.ametys.web.repository.site.SiteManager;
057import org.ametys.web.repository.sitemap.Sitemap;
058import org.ametys.web.service.Service;
059import org.ametys.web.service.ServiceExtensionPoint;
060import org.ametys.web.skin.SkinsManager;
061
062/**
063 * Web statistics
064 */
065public class WebStatisticsProvider implements StatisticsProvider, Serviceable, PluginAware
066{
067    private String _id;
068    private SiteManager _siteManager;
069    private PageHelper _pageHelper;
070    private ServiceExtensionPoint _serviceEP;
071    private I18nUtils _i18nUtils;
072    private AmetysObjectResolver _ametysResolver;
073    private SkinsManager _skinsManager;
074    private GDPRComponentEnumerator _gDPRComponentEnumerator;
075
076    public void service(ServiceManager manager) throws ServiceException
077    {
078        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
079        _pageHelper = (PageHelper) manager.lookup(PageHelper.ROLE);
080        _serviceEP = (ServiceExtensionPoint) manager.lookup(ServiceExtensionPoint.ROLE);
081        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
082        _ametysResolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
083        _skinsManager = (SkinsManager) manager.lookup(SkinsManager.ROLE);
084        _gDPRComponentEnumerator = (GDPRComponentEnumerator) manager.lookup(GDPRComponentEnumerator.ROLE);
085    }
086    
087    public void setPluginInfo(String pluginName, String featureName, String id)
088    {
089        _id = id;
090    }
091    
092    public Statistics getStatistics()
093    {
094        List<List<Map<String, Long>>> countPages = _countPages();
095        int siteCount = countPages.size();
096        int sitemapCount = countPages.stream().map(List<Map<String, Long>>::size).reduce(0, Integer::sum);
097        List<Long> nbPagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBPAGES")).sorted().toList();
098        long pagesCount = nbPagesPerSitemap.stream().reduce(0L, Long::sum);
099        List<Long> nbContentsRefPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("CONTENTS")).sorted().toList();
100        long contentsRefsCount = nbContentsRefPerSitemap.stream().reduce(0L, Long::sum);
101        List<Long> nbServicesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("SERVICES")).sorted().toList();
102        long servicesCount = nbServicesPerSitemap.stream().reduce(0L, Long::sum);
103        List<Long> nbCacheablePagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBCACHEABLE")).sorted().toList();
104        long cacheablePagesCount = nbCacheablePagesPerSitemap.stream().reduce(0L, Long::sum);
105        
106        Map<String, Long> contentsForSites = _processContentsForSites();
107        
108        List<Statistics> skins = _skinsManager.getResourceSkins()
109                                              .stream()
110                                              .sorted(Comparator.naturalOrder())
111                                              .map(skinName -> new StatisticsValue(skinName,
112                                                                                   new I18nizableText(skinName),
113                                                                                   "ametysicon-puzzle-piece1", 
114                                                                                   _skinsManager.getVersion(skinName)))
115                                              .map(Statistics.class::cast)
116                                              .toList();
117
118        
119        return new StatisticsNode(
120            _id,
121            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_LABEL"),
122            "ametysicon-code-html-link",
123            null,
124            List.of(
125                new StatisticsNode(
126                    "sites",
127                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_LABEL"),
128                    "ametysicon-world-earth-black",
129                    siteCount,
130                    List.of(
131                        new StatisticsNode(
132                            "sitemaps",
133                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_LABEL"),
134                            "ametysicon-world-flag",
135                            sitemapCount,
136                            List.of(
137                                new StatisticsNode(
138                                    "pages",
139                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_LABEL"),
140                                    "ametysicon-website38",
141                                    pagesCount,
142                                    List.of(
143                                        new StatisticsValue(
144                                            "max",
145                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MAX_LABEL"),
146                                            "ametysicon-sort51",
147                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() - 1) : 0
148                                        ),
149                                        new StatisticsValue(
150                                            "median",
151                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MEDIAN_LABEL"),
152                                            "ametysicon-maths-window-symbol-x",
153                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() / 2) : 0
154                                        ),
155                                        new StatisticsValue(
156                                            "cacheable",
157                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_CACHEABLE_LABEL"),
158                                            "ametysicon-desktop-archive",
159                                            cacheablePagesCount
160                                        )
161                                    ),
162                                    false
163                                ),
164                                new StatisticsNode(
165                                    "content-references",
166                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_LABEL"),
167                                    "ametysicon-text70",
168                                    contentsRefsCount,
169                                    List.of(
170                                        new StatisticsValue(
171                                            "max",
172                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MAX_LABEL"),
173                                            "ametysicon-sort51",
174                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() - 1) : 0
175                                        ),
176                                        new StatisticsValue(
177                                            "median",
178                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MEDIAN_LABEL"),
179                                            "ametysicon-maths-window-symbol-x",
180                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() / 2) : 0
181                                        )
182                                    ),
183                                    false
184                                ),
185                                new StatisticsNode(
186                                    "content-references",
187                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_LABEL"),
188                                    "ametysicon-text70",
189                                    servicesCount,
190                                    List.of(
191                                        new StatisticsValue(
192                                            "max",
193                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MAX_LABEL"),
194                                            "ametysicon-sort51",
195                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() - 1) : 0
196                                        ),
197                                        new StatisticsValue(
198                                            "median",
199                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MEDIAN_LABEL"),
200                                            "ametysicon-maths-window-symbol-x",
201                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() / 2) : 0
202                                        )
203                                    ),
204                                    false
205                                )
206                            ),
207                            false
208                        ),
209                        new StatisticsNode(
210                            "contents",
211                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_LABEL"),
212                            "ametysicon-text70",
213                            contentsForSites.get("COUNT"),
214                            List.of(
215                                new StatisticsValue(
216                                    "orphans",
217                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_ORPHANS_LABEL"),
218                                    "ametysicon-sort51",
219                                    contentsForSites.get("ORPHANS")
220                                ),
221                                new StatisticsValue(
222                                    "shared",
223                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_EXTERNAL_LABEL"),
224                                    "ametysicon-maths-window-symbol-x",
225                                    contentsForSites.get("EXTERNAL")
226                                )
227                            ),
228                            false
229                        ),
230                        new StatisticsNode(
231                            "services",
232                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_LABEL"),
233                            "ametysicon-system-monitoring",
234                            null,
235                            _getServices(),
236                            false
237                        ),
238                        _processResources(),
239                        new StatisticsNode(
240                            "skins",
241                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SKINS_LABEL"),
242                            "ametysicon-puzzle-piece1",
243                            null,
244                            skins,
245                            false
246                        ),
247                        new StatisticsNode(
248                                "gdpr",
249                                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_GDPR_LABEL"),
250                                "ametysicon-desktop-clipboard-list",
251                                _getGDPRGlobal(),
252                                _getGDPRBySite(),
253                                false
254                        )
255                    ),
256                    true
257                )
258            ),
259            true
260        );
261    }
262
263    private List<List<Map<String, Long>>> _countPages()
264    {
265        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
266        {
267            List<List<Map<String, Long>>> sitesValues = new ArrayList<>();
268            
269            for (Site site : sites)
270            {
271                sitesValues.add(_countPages(site));
272            }
273            
274            return sitesValues;
275        }
276    }
277    
278    private List<Map<String, Long>> _countPages(Site site)
279    {
280        List<Map<String, Long>> siteValues = new ArrayList<>();
281
282        for (Sitemap sitemap : site.getSitemaps())
283        {
284            siteValues.add(_countPages(sitemap));
285        }
286        
287        return siteValues;
288    }
289    
290    private Map<String, Long> _countPages(SitemapElement pages)
291    {
292        Map<String, Long> sitemapElementValues = _newCountMap();
293        
294        AmetysObjectIterable<? extends Page> childPages = pages.getChildrenPages();
295        
296        sitemapElementValues.put("NBPAGES", sitemapElementValues.get("NBPAGES") + childPages.getSize());
297        
298        int cacheableCount = 0;
299        
300        for (Page page : childPages)
301        {
302            boolean isCacheable = _pageHelper.isCacheable(page);
303            for (Zone zone : page.getZones())
304            {
305                AmetysObjectIterable<? extends ZoneItem> zoneItems = zone.getZoneItems();
306                for (ZoneItem item : zoneItems)
307                {
308                    switch (item.getType())
309                    {
310                        case CONTENT:
311                            sitemapElementValues.put("CONTENTS", sitemapElementValues.get("CONTENTS") + 1);
312                            break;
313                        case SERVICE:
314                            sitemapElementValues.put("SERVICES", sitemapElementValues.get("SERVICES") + 1);
315                            break;
316                        default:
317                            break;
318                    }
319                }
320            }
321            
322            if (isCacheable)
323            {
324                cacheableCount++;
325            }
326            
327            _sumMaps(sitemapElementValues, _countPages(page));
328        }
329        
330        sitemapElementValues.put("NBCACHEABLE", sitemapElementValues.get("NBCACHEABLE") + cacheableCount);
331        
332        return sitemapElementValues;
333    }
334    
335    
336    private Map<String, Long> _newCountMap()
337    {
338        Map<String, Long> countMap = new HashMap<>();
339        countMap.put("NBPAGES", 0L); // Number of pages
340        countMap.put("NBCACHEABLE", 0L); // Number of cacheable pages
341        countMap.put("SERVICES", 0L); // Number of services
342        countMap.put("CONTENTS", 0L); // Number of contents
343        return countMap;
344    }
345    
346    private void _sumMaps(Map<String, Long> mainMap, Map<String, Long> mapToAdd)
347    {
348        for (String k : mainMap.keySet())
349        {
350            mainMap.put(k, mainMap.get(k) + mapToAdd.get(k));
351        }
352    }
353    
354    private List<Statistics> _getServices()
355    {
356        List<Statistics> stats = new ArrayList<>();
357        
358        for (String serviceId : _serviceEP.getExtensionsIds())
359        {
360            Service service = _serviceEP.getExtension(serviceId);
361            
362            stats.add(new StatisticsValue(
363                serviceId,
364                new I18nizableText(_i18nUtils.translate(service.getLabel())), // Translate now for sorting
365                StringUtils.defaultIfBlank(service.getIconGlyph(), "ametysicon-system-monitoring"),
366                _countAllServices(serviceId)
367            ));
368        }
369        
370        stats.sort(new Comparator<>() {
371            public int compare(Statistics o1, Statistics o2)
372            {
373                return o1.getLabel().getLabel().compareTo(o2.getLabel().getLabel());
374            }
375        });
376        
377        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
378                + _serviceEP.getExtensionsIds().stream().map(id -> " and not(@ametys-internal:service='" + id + "')").collect(Collectors.joining())
379                + "]";
380        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
381        {
382            stats.add(0, new StatisticsValue(
383                "unknown",
384                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_UNKNOWN_LABEL"),
385                "ametysicon-file-extension-generic-unknown",
386                serviceZoneItems.getSize()
387            ));
388        }
389        
390        return stats;
391    }
392
393    private long _countAllServices(String serviceId)
394    {
395        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
396                + (serviceId != null ? " and @ametys-internal:service='" + serviceId + "'" : "")
397                + "]";
398        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
399        {
400            return serviceZoneItems.getSize();
401        }
402    }
403    
404    private Map<String, Long> _processContentsForSites()
405    {
406        Map<String, Long> values = new HashMap<>();
407        values.put("COUNT", 0L);
408        values.put("ORPHANS", 0L);
409        values.put("EXTERNAL", 0L);
410        
411        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
412        {
413            for (Site site : sites)
414            {
415                _processContents (site, values);
416            }
417        }
418        catch (Exception e)
419        {
420            throw new IllegalStateException(e);
421        }
422        
423        return values;
424    }
425    
426    private void _processContents (Site site, Map<String, Long> values)
427    {
428        // Number of contents
429        AmetysObjectIterable<Content> contents = site.getContents();
430        values.put("COUNT", values.get("COUNT") + contents.getSize());
431        
432        int orphans = 0;
433        int external = 0;
434        for (Content content : contents)
435        {
436            if (content instanceof WebContent webContent
437                && webContent.getReferencingPages().size() == 0)
438            {
439                orphans++;
440            }
441            if (content instanceof SharedContent)
442            {
443                external++;
444            }
445        }
446        
447        // Number of orphan contents
448        values.put("ORPHANS", values.get("ORPHANS") + orphans);
449        
450        // Number of shared contents
451        values.put("EXTERNAL", values.get("EXTERNAL") + external);
452    }
453    
454    private Statistics _processResources()
455    {
456        HashMap<String, Long> values = new HashMap<>();
457        values.put("FOLDERCOUNT", 0L);
458        values.put("RESOURCECOUNT", 0L);
459        values.put("TOTALSIZE", 0L);
460        
461        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
462        {
463            for (Site site : sites)
464            {
465                _processResources (site.getRootResources(), values);
466            }
467        }
468        
469        return new StatisticsNode(
470            "explorer",
471            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_LABEL"),
472            "ametysicon-folder249",
473            values.get("TOTALSIZE"),
474            List.of(
475                new StatisticsValue(
476                    "folders",
477                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FOLDER_LABEL"),
478                    "ametysicon-folder250",
479                    values.get("FOLDERCOUNT")
480                ),
481                new StatisticsValue(
482                    "files",
483                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FILES_LABEL"),
484                    "ametysicon-document77",
485                    values.get("RESOURCECOUNT")
486                )
487            ),
488            false
489        );
490    }
491    
492    private void _processResources(TraversableAmetysObject resourceContainer, HashMap<String, Long> values)
493    {
494        AmetysObjectIterable<? extends AmetysObject> objects = resourceContainer.getChildren();
495        
496        for (AmetysObject object : objects)
497        {
498            // Traverse the child nodes if depth < 0 (generate all) or depth > 0 (we're not in the last level).
499            if (object instanceof ResourceCollection)
500            {
501                values.put("FOLDERCOUNT", values.get("FOLDERCOUNT") + 1);
502                
503                _processResources((ResourceCollection) object, values);
504            }
505            else if (object instanceof Resource)
506            {
507                Resource resource = (Resource) object;
508                
509                values.put("RESOURCECOUNT", values.get("RESOURCECOUNT") + 1);
510                values.put("TOTALSIZE", values.get("TOTALSIZE") + resource.getLength());
511            }
512        }
513    }
514
515    private I18nizableText _getGDPRGlobal()
516    {
517        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
518        {
519            return _gDPRComponentEnumerator.getEntry(Config.getInstance().getValue("plugins.web.gdpr.choice"));
520        }
521        catch (Exception e)
522        {
523            throw new IllegalStateException(e);
524        }
525    }
526    
527    private List<Statistics> _getGDPRBySite()
528    {
529        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
530        {
531            List<Statistics> gdpr = new ArrayList<>();
532
533            Map<String, Long> gdprChoiceNumber = sites.stream()
534                    .map(site -> (String) (site.hasValue("gdpr-component-choice") ? site.getValue("gdpr-component-choice") : "_ametys_general_configuration"))
535                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
536
537            gdpr.add(new StatisticsValue("_ametys_general_configuration",
538                    new I18nizableText("plugin.web", "PLUGINS_WEB_SITE_COOKIES_GENERAL_CONFIGURATION"),
539                    "ametysicon-system-monitoring",
540                    gdprChoiceNumber.getOrDefault("_ametys_general_configuration", 0L)));
541            for (Entry<String, I18nizableText> entry : _gDPRComponentEnumerator.getEntries().entrySet())
542            {
543                gdpr.add(new StatisticsValue(entry.getKey(),
544                        entry.getValue(),
545                        "ametysicon-system-monitoring",
546                        gdprChoiceNumber.getOrDefault(entry.getKey(), 0L)));
547            }
548            return gdpr;
549        }
550        catch (Exception e)
551        {
552            throw new IllegalStateException(e);
553        }
554        
555    }
556}