Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.statistics;
017
018import java.util.ArrayList;
019import java.util.Comparator;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Map;
023import java.util.Map.Entry;
024import java.util.function.Function;
025import java.util.stream.Collectors;
026
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang3.StringUtils;
031
032import org.ametys.cms.repository.Content;
033import org.ametys.core.util.I18nUtils;
034import org.ametys.plugins.explorer.resources.Resource;
035import org.ametys.plugins.explorer.resources.ResourceCollection;
036import org.ametys.plugins.repository.AmetysObject;
037import org.ametys.plugins.repository.AmetysObjectIterable;
038import org.ametys.plugins.repository.AmetysObjectResolver;
039import org.ametys.plugins.repository.TraversableAmetysObject;
040import org.ametys.runtime.config.Config;
041import org.ametys.runtime.i18n.I18nizableText;
042import org.ametys.runtime.plugin.component.PluginAware;
043import org.ametys.runtime.plugins.admin.statistics.Statistics;
044import org.ametys.runtime.plugins.admin.statistics.StatisticsNode;
045import org.ametys.runtime.plugins.admin.statistics.StatisticsProvider;
046import org.ametys.runtime.plugins.admin.statistics.StatisticsValue;
047import org.ametys.web.cache.PageHelper;
048import org.ametys.web.gdpr.GDPRComponentEnumerator;
049import org.ametys.web.repository.content.SharedContent;
050import org.ametys.web.repository.content.WebContent;
051import org.ametys.web.repository.page.Page;
052import org.ametys.web.repository.page.SitemapElement;
053import org.ametys.web.repository.page.Zone;
054import org.ametys.web.repository.page.ZoneItem;
055import org.ametys.web.repository.site.Site;
056import org.ametys.web.repository.site.SiteManager;
057import org.ametys.web.repository.sitemap.Sitemap;
058import org.ametys.web.service.Service;
059import org.ametys.web.service.ServiceExtensionPoint;
060import org.ametys.web.skin.SkinsManager;
061
062/**
063 * Web statistics
064 */
065public class WebStatisticsProvider implements StatisticsProvider, Serviceable, PluginAware
066{
067    private String _id;
068    private SiteManager _siteManager;
069    private PageHelper _pageHelper;
070    private ServiceExtensionPoint _serviceEP;
071    private I18nUtils _i18nUtils;
072    private AmetysObjectResolver _ametysResolver;
073    private SkinsManager _skinsManager;
074    private GDPRComponentEnumerator _gDPRComponentEnumerator;
075
076    public void service(ServiceManager manager) throws ServiceException
077    {
078        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
079        _pageHelper = (PageHelper) manager.lookup(PageHelper.ROLE);
080        _serviceEP = (ServiceExtensionPoint) manager.lookup(ServiceExtensionPoint.ROLE);
081        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
082        _ametysResolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
083        _skinsManager = (SkinsManager) manager.lookup(SkinsManager.ROLE);
084        _gDPRComponentEnumerator = (GDPRComponentEnumerator) manager.lookup(GDPRComponentEnumerator.ROLE);
085    }
086    
087    public void setPluginInfo(String pluginName, String featureName, String id)
088    {
089        _id = id;
090    }
091    
092    public Statistics getStatistics()
093    {
094        List<List<Map<String, Long>>> countPages = _countPages();
095        int siteCount = countPages.size();
096        int sitemapCount = countPages.stream().map(List<Map<String, Long>>::size).reduce(0, Integer::sum);
097        List<Long> nbPagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBPAGES")).sorted().toList();
098        long pagesCount = nbPagesPerSitemap.stream().reduce(0L, Long::sum);
099        List<Long> nbContentsRefPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("CONTENTS")).sorted().toList();
100        long contentsRefsCount = nbContentsRefPerSitemap.stream().reduce(0L, Long::sum);
101        List<Long> nbServicesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("SERVICES")).sorted().toList();
102        long servicesCount = nbServicesPerSitemap.stream().reduce(0L, Long::sum);
103        List<Long> nbCacheablePagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBCACHEABLE")).sorted().toList();
104        long cacheablePagesCount = nbCacheablePagesPerSitemap.stream().reduce(0L, Long::sum);
105        
106        Map<String, Long> contentsForSites = _processContentsForSites();
107        
108        return new StatisticsNode(
109            _id,
110            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_LABEL"),
111            "ametysicon-code-html-link",
112            null,
113            List.of(
114                new StatisticsNode(
115                    "sites",
116                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_LABEL"),
117                    "ametysicon-world-earth-black",
118                    siteCount,
119                    List.of(
120                        new StatisticsNode(
121                            "sitemaps",
122                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_LABEL"),
123                            "ametysicon-world-flag",
124                            sitemapCount,
125                            List.of(
126                                new StatisticsNode(
127                                    "pages",
128                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_LABEL"),
129                                    "ametysicon-website38",
130                                    pagesCount,
131                                    List.of(
132                                        new StatisticsValue(
133                                            "max",
134                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MAX_LABEL"),
135                                            "ametysicon-sort51",
136                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() - 1) : 0
137                                        ),
138                                        new StatisticsValue(
139                                            "median",
140                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MEDIAN_LABEL"),
141                                            "ametysicon-maths-window-symbol-x",
142                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() / 2) : 0
143                                        ),
144                                        new StatisticsValue(
145                                            "cacheable",
146                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_CACHEABLE_LABEL"),
147                                            "ametysicon-desktop-archive",
148                                            cacheablePagesCount
149                                        )
150                                    ),
151                                    false
152                                ),
153                                new StatisticsNode(
154                                    "content-references",
155                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_LABEL"),
156                                    "ametysicon-text70",
157                                    contentsRefsCount,
158                                    List.of(
159                                        new StatisticsValue(
160                                            "max",
161                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MAX_LABEL"),
162                                            "ametysicon-sort51",
163                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() - 1) : 0
164                                        ),
165                                        new StatisticsValue(
166                                            "median",
167                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MEDIAN_LABEL"),
168                                            "ametysicon-maths-window-symbol-x",
169                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() / 2) : 0
170                                        )
171                                    ),
172                                    false
173                                ),
174                                new StatisticsNode(
175                                    "content-references",
176                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_LABEL"),
177                                    "ametysicon-text70",
178                                    servicesCount,
179                                    List.of(
180                                        new StatisticsValue(
181                                            "max",
182                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MAX_LABEL"),
183                                            "ametysicon-sort51",
184                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() - 1) : 0
185                                        ),
186                                        new StatisticsValue(
187                                            "median",
188                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MEDIAN_LABEL"),
189                                            "ametysicon-maths-window-symbol-x",
190                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() / 2) : 0
191                                        )
192                                    ),
193                                    false
194                                )
195                            ),
196                            false
197                        ),
198                        new StatisticsNode(
199                            "contents",
200                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_LABEL"),
201                            "ametysicon-text70",
202                            contentsForSites.get("COUNT"),
203                            List.of(
204                                new StatisticsValue(
205                                    "orphans",
206                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_ORPHANS_LABEL"),
207                                    "ametysicon-sort51",
208                                    contentsForSites.get("ORPHANS")
209                                ),
210                                new StatisticsValue(
211                                    "shared",
212                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_EXTERNAL_LABEL"),
213                                    "ametysicon-maths-window-symbol-x",
214                                    contentsForSites.get("EXTERNAL")
215                                )
216                            ),
217                            false
218                        ),
219                        new StatisticsNode(
220                            "services",
221                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_LABEL"),
222                            "ametysicon-system-monitoring",
223                            null,
224                            _getServices(),
225                            false
226                        ),
227                        _processResources(),
228                        new StatisticsValue(
229                            "skins",
230                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SKINS_LABEL"),
231                            "ametysicon-puzzle-piece1",
232                            _skinsManager.getResourceSkins()
233                        ),
234                        new StatisticsNode(
235                                "gdpr",
236                                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_GDPR_LABEL"),
237                                "ametysicon-desktop-clipboard-list",
238                                _getGDPRGlobal(),
239                                _getGDPRBySite(),
240                                false
241                        )
242                    ),
243                    true
244                )
245            ),
246            true
247        );
248    }
249
250    private List<List<Map<String, Long>>> _countPages()
251    {
252        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
253        {
254            List<List<Map<String, Long>>> sitesValues = new ArrayList<>();
255            
256            for (Site site : sites)
257            {
258                sitesValues.add(_countPages(site));
259            }
260            
261            return sitesValues;
262        }
263    }
264    
265    private List<Map<String, Long>> _countPages(Site site)
266    {
267        List<Map<String, Long>> siteValues = new ArrayList<>();
268
269        for (Sitemap sitemap : site.getSitemaps())
270        {
271            siteValues.add(_countPages(sitemap));
272        }
273        
274        return siteValues;
275    }
276    
277    private Map<String, Long> _countPages(SitemapElement pages)
278    {
279        Map<String, Long> sitemapElementValues = _newCountMap();
280        
281        AmetysObjectIterable<? extends Page> childPages = pages.getChildrenPages();
282        
283        sitemapElementValues.put("NBPAGES", sitemapElementValues.get("NBPAGES") + childPages.getSize());
284        
285        int cacheableCount = 0;
286        
287        for (Page page : childPages)
288        {
289            boolean isCacheable = _pageHelper.isCacheable(page);
290            for (Zone zone : page.getZones())
291            {
292                AmetysObjectIterable<? extends ZoneItem> zoneItems = zone.getZoneItems();
293                for (ZoneItem item : zoneItems)
294                {
295                    switch (item.getType())
296                    {
297                        case CONTENT:
298                            sitemapElementValues.put("CONTENTS", sitemapElementValues.get("CONTENTS") + 1);
299                            break;
300                        case SERVICE:
301                            sitemapElementValues.put("SERVICES", sitemapElementValues.get("SERVICES") + 1);
302                            break;
303                        default:
304                            break;
305                    }
306                }
307            }
308            
309            if (isCacheable)
310            {
311                cacheableCount++;
312            }
313            
314            _sumMaps(sitemapElementValues, _countPages(page));
315        }
316        
317        sitemapElementValues.put("NBCACHEABLE", sitemapElementValues.get("NBCACHEABLE") + cacheableCount);
318        
319        return sitemapElementValues;
320    }
321    
322    
323    private Map<String, Long> _newCountMap()
324    {
325        Map<String, Long> countMap = new HashMap<>();
326        countMap.put("NBPAGES", 0L); // Number of pages
327        countMap.put("NBCACHEABLE", 0L); // Number of cacheable pages
328        countMap.put("SERVICES", 0L); // Number of services
329        countMap.put("CONTENTS", 0L); // Number of contents
330        return countMap;
331    }
332    
333    private void _sumMaps(Map<String, Long> mainMap, Map<String, Long> mapToAdd)
334    {
335        for (String k : mainMap.keySet())
336        {
337            mainMap.put(k, mainMap.get(k) + mapToAdd.get(k));
338        }
339    }
340    
341    private List<Statistics> _getServices()
342    {
343        List<Statistics> stats = new ArrayList<>();
344        
345        for (String serviceId : _serviceEP.getExtensionsIds())
346        {
347            Service service = _serviceEP.getExtension(serviceId);
348            
349            stats.add(new StatisticsValue(
350                serviceId,
351                new I18nizableText(_i18nUtils.translate(service.getLabel())), // Translate now for sorting
352                StringUtils.defaultIfBlank(service.getIconGlyph(), "ametysicon-system-monitoring"),
353                _countAllServices(serviceId)
354            ));
355        }
356        
357        stats.sort(new Comparator<>() {
358            public int compare(Statistics o1, Statistics o2)
359            {
360                return o1.getLabel().getLabel().compareTo(o2.getLabel().getLabel());
361            }
362        });
363        
364        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
365                + _serviceEP.getExtensionsIds().stream().map(id -> " and not(@ametys-internal:service='" + id + "')").collect(Collectors.joining())
366                + "]";
367        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
368        {
369            stats.add(0, new StatisticsValue(
370                "unknown",
371                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_UNKNOWN_LABEL"),
372                "ametysicon-file-extension-generic-unknown",
373                serviceZoneItems.getSize()
374            ));
375        }
376        
377        return stats;
378    }
379
380    private long _countAllServices(String serviceId)
381    {
382        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
383                + (serviceId != null ? " and @ametys-internal:service='" + serviceId + "'" : "")
384                + "]";
385        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
386        {
387            return serviceZoneItems.getSize();
388        }
389    }
390    
391    private Map<String, Long> _processContentsForSites()
392    {
393        Map<String, Long> values = new HashMap<>();
394        values.put("COUNT", 0L);
395        values.put("ORPHANS", 0L);
396        values.put("EXTERNAL", 0L);
397        
398        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
399        {
400            for (Site site : sites)
401            {
402                _processContents (site, values);
403            }
404        }
405        catch (Exception e)
406        {
407            throw new IllegalStateException(e);
408        }
409        
410        return values;
411    }
412    
413    private void _processContents (Site site, Map<String, Long> values)
414    {
415        // Number of contents
416        AmetysObjectIterable<Content> contents = site.getContents();
417        values.put("COUNT", values.get("COUNT") + contents.getSize());
418        
419        int orphans = 0;
420        int external = 0;
421        for (Content content : contents)
422        {
423            if (content instanceof WebContent webContent
424                && webContent.getReferencingPages().size() == 0)
425            {
426                orphans++;
427            }
428            if (content instanceof SharedContent)
429            {
430                external++;
431            }
432        }
433        
434        // Number of orphan contents
435        values.put("ORPHANS", values.get("ORPHANS") + orphans);
436        
437        // Number of shared contents
438        values.put("EXTERNAL", values.get("EXTERNAL") + external);
439    }
440    
441    private Statistics _processResources()
442    {
443        HashMap<String, Long> values = new HashMap<>();
444        values.put("FOLDERCOUNT", 0L);
445        values.put("RESOURCECOUNT", 0L);
446        values.put("TOTALSIZE", 0L);
447        
448        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
449        {
450            for (Site site : sites)
451            {
452                _processResources (site.getRootResources(), values);
453            }
454        }
455        
456        return new StatisticsNode(
457            "explorer",
458            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_LABEL"),
459            "ametysicon-folder249",
460            values.get("TOTALSIZE"),
461            List.of(
462                new StatisticsValue(
463                    "folders",
464                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FOLDER_LABEL"),
465                    "ametysicon-folder250",
466                    values.get("FOLDERCOUNT")
467                ),
468                new StatisticsValue(
469                    "files",
470                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FILES_LABEL"),
471                    "ametysicon-document77",
472                    values.get("RESOURCECOUNT")
473                )
474            ),
475            false
476        );
477    }
478    
479    private void _processResources(TraversableAmetysObject resourceContainer, HashMap<String, Long> values)
480    {
481        AmetysObjectIterable<? extends AmetysObject> objects = resourceContainer.getChildren();
482        
483        for (AmetysObject object : objects)
484        {
485            // Traverse the child nodes if depth < 0 (generate all) or depth > 0 (we're not in the last level).
486            if (object instanceof ResourceCollection)
487            {
488                values.put("FOLDERCOUNT", values.get("FOLDERCOUNT") + 1);
489                
490                _processResources((ResourceCollection) object, values);
491            }
492            else if (object instanceof Resource)
493            {
494                Resource resource = (Resource) object;
495                
496                values.put("RESOURCECOUNT", values.get("RESOURCECOUNT") + 1);
497                values.put("TOTALSIZE", values.get("TOTALSIZE") + resource.getLength());
498            }
499        }
500    }
501
502    private I18nizableText _getGDPRGlobal()
503    {
504        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
505        {
506            return _gDPRComponentEnumerator.getEntry(Config.getInstance().getValue("plugins.web.gdpr.choice"));
507        }
508        catch (Exception e)
509        {
510            throw new IllegalStateException(e);
511        }
512    }
513    
514    private List<Statistics> _getGDPRBySite()
515    {
516        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
517        {
518            List<Statistics> gdpr = new ArrayList<>();
519
520            Map<String, Long> gdprChoiceNumber = sites.stream()
521                    .map(site -> (String) (site.hasValue("gdpr-component-choice") ? site.getValue("gdpr-component-choice") : "_ametys_general_configuration"))
522                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
523
524            gdpr.add(new StatisticsValue("_ametys_general_configuration",
525                    new I18nizableText("plugin.web", "PLUGINS_WEB_SITE_COOKIES_GENERAL_CONFIGURATION"),
526                    "ametysicon-system-monitoring",
527                    gdprChoiceNumber.getOrDefault("_ametys_general_configuration", 0L)));
528            for (Entry<String, I18nizableText> entry : _gDPRComponentEnumerator.getTypedEntries().entrySet())
529            {
530                gdpr.add(new StatisticsValue(entry.getKey(),
531                        entry.getValue(),
532                        "ametysicon-system-monitoring",
533                        gdprChoiceNumber.getOrDefault(entry.getKey(), 0L)));
534            }
535            return gdpr;
536        }
537        catch (Exception e)
538        {
539            throw new IllegalStateException(e);
540        }
541        
542    }
543}