001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.statistics;
017
018import java.util.ArrayList;
019import java.util.Comparator;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Map;
023import java.util.stream.Collectors;
024
025import org.apache.avalon.framework.service.ServiceException;
026import org.apache.avalon.framework.service.ServiceManager;
027import org.apache.avalon.framework.service.Serviceable;
028import org.apache.commons.lang3.StringUtils;
029
030import org.ametys.cms.repository.Content;
031import org.ametys.core.util.I18nUtils;
032import org.ametys.plugins.explorer.resources.Resource;
033import org.ametys.plugins.explorer.resources.ResourceCollection;
034import org.ametys.plugins.repository.AmetysObject;
035import org.ametys.plugins.repository.AmetysObjectIterable;
036import org.ametys.plugins.repository.AmetysObjectResolver;
037import org.ametys.plugins.repository.TraversableAmetysObject;
038import org.ametys.runtime.i18n.I18nizableText;
039import org.ametys.runtime.plugin.component.PluginAware;
040import org.ametys.runtime.plugins.admin.statistics.Statistics;
041import org.ametys.runtime.plugins.admin.statistics.StatisticsNode;
042import org.ametys.runtime.plugins.admin.statistics.StatisticsProvider;
043import org.ametys.runtime.plugins.admin.statistics.StatisticsValue;
044import org.ametys.web.cache.PageHelper;
045import org.ametys.web.repository.content.SharedContent;
046import org.ametys.web.repository.content.WebContent;
047import org.ametys.web.repository.page.Page;
048import org.ametys.web.repository.page.SitemapElement;
049import org.ametys.web.repository.page.Zone;
050import org.ametys.web.repository.page.ZoneItem;
051import org.ametys.web.repository.site.Site;
052import org.ametys.web.repository.site.SiteManager;
053import org.ametys.web.repository.sitemap.Sitemap;
054import org.ametys.web.service.Service;
055import org.ametys.web.service.ServiceExtensionPoint;
056
057/**
058 * Web statistics
059 */
060public class WebStatisticsProvider implements StatisticsProvider, Serviceable, PluginAware
061{
062    private String _id;
063    private SiteManager _siteManager;
064    private PageHelper _pageHelper;
065    private ServiceExtensionPoint _serviceEP;
066    private I18nUtils _i18nUtils;
067    private AmetysObjectResolver _ametysResolver;
068
069    public void service(ServiceManager manager) throws ServiceException
070    {
071        _siteManager = (SiteManager) manager.lookup(SiteManager.ROLE);
072        _pageHelper = (PageHelper) manager.lookup(PageHelper.ROLE);
073        _serviceEP = (ServiceExtensionPoint) manager.lookup(ServiceExtensionPoint.ROLE);
074        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
075        _ametysResolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
076    }
077    
078    public void setPluginInfo(String pluginName, String featureName, String id)
079    {
080        _id = id;
081    }
082    
083    public Statistics getStatistics()
084    {
085        List<List<Map<String, Long>>> countPages = _countPages();
086        int siteCount = countPages.size();
087        int sitemapCount = countPages.stream().map(List<Map<String, Long>>::size).reduce(0, Integer::sum);
088        List<Long> nbPagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBPAGES")).sorted().toList();
089        long pagesCount = nbPagesPerSitemap.stream().reduce(0L, Long::sum);
090        List<Long> nbContentsRefPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("CONTENTS")).sorted().toList();
091        long contentsRefsCount = nbContentsRefPerSitemap.stream().reduce(0L, Long::sum);
092        List<Long> nbServicesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("SERVICES")).sorted().toList();
093        long servicesCount = nbServicesPerSitemap.stream().reduce(0L, Long::sum);
094        List<Long> nbCacheablePagesPerSitemap = countPages.stream().flatMap(List<Map<String, Long>>::stream).map(m -> m.get("NBCACHEABLE")).sorted().toList();
095        long cacheablePagesCount = nbCacheablePagesPerSitemap.stream().reduce(0L, Long::sum);
096        
097        Map<String, Long> contentsForSites = _processContentsForSites();
098        
099        return new StatisticsNode(
100            _id,
101            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_LABEL"),
102            "ametysicon-code-html-link",
103            null,
104            List.of(
105                new StatisticsNode(
106                    "sites",
107                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_LABEL"),
108                    "ametysicon-world-earth-black",
109                    siteCount,
110                    List.of(
111                        new StatisticsNode(
112                            "sitemaps",
113                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_LABEL"),
114                            "ametysicon-world-flag",
115                            sitemapCount,
116                            List.of(
117                                new StatisticsNode(
118                                    "pages",
119                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_LABEL"),
120                                    "ametysicon-website38",
121                                    pagesCount,
122                                    List.of(
123                                        new StatisticsValue(
124                                            "max",
125                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MAX_LABEL"),
126                                            "ametysicon-sort51",
127                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() - 1) : 0
128                                        ),
129                                        new StatisticsValue(
130                                            "median",
131                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_MEDIAN_LABEL"),
132                                            "ametysicon-maths-window-symbol-x",
133                                            nbPagesPerSitemap.size() > 0 ? nbPagesPerSitemap.get(nbPagesPerSitemap.size() / 2) : 0
134                                        ),
135                                        new StatisticsValue(
136                                            "cacheable",
137                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_PAGES_CACHEABLE_LABEL"),
138                                            "ametysicon-desktop-archive",
139                                            cacheablePagesCount 
140                                        )                                             
141                                    ),
142                                    false
143                                ),
144                                new StatisticsNode(
145                                    "content-references",
146                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_LABEL"),
147                                    "ametysicon-text70",
148                                    contentsRefsCount,
149                                    List.of(
150                                        new StatisticsValue(
151                                            "max",
152                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MAX_LABEL"),
153                                            "ametysicon-sort51",
154                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() - 1) : 0
155                                        ),
156                                        new StatisticsValue(
157                                            "median",
158                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_CONTENTREFERENCES_MEDIAN_LABEL"),
159                                            "ametysicon-maths-window-symbol-x",
160                                            nbContentsRefPerSitemap.size() > 0 ? nbContentsRefPerSitemap.get(nbContentsRefPerSitemap.size() / 2) : 0
161                                        )                                             
162                                    ),
163                                    false
164                                ),
165                                new StatisticsNode(
166                                    "content-references",
167                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_LABEL"),
168                                    "ametysicon-text70",
169                                    servicesCount, 
170                                    List.of(
171                                        new StatisticsValue(
172                                            "max",
173                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MAX_LABEL"),
174                                            "ametysicon-sort51",
175                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() - 1) : 0
176                                        ),
177                                        new StatisticsValue(
178                                            "median",
179                                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SITEMAP_SERVICES_MEDIAN_LABEL"),
180                                            "ametysicon-maths-window-symbol-x",
181                                            nbServicesPerSitemap.size() > 0 ? nbServicesPerSitemap.get(nbServicesPerSitemap.size() / 2) : 0
182                                        )                                             
183                                    ),
184                                    false
185                                )
186                            ),
187                            false
188                        ),
189                        new StatisticsNode(
190                            "contents",
191                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_LABEL"),
192                            "ametysicon-text70",
193                            contentsForSites.get("COUNT"),
194                            List.of(
195                                new StatisticsValue(
196                                    "orphans",
197                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_ORPHANS_LABEL"),
198                                    "ametysicon-sort51",
199                                    contentsForSites.get("ORPHANS")
200                                ),
201                                new StatisticsValue(
202                                    "shared",
203                                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_CONTENTS_EXTERNAL_LABEL"),
204                                    "ametysicon-maths-window-symbol-x",
205                                    contentsForSites.get("EXTERNAL")
206                                )                                       
207                            ),
208                            false
209                        ),
210                        new StatisticsNode(
211                            "services",
212                            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_LABEL"),
213                            "ametysicon-system-monitoring",
214                            null,
215                            _getServices(),
216                            false
217                        ),
218                        _processResources()
219                    ),
220                    true
221                )
222            ),
223            true
224        );
225    }
226    
227    private List<List<Map<String, Long>>> _countPages()
228    {
229        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
230        {
231            List<List<Map<String, Long>>> sitesValues = new ArrayList<>();
232            
233            for (Site site : sites)
234            {
235                sitesValues.add(_countPages(site));
236            }
237            
238            return sitesValues;
239        }
240    }
241    
242    private List<Map<String, Long>> _countPages(Site site)
243    {
244        List<Map<String, Long>> siteValues = new ArrayList<>();
245
246        for (Sitemap sitemap : site.getSitemaps())
247        {
248            siteValues.add(_countPages(sitemap));
249        }
250        
251        return siteValues;
252    }   
253    
254    private Map<String, Long> _countPages(SitemapElement pages)
255    {
256        Map<String, Long> sitemapElementValues = _newCountMap();
257        
258        AmetysObjectIterable<? extends Page> childPages = pages.getChildrenPages();
259        
260        sitemapElementValues.put("NBPAGES", sitemapElementValues.get("NBPAGES") + childPages.getSize());
261        
262        int cacheableCount = 0;
263        
264        for (Page page : childPages)
265        {
266            boolean isCacheable = _pageHelper.isCacheable(page);
267            for (Zone zone : page.getZones())
268            {
269                AmetysObjectIterable<? extends ZoneItem> zoneItems = zone.getZoneItems();
270                for (ZoneItem item : zoneItems)
271                {
272                    switch (item.getType())
273                    {
274                        case CONTENT:
275                            sitemapElementValues.put("CONTENTS", sitemapElementValues.get("CONTENTS") + 1);
276                            break;
277                        case SERVICE:
278                            sitemapElementValues.put("SERVICES", sitemapElementValues.get("SERVICES") + 1);
279                            break;
280                        default:
281                            break;
282                    }
283                }
284            }
285            
286            if (isCacheable)
287            {
288                cacheableCount++;
289            }
290            
291            _sumMaps(sitemapElementValues, _countPages(page));
292        }
293        
294        sitemapElementValues.put("NBCACHEABLE", sitemapElementValues.get("NBCACHEABLE") + cacheableCount);
295        
296        return sitemapElementValues;
297    }
298    
299    
300    private Map<String, Long> _newCountMap()
301    {
302        Map<String, Long> countMap = new HashMap<>();
303        countMap.put("NBPAGES", 0L); // Number of pages
304        countMap.put("NBCACHEABLE", 0L); // Number of cacheable pages
305        countMap.put("SERVICES", 0L); // Number of services
306        countMap.put("CONTENTS", 0L); // Number of contents
307        return countMap;
308    }
309    
310    private void _sumMaps(Map<String, Long> mainMap, Map<String, Long> mapToAdd)
311    {
312        for (String k : mainMap.keySet())
313        {
314            mainMap.put(k, mainMap.get(k) + mapToAdd.get(k));
315        }
316    }
317    
318    private List<Statistics> _getServices()
319    {
320        List<Statistics> stats = new ArrayList<>();
321        
322        for (String serviceId : _serviceEP.getExtensionsIds())
323        {
324            Service service = _serviceEP.getExtension(serviceId);
325            
326            stats.add(new StatisticsValue(
327                serviceId,
328                new I18nizableText(_i18nUtils.translate(service.getLabel())), // Translate now for sorting
329                StringUtils.defaultIfBlank(service.getIconGlyph(), "ametysicon-system-monitoring"),
330                _countAllServices(serviceId)
331            ));
332        }
333        
334        stats.sort(new Comparator<>() {
335            public int compare(Statistics o1, Statistics o2)
336            {
337                return o1.getLabel().getLabel().compareTo(o2.getLabel().getLabel());
338            }
339        });
340        
341        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'"
342                + _serviceEP.getExtensionsIds().stream().map(id -> " and not(@ametys-internal:service='" + id + "')").collect(Collectors.joining())
343                + "]";        
344        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
345        {
346            stats.add(0, new StatisticsValue(
347                "unknown",
348                new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_SERVICES_UNKNOWN_LABEL"),
349                "ametysicon-file-extension-generic-unknown",
350                serviceZoneItems.getSize()
351            ));
352        }
353        
354        return stats;
355    }
356
357    private long _countAllServices(String serviceId)
358    {
359        String query = "//element(ametys:zoneItem, ametys:zoneItem)[@ametys-internal:type='SERVICE'" 
360                + (serviceId != null ? " and @ametys-internal:service='" + serviceId + "'" : "")
361                + "]";
362        try (AmetysObjectIterable<ZoneItem> serviceZoneItems = _ametysResolver.query(query))
363        {
364            return serviceZoneItems.getSize();
365        }
366    }    
367    
368    private Map<String, Long> _processContentsForSites()
369    {
370        Map<String, Long> values = new HashMap<>();
371        values.put("COUNT", 0L);
372        values.put("ORPHANS", 0L);
373        values.put("EXTERNAL", 0L);
374        
375        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
376        {
377            for (Site site : sites)
378            {
379                _processContents (site, values);
380            }
381        }   
382        catch (Exception e)
383        {
384            throw new IllegalStateException(e);
385        }
386        
387        return values;
388    }
389    
390    private void _processContents (Site site, Map<String, Long> values)
391    {
392        // Number of contents
393        AmetysObjectIterable<Content> contents = site.getContents();
394        values.put("COUNT", values.get("COUNT") + contents.getSize());
395        
396        int orphans = 0;
397        int external = 0;
398        for (Content content : contents)
399        {
400            if (content instanceof WebContent webContent 
401                && webContent.getReferencingPages().size() == 0)
402            {
403                orphans++;
404            }
405            if (content instanceof SharedContent)
406            {
407                external++;
408            }
409        }
410        
411        // Number of orphan contents
412        values.put("ORPHANS", values.get("ORPHANS") + orphans);
413        
414        // Number of shared contents
415        values.put("EXTERNAL", values.get("EXTERNAL") + external);
416    }
417    
418    private Statistics _processResources()
419    {
420        HashMap<String, Long> values = new HashMap<>();
421        values.put("FOLDERCOUNT", 0L);
422        values.put("RESOURCECOUNT", 0L);
423        values.put("TOTALSIZE", 0L);
424        
425        try (AmetysObjectIterable<Site> sites = _siteManager.getSites())
426        {
427            for (Site site : sites)
428            {
429                _processResources (site.getRootResources(), values);
430            }
431        }
432        
433        return new StatisticsNode(
434            "explorer",
435            new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_LABEL"),
436            "ametysicon-folder249",
437            values.get("TOTALSIZE"),
438            List.of(
439                new StatisticsValue(
440                    "folders",
441                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FOLDER_LABEL"),
442                    "ametysicon-folder250",
443                    values.get("FOLDERCOUNT")
444                ),
445                new StatisticsValue(
446                    "files",
447                    new I18nizableText("plugin.web", "PLUGINS_WEB_STATISTICS_SITES_EXPLORER_FILES_LABEL"),
448                    "ametysicon-document77",
449                    values.get("RESOURCECOUNT")
450                )                
451            ),
452            false
453        );
454    }
455    
456    private void _processResources(TraversableAmetysObject resourceContainer, HashMap<String, Long> values)
457    {
458        AmetysObjectIterable<? extends AmetysObject> objects = resourceContainer.getChildren();
459        
460        for (AmetysObject object : objects)
461        {
462            // Traverse the child nodes if depth < 0 (generate all) or depth > 0 (we're not in the last level).
463            if (object instanceof ResourceCollection)
464            {
465                values.put("FOLDERCOUNT", values.get("FOLDERCOUNT") + 1);
466                
467                _processResources((ResourceCollection) object, values);
468            }
469            else if (object instanceof Resource)
470            {
471                Resource resource = (Resource) object;
472                
473                values.put("RESOURCECOUNT", values.get("RESOURCECOUNT") + 1);
474                values.put("TOTALSIZE", values.get("TOTALSIZE") + resource.getLength());
475            }
476        }
477    }
478}