Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.webanalytics.matomo;
017
018import java.time.Duration;
019import java.time.LocalDate;
020import java.util.List;
021import java.util.Map;
022import java.util.Optional;
023
024import org.apache.avalon.framework.activity.Disposable;
025import org.apache.avalon.framework.activity.Initializable;
026import org.apache.avalon.framework.component.Component;
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang.StringUtils;
031import org.apache.hc.client5.http.classic.methods.HttpPost;
032import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
033import org.apache.hc.core5.http.io.entity.EntityUtils;
034import org.apache.hc.core5.io.CloseMode;
035
036import org.ametys.core.cache.AbstractCacheManager;
037import org.ametys.core.cache.Cache;
038import org.ametys.core.util.HttpUtils;
039import org.ametys.core.util.JSONUtils;
040import org.ametys.core.util.LambdaUtils;
041import org.ametys.core.util.LambdaUtils.LambdaException;
042import org.ametys.core.util.URIUtils;
043import org.ametys.plugins.core.impl.cache.AbstractCacheKey;
044import org.ametys.runtime.config.Config;
045import org.ametys.runtime.i18n.I18nizableText;
046import org.ametys.runtime.plugin.component.AbstractLogEnabled;
047import org.ametys.web.repository.site.Site;
048
049/**
050 * The matomo data helper
051 */
052public class MatomoDataHelper extends AbstractLogEnabled implements Component, Initializable, Serviceable, Disposable
053{
054    /** The component role. */
055    public static final String ROLE = MatomoDataHelper.class.getName();
056    
057    /** The Matomo URL config parameter name */
058    public static final String MATOMO_URL_CONFIG = "piwik-url";
059    
060    /** The Matomo site id config parameter name */
061    public static final String MATOMO_SITE_ID_SITE_CONFIG = "piwik-id";
062
063    private static final String __MATOMO_TOKEN_CONFIG = "matomo-token";
064    private static final String __MATOMO_CACHE_LIFETIME_CONFIG = "matomo-cache-lifetime";
065    private static final String __MATOMO_SITE_LIVE_STATS_CONFIG = "matomo-live-stats";
066    private static final String __MATOMO_LAST_DAYS_SITE_CONFIG = "matomo-last-days";
067    private static final String __MATOMO_BEGIN_YEAR_SITE_CONFIG = "matomo-begin-year";
068    
069    private static final String __CACHE_COMPUTED_VISITS = "computed-visits";
070    private static final String __CACHE_LIVE_VISITS = "live-visits";
071    
072    /** The cache manager */
073    protected AbstractCacheManager _cacheManager;
074    
075    /** JSON Utils */
076    protected JSONUtils _jsonUtils;
077    
078    private CloseableHttpClient _httpClient;
079    private String _matomoServerURL; 
080    private String _matomoToken; 
081    
082    /**
083     * The computed matomo request type.
084     */
085    private enum RequestType
086    {
087        /** Computed visits from begin year */
088        BEGIN_YEAR,
089        /** Computed visits of the current day */
090        CURRENT_DAY,
091        /** Computed visits of from the last days */
092        LAST_DAYS;
093    }
094    
095    
096    public void service(ServiceManager manager) throws ServiceException
097    {
098        _cacheManager = (AbstractCacheManager) manager.lookup(AbstractCacheManager.ROLE);
099        _jsonUtils = (JSONUtils) manager.lookup(JSONUtils.ROLE);
100    }
101    
102    @Override
103    public void initialize() throws Exception
104    {
105        // The duration of 6 hours is an arbitrary value because computed data in Matomo are re-computed every 6 hours. But the cache need to be recalculate only once every day.
106        // But if the value is put in the cache at 23pm, we don't want the cache to be recalculate at 23pm the next day ... 6 hours seems to be fine ...
107        _cacheManager.createMemoryCache(__CACHE_COMPUTED_VISITS, 
108                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_COMPUTED_VISITS_LABEL"),
109                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_COMPUTED_VISITS_DESC"),
110                true,
111                Duration.ofHours(6)); 
112        
113        // Live stats for pages will be computed every n minutes (n is a config parameter)
114        Long cacheLifeTime = Config.getInstance().getValue(__MATOMO_CACHE_LIFETIME_CONFIG, true, 5L);
115        _cacheManager.createMemoryCache(__CACHE_LIVE_VISITS, 
116                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_LIVE_VISITS_LABEL"),
117                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_LIVE_VISITS_DESC"),
118                true,
119                Duration.ofMinutes(cacheLifeTime)); 
120        
121        _httpClient = HttpUtils.createHttpClient(5, 5); // Max 5 attempts simultany and 5 sec of timeout
122        _matomoServerURL = Config.getInstance().getValue(MATOMO_URL_CONFIG);
123        _matomoToken = Config.getInstance().getValue(__MATOMO_TOKEN_CONFIG);
124    }
125    
126    private void _checkMatomoConfig(Site site) throws MatomoException
127    {
128        if (!site.getValue(__MATOMO_SITE_LIVE_STATS_CONFIG, true, false))
129        {
130            throw new MatomoException("Can't request the matomo stats because the site does not enable it.");
131        }
132        
133        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
134        if (StringUtils.isBlank(_matomoServerURL) || StringUtils.isBlank(_matomoToken) || StringUtils.isBlank(siteId))
135        {
136            throw new MatomoException("One or more of these following parameters are empty: the matamo server URL, the matomo token or the matomo site id. Can't access to matomo data without these parameters");
137        }
138    }
139    
140    /**
141     * Get the number of days from now to computed the number of visits
142     * @param site the site
143     * @return the number of days
144     */
145    public long getMatomoSiteNbLastDays(Site site)
146    {
147        return site.getValue(__MATOMO_LAST_DAYS_SITE_CONFIG, true, 30L);
148    }
149    
150    /**
151     * Get the number of visits since the begin year for the given page and site
152     * @param site the site 
153     * @param pageUrl the page URL
154     * @return the number of visits
155     * @throws MatomoException if an error with Matomo occurred
156     */
157    public int getNbTotalVisits(Site site, String pageUrl) throws MatomoException
158    {
159        _checkMatomoConfig(site);
160
161        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
162        String beginYear = site.getValue(__MATOMO_BEGIN_YEAR_SITE_CONFIG);
163        
164        // Number of visits of the current day (Live from Matomo)
165        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl);
166        
167        // Number of visits of the current day (Computed from Matomo)
168        Integer nbComputedVisitsToday = _getComputedVisitsOfCurrentDay(siteId, pageUrl);
169        
170        // Number of visits from the begin years but with the computed current day
171        Integer nbComputedVisitsFromLastYears = _getComputedVisitsFromBeginYear(siteId, pageUrl, beginYear);
172        
173        return nbComputedVisitsFromLastYears - nbComputedVisitsToday + liveVisitsOfDay;
174    }
175    
176    /**
177     * Get the number of visits from the last days for the given page and site 
178     * @param site the site 
179     * @param pageUrl the page URL
180     * @return the number of visits
181     * @throws MatomoException if an error with Matomo occurred
182     */
183    public int getNbVisitsFromLastDays(Site site, String pageUrl) throws MatomoException
184    {
185        _checkMatomoConfig(site);
186        
187        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
188        long lastDays = getMatomoSiteNbLastDays(site);
189        
190        // Number of visits of the current day (Live from Matomo)
191        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl);
192
193        // Visits from the last n days (starting from yesterday)
194        Integer computedVisitsFromLastDays = _getComputedVisitsFromLastDays(siteId, pageUrl, lastDays);
195        
196        return computedVisitsFromLastDays + liveVisitsOfDay;
197    }
198    
199    /**
200     * Get the live number of visits of today for the given page and site
201     * @param siteId the site identifier in Matomo
202     * @param pageUrl the page URL
203     * @return the number of visits
204     * @throws MatomoException if an error with Matomo occurred
205     */
206    protected int _getLiveVisitsOfDay(String siteId, String pageUrl) throws MatomoException
207    {
208        StringBuilder url = new StringBuilder(_matomoServerURL);
209        url.append("/?module=API&method=Live.getLastVisitsDetails");
210        url.append("&format=JSON");
211        url.append("&period=day");
212        url.append("&date=today");
213        url.append("&idSite=");
214        url.append(siteId);
215        url.append("&token_auth=");
216        url.append(_matomoToken);
217        
218        try
219        {
220            List<Map<String, Object>> liveVisits = _getLiveVisitsCache().get(siteId, LambdaUtils.wrap(k -> _requestLiveVisitsOfDay(url.toString())));
221            return _getTodayNbVisits(liveVisits, pageUrl);
222        }
223        catch (LambdaException e) 
224        {
225            if (e.getCause() instanceof RuntimeException re) 
226            {
227                throw re;
228            }
229            else if (e.getCause() instanceof MatomoException m) 
230            {
231                throw m;
232            }
233            else 
234            {
235                throw new RuntimeException(e.getCause());
236            }
237        }
238    }
239    
240    @SuppressWarnings("unchecked")
241    private List<Map<String, Object>> _requestLiveVisitsOfDay(String url) throws MatomoException
242    {
243        return (List<Map<String, Object>>) _executeMatomoRequest(url, "list");
244    }
245    
246    /**
247     * Get the computed number of visits from the last years starting from the begin year for the given page and site
248     * @param siteId the site identifier in matomo
249     * @param pageUrl the page URL
250     * @param beginYear the begin year
251     * @return the number of visits
252     * @throws MatomoException if an error with Matomo occurred
253     */
254    protected int _getComputedVisitsFromBeginYear(String siteId, String pageUrl, String beginYear) throws MatomoException
255    {
256        String truncateURL = _truncateURL(pageUrl);
257
258        LocalDate beginDate = LocalDate.of(Integer.valueOf(beginYear), 1, 1);
259        LocalDate now = LocalDate.now();
260        
261        int nbYear = now.getYear() - beginDate.getYear() + 1;
262        
263        StringBuilder url = new StringBuilder(_matomoServerURL);
264        url.append("/?module=API&method=Actions.getPageUrl");
265        url.append("&format=JSON");
266        url.append("&period=year");
267        url.append("&date=last");
268        url.append(nbYear);
269        url.append("&idSite=");
270        url.append(siteId);
271        url.append("&pageUrl=");
272        url.append(URIUtils.encodeParameter(truncateURL));
273        url.append("&token_auth=");
274        url.append(_matomoToken);
275        
276        try
277        {
278            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.BEGIN_YEAR, truncateURL);
279            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsFromBeginYear(url.toString())));
280        }
281        catch (LambdaException e)
282        {
283            if (e.getCause() instanceof RuntimeException re) 
284            {
285                throw re;
286            }
287            else if (e.getCause() instanceof MatomoException m) 
288            {
289                throw m;
290            }
291            else 
292            {
293                throw new RuntimeException(e.getCause());
294            }
295        }
296    }
297    
298    private Integer _requestComputedVisitsFromBeginYear(String url) throws MatomoException
299    {
300        @SuppressWarnings("unchecked")
301        Map<String, Object> json = (Map<String, Object>) _executeMatomoRequest(url, "map");
302        return _getNbVisitsFromMap(json);
303    }
304    
305    /**
306     * Get the computed number of visits of today for the given page and site
307     * @param siteId the site identifier in matomo
308     * @param pageUrl the page URL
309     * @return the number of visits
310     * @throws MatomoException if an error with Matomo occurred
311     */
312    protected int _getComputedVisitsOfCurrentDay(String siteId, String pageUrl) throws MatomoException
313    {
314        String truncateURL = _truncateURL(pageUrl);
315
316        StringBuilder url = new StringBuilder(_matomoServerURL);
317        url.append("/?module=API&method=Actions.getPageUrl");
318        url.append("&format=JSON");
319        url.append("&period=day");
320        url.append("&date=today");
321        url.append("&idSite=");
322        url.append(siteId);
323        url.append("&pageUrl=");
324        url.append(URIUtils.encodeParameter(truncateURL));
325        url.append("&token_auth=");
326        url.append(_matomoToken);
327        
328        try
329        {
330            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.CURRENT_DAY, truncateURL);
331            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsOfCurrentDay(url.toString())));
332        }
333        catch (LambdaException e) 
334        {
335            if (e.getCause() instanceof RuntimeException re) 
336            {
337                throw re;
338            }
339            else if (e.getCause() instanceof MatomoException m) 
340            {
341                throw m;
342            }
343            else 
344            {
345                throw new RuntimeException(e.getCause());
346            }
347        }
348    }
349    
350    private Integer _requestComputedVisitsOfCurrentDay(String url) throws MatomoException
351    {
352        @SuppressWarnings("unchecked")
353        List<Map<String, Object>> json = (List<Map<String, Object>>) _executeMatomoRequest(url, "list");
354        return _getNbVisitsFromList(json);
355    }
356    
357    /**
358     * Get the computed number of visits from the last n days (starting from yesterday) for the given page and site
359     * @param siteId the site identifier in matomo
360     * @param pageUrl the page URL
361     * @param nbDay the number of day
362     * @return the number of visits
363     * @throws MatomoException if an error with Matomo occurred
364     */
365    protected int _getComputedVisitsFromLastDays(String siteId, String pageUrl, long nbDay) throws MatomoException
366    {
367        String truncateURL = _truncateURL(pageUrl);
368
369        StringBuilder url = new StringBuilder(_matomoServerURL);
370        url.append("/?module=API&method=Actions.getPageUrl");
371        url.append("&format=JSON");
372        url.append("&period=day");
373        url.append("&date=previous");
374        url.append(nbDay);
375        url.append("&idSite=");
376        url.append(siteId);
377        url.append("&pageUrl=");
378        url.append(URIUtils.encodeParameter(truncateURL));
379        url.append("&token_auth=");
380        url.append(_matomoToken);
381        
382        try
383        {
384            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.LAST_DAYS, truncateURL);
385            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsFromLastDays(url.toString())));
386        }
387        catch (LambdaException e) 
388        {
389            if (e.getCause() instanceof RuntimeException re) 
390            {
391                throw re;
392            }
393            else if (e.getCause() instanceof MatomoException m) 
394            {
395                throw m;
396            }
397            else 
398            {
399                throw new RuntimeException(e.getCause());
400            }
401        }
402    }
403    
404    private Integer _requestComputedVisitsFromLastDays(String url) throws MatomoException
405    {
406        @SuppressWarnings("unchecked")
407        Map<String, Object> json = (Map<String, Object>) _executeMatomoRequest(url, "map");
408        return _getNbVisitsFromMap(json);
409    }
410    
411    private Object _executeMatomoRequest(String matomoUrl, String returnType) throws MatomoException
412    {
413        try
414        {
415            if (getLogger().isInfoEnabled())
416            {
417                getLogger().info("A request is send to Matomo with url '" + matomoUrl + "'");
418            }
419            
420            HttpPost post = new HttpPost(matomoUrl);
421            return _httpClient.execute(post, httpResponse -> {
422                if (httpResponse.getCode() != 200)
423                {
424                    throw new IllegalStateException("Could not join matomo for URL " + matomoUrl + ". Error code " + httpResponse.getCode());
425                }
426                
427                if ("map".equals(returnType))
428                {
429                    return _jsonUtils.convertJsonToMap(EntityUtils.toString(httpResponse.getEntity()));
430                }
431                else if ("list".equals(returnType))
432                {
433                    return _jsonUtils.convertJsonToList(EntityUtils.toString(httpResponse.getEntity()));
434                }
435                
436                throw new IllegalArgumentException("Invalid return type for request with url '" + matomoUrl + "'");
437            });
438        }
439        catch (Exception e)
440        {
441            throw new MatomoException("An error occurred requesting Matomo for url '" + matomoUrl + "'", e);
442        }
443    }
444    
445    @SuppressWarnings("unchecked")
446    private int _getNbVisitsFromMap(Map<String, Object> jsonAsMap)
447    {
448        int nbVisits = 0;
449        for (String key : jsonAsMap.keySet())
450        {
451            nbVisits += _getNbVisitsFromList((List<Map<String, Object>>) jsonAsMap.get(key));
452        }
453        
454        return nbVisits;
455    }
456    
457    private int _getNbVisitsFromList(List<Map<String, Object>> data)
458    {
459        if (!data.isEmpty())
460        {
461            return (Integer) data.get(0).getOrDefault("nb_visits", 0);
462        }
463        
464        return 0;
465    }
466    
467    private int _getTodayNbVisits(List<Map<String, Object>> jsonAsMap, String pageUrl)
468    {
469        int nbVisits = 0;
470        if (!jsonAsMap.isEmpty())
471        {
472            String truncateURL = _truncateURL(pageUrl);
473            
474            LocalDate today = LocalDate.now();
475            for (Map<String, Object> visitInfo : jsonAsMap)
476            {
477                String visitDateAsString = (String) visitInfo.get("serverDate");
478                LocalDate visitDate = LocalDate.parse(visitDateAsString);
479                if (visitDate.equals(today))
480                {
481                    @SuppressWarnings("unchecked")
482                    List<Map<String, Object>> actions = (List<Map<String, Object>>) visitInfo.get("actionDetails");
483                    Optional<String> pageVisits = actions.stream()
484                            .map(a -> (String) a.get("url"))
485                            .map(this::_truncateURL)
486                            .filter(url -> truncateURL.equals(url))
487                            .findAny();
488                    
489                    if (pageVisits.isPresent())
490                    {
491                        nbVisits++;
492                    }
493                }
494            }
495        }
496        return nbVisits;
497    }
498    
499    /**
500     * Truncate parameters and fragments of the URL
501     * @param pageUrl the page URL
502     * @return the page URL without parameters and fragments
503     */
504    protected String _truncateURL(String pageUrl)
505    {
506        return StringUtils.substringBefore(StringUtils.substringBefore(pageUrl, "?"), "#");
507    }
508    
509    private Cache<MatomoComputedCacheKey, Integer> _getComputedVisitsCache()
510    {
511        return _cacheManager.get(__CACHE_COMPUTED_VISITS);
512    }
513    
514    private Cache<String, List<Map<String, Object>>> _getLiveVisitsCache()
515    {
516        return _cacheManager.get(__CACHE_LIVE_VISITS);
517    }
518    
519    private static final class MatomoComputedCacheKey extends AbstractCacheKey
520    {
521        public MatomoComputedCacheKey(RequestType type, String pageUrl)
522        {
523            super(type, pageUrl);
524        }
525        
526        public static MatomoComputedCacheKey of(RequestType type, String pageUrl)
527        {
528            return new MatomoComputedCacheKey(type, pageUrl);
529        }
530    }
531    
532    public void dispose()
533    {
534        _httpClient.close(CloseMode.GRACEFUL);
535    }
536}