Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.webanalytics.matomo;
017
018import java.time.Duration;
019import java.time.LocalDate;
020import java.util.List;
021import java.util.Map;
022import java.util.Optional;
023
024import org.apache.avalon.framework.activity.Disposable;
025import org.apache.avalon.framework.activity.Initializable;
026import org.apache.avalon.framework.component.Component;
027import org.apache.avalon.framework.service.ServiceException;
028import org.apache.avalon.framework.service.ServiceManager;
029import org.apache.avalon.framework.service.Serviceable;
030import org.apache.commons.lang3.StringUtils;
031import org.apache.hc.client5.http.classic.methods.HttpPost;
032import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
033import org.apache.hc.core5.http.io.entity.EntityUtils;
034import org.apache.hc.core5.io.CloseMode;
035
036import org.ametys.core.cache.AbstractCacheManager;
037import org.ametys.core.cache.Cache;
038import org.ametys.core.util.HttpUtils;
039import org.ametys.core.util.JSONUtils;
040import org.ametys.core.util.LambdaUtils;
041import org.ametys.core.util.LambdaUtils.LambdaException;
042import org.ametys.core.util.URIUtils;
043import org.ametys.plugins.core.impl.cache.AbstractCacheKey;
044import org.ametys.runtime.config.Config;
045import org.ametys.runtime.i18n.I18nizableText;
046import org.ametys.runtime.plugin.component.AbstractLogEnabled;
047import org.ametys.web.repository.site.Site;
048
049/**
050 * The matomo data helper
051 */
052public class MatomoDataHelper extends AbstractLogEnabled implements Component, Initializable, Serviceable, Disposable
053{
054    /** The component role. */
055    public static final String ROLE = MatomoDataHelper.class.getName();
056    
057    /** The Matomo URL config parameter name */
058    public static final String MATOMO_URL_CONFIG = "piwik-url";
059    
060    /** The Matomo token config parameter name */
061    public static final String MATOMO_TOKEN_CONFIG = "matomo-token";
062
063    /** The Matomo site id config parameter name */
064    public static final String MATOMO_SITE_ID_SITE_CONFIG = "piwik-id";
065
066    private static final String __MATOMO_CACHE_LIFETIME_CONFIG = "matomo-cache-lifetime";
067    private static final String __MATOMO_SITE_LIVE_STATS_CONFIG = "matomo-live-stats";
068    private static final String __MATOMO_LAST_DAYS_SITE_CONFIG = "matomo-last-days";
069    private static final String __MATOMO_BEGIN_YEAR_SITE_CONFIG = "matomo-begin-year";
070    
071    private static final String __CACHE_COMPUTED_VISITS = "computed-visits";
072    private static final String __CACHE_LIVE_VISITS = "live-visits";
073    
074    /** The cache manager */
075    protected AbstractCacheManager _cacheManager;
076    
077    /** JSON Utils */
078    protected JSONUtils _jsonUtils;
079    
080    private CloseableHttpClient _httpClient;
081    private String _matomoServerURL; 
082    private String _matomoToken; 
083    
084    /**
085     * The computed matomo request type.
086     */
087    private enum RequestType
088    {
089        /** Computed visits from begin year */
090        BEGIN_YEAR,
091        /** Computed visits of the current day */
092        CURRENT_DAY,
093        /** Computed visits of from the last days */
094        LAST_DAYS;
095    }
096    
097    
098    public void service(ServiceManager manager) throws ServiceException
099    {
100        _cacheManager = (AbstractCacheManager) manager.lookup(AbstractCacheManager.ROLE);
101        _jsonUtils = (JSONUtils) manager.lookup(JSONUtils.ROLE);
102    }
103    
104    @Override
105    public void initialize() throws Exception
106    {
107        // The duration of 6 hours is an arbitrary value because computed data in Matomo are re-computed every 6 hours. But the cache need to be recalculate only once every day.
108        // But if the value is put in the cache at 23pm, we don't want the cache to be recalculate at 23pm the next day ... 6 hours seems to be fine ...
109        _cacheManager.createMemoryCache(__CACHE_COMPUTED_VISITS, 
110                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_COMPUTED_VISITS_LABEL"),
111                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_COMPUTED_VISITS_DESC"),
112                true,
113                Duration.ofHours(6)); 
114        
115        // Live stats for pages will be computed every n minutes (n is a config parameter)
116        Long cacheLifeTime = Config.getInstance().getValue(__MATOMO_CACHE_LIFETIME_CONFIG, true, 5L);
117        _cacheManager.createMemoryCache(__CACHE_LIVE_VISITS, 
118                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_LIVE_VISITS_LABEL"),
119                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_LIVE_VISITS_DESC"),
120                true,
121                Duration.ofMinutes(cacheLifeTime)); 
122        
123        _httpClient = HttpUtils.createHttpClient(5, 5); // Max 5 attempts simultany and 5 sec of timeout
124        _matomoServerURL = Config.getInstance().getValue(MATOMO_URL_CONFIG);
125        _matomoToken = Config.getInstance().getValue(MATOMO_TOKEN_CONFIG);
126    }
127    
128    private void _checkMatomoConfig(Site site) throws MatomoException
129    {
130        if (!site.getValue(__MATOMO_SITE_LIVE_STATS_CONFIG, true, false))
131        {
132            throw new MatomoException("Can't request the matomo stats because the site does not enable it.");
133        }
134        
135        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
136        if (StringUtils.isBlank(_matomoServerURL) || StringUtils.isBlank(_matomoToken) || StringUtils.isBlank(siteId))
137        {
138            throw new MatomoException("One or more of these following parameters are empty: the matamo server URL, the matomo token or the matomo site id. Can't access to matomo data without these parameters");
139        }
140    }
141    
142    /**
143     * Get the number of days from now to computed the number of visits
144     * @param site the site
145     * @return the number of days
146     */
147    public long getMatomoSiteNbLastDays(Site site)
148    {
149        return site.getValue(__MATOMO_LAST_DAYS_SITE_CONFIG, true, 30L);
150    }
151    
152    /**
153     * Get the number of visits since the begin year for the given page and site
154     * @param site the site 
155     * @param pageUrl the page URL
156     * @return the number of visits
157     * @throws MatomoException if an error with Matomo occurred
158     */
159    public int getNbTotalVisits(Site site, String pageUrl) throws MatomoException
160    {
161        _checkMatomoConfig(site);
162
163        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
164        String beginYear = site.getValue(__MATOMO_BEGIN_YEAR_SITE_CONFIG);
165        
166        // Number of visits of the current day (Live from Matomo)
167        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl);
168        
169        // Number of visits of the current day (Computed from Matomo)
170        Integer nbComputedVisitsToday = _getComputedVisitsOfCurrentDay(siteId, pageUrl);
171        
172        // Number of visits from the begin years but with the computed current day
173        Integer nbComputedVisitsFromLastYears = _getComputedVisitsFromBeginYear(siteId, pageUrl, beginYear);
174        
175        return nbComputedVisitsFromLastYears - nbComputedVisitsToday + liveVisitsOfDay;
176    }
177    
178    /**
179     * Get the number of visits from the last days for the given page and site 
180     * @param site the site 
181     * @param pageUrl the page URL
182     * @return the number of visits
183     * @throws MatomoException if an error with Matomo occurred
184     */
185    public int getNbVisitsFromLastDays(Site site, String pageUrl) throws MatomoException
186    {
187        _checkMatomoConfig(site);
188        
189        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
190        long lastDays = getMatomoSiteNbLastDays(site);
191        
192        // Number of visits of the current day (Live from Matomo)
193        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl);
194
195        // Visits from the last n days (starting from yesterday)
196        Integer computedVisitsFromLastDays = _getComputedVisitsFromLastDays(siteId, pageUrl, lastDays);
197        
198        return computedVisitsFromLastDays + liveVisitsOfDay;
199    }
200    
201    /**
202     * Get the live number of visits of today for the given page and site
203     * @param siteId the site identifier in Matomo
204     * @param pageUrl the page URL
205     * @return the number of visits
206     * @throws MatomoException if an error with Matomo occurred
207     */
208    protected int _getLiveVisitsOfDay(String siteId, String pageUrl) throws MatomoException
209    {
210        StringBuilder url = new StringBuilder(_matomoServerURL);
211        url.append("/?module=API&method=Live.getLastVisitsDetails");
212        url.append("&format=JSON");
213        url.append("&period=day");
214        url.append("&date=today");
215        url.append("&idSite=");
216        url.append(siteId);
217        url.append("&token_auth=");
218        url.append(_matomoToken);
219        
220        try
221        {
222            List<Map<String, Object>> liveVisits = _getLiveVisitsCache().get(siteId, LambdaUtils.wrap(k -> _requestLiveVisitsOfDay(url.toString())));
223            return _getTodayNbVisits(liveVisits, pageUrl);
224        }
225        catch (LambdaException e) 
226        {
227            if (e.getCause() instanceof RuntimeException re) 
228            {
229                throw re;
230            }
231            else if (e.getCause() instanceof MatomoException m) 
232            {
233                throw m;
234            }
235            else 
236            {
237                throw new RuntimeException(e.getCause());
238            }
239        }
240    }
241    
242    @SuppressWarnings("unchecked")
243    private List<Map<String, Object>> _requestLiveVisitsOfDay(String url) throws MatomoException
244    {
245        return (List<Map<String, Object>>) _executeMatomoRequest(url, "list");
246    }
247    
248    /**
249     * Get the computed number of visits from the last years starting from the begin year for the given page and site
250     * @param siteId the site identifier in matomo
251     * @param pageUrl the page URL
252     * @param beginYear the begin year
253     * @return the number of visits
254     * @throws MatomoException if an error with Matomo occurred
255     */
256    protected int _getComputedVisitsFromBeginYear(String siteId, String pageUrl, String beginYear) throws MatomoException
257    {
258        String truncateURL = _truncateURL(pageUrl);
259
260        LocalDate beginDate = LocalDate.of(Integer.valueOf(beginYear), 1, 1);
261        LocalDate now = LocalDate.now();
262        
263        int nbYear = now.getYear() - beginDate.getYear() + 1;
264        
265        StringBuilder url = new StringBuilder(_matomoServerURL);
266        url.append("/?module=API&method=Actions.getPageUrl");
267        url.append("&format=JSON");
268        url.append("&period=month");
269        url.append("&date=last");
270        url.append(nbYear * 12);
271        url.append("&idSite=");
272        url.append(siteId);
273        url.append("&pageUrl=");
274        url.append(URIUtils.encodeParameter(truncateURL));
275        url.append("&token_auth=");
276        url.append(_matomoToken);
277        
278        try
279        {
280            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.BEGIN_YEAR, truncateURL);
281            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsFromBeginYear(url.toString())));
282        }
283        catch (LambdaException e)
284        {
285            if (e.getCause() instanceof RuntimeException re) 
286            {
287                throw re;
288            }
289            else if (e.getCause() instanceof MatomoException m) 
290            {
291                throw m;
292            }
293            else 
294            {
295                throw new RuntimeException(e.getCause());
296            }
297        }
298    }
299    
300    private Integer _requestComputedVisitsFromBeginYear(String url) throws MatomoException
301    {
302        @SuppressWarnings("unchecked")
303        Map<String, Object> json = (Map<String, Object>) _executeMatomoRequest(url, "map");
304        return _getNbVisitsFromMap(json);
305    }
306    
307    /**
308     * Get the computed number of visits of today for the given page and site
309     * @param siteId the site identifier in matomo
310     * @param pageUrl the page URL
311     * @return the number of visits
312     * @throws MatomoException if an error with Matomo occurred
313     */
314    protected int _getComputedVisitsOfCurrentDay(String siteId, String pageUrl) throws MatomoException
315    {
316        String truncateURL = _truncateURL(pageUrl);
317
318        StringBuilder url = new StringBuilder(_matomoServerURL);
319        url.append("/?module=API&method=Actions.getPageUrl");
320        url.append("&format=JSON");
321        url.append("&period=day");
322        url.append("&date=today");
323        url.append("&idSite=");
324        url.append(siteId);
325        url.append("&pageUrl=");
326        url.append(URIUtils.encodeParameter(truncateURL));
327        url.append("&token_auth=");
328        url.append(_matomoToken);
329        
330        try
331        {
332            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.CURRENT_DAY, truncateURL);
333            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsOfCurrentDay(url.toString())));
334        }
335        catch (LambdaException e) 
336        {
337            if (e.getCause() instanceof RuntimeException re) 
338            {
339                throw re;
340            }
341            else if (e.getCause() instanceof MatomoException m) 
342            {
343                throw m;
344            }
345            else 
346            {
347                throw new RuntimeException(e.getCause());
348            }
349        }
350    }
351    
352    private Integer _requestComputedVisitsOfCurrentDay(String url) throws MatomoException
353    {
354        @SuppressWarnings("unchecked")
355        List<Map<String, Object>> json = (List<Map<String, Object>>) _executeMatomoRequest(url, "list");
356        return _getNbVisitsFromList(json);
357    }
358    
359    /**
360     * Get the computed number of visits from the last n days (starting from yesterday) for the given page and site
361     * @param siteId the site identifier in matomo
362     * @param pageUrl the page URL
363     * @param nbDay the number of day
364     * @return the number of visits
365     * @throws MatomoException if an error with Matomo occurred
366     */
367    protected int _getComputedVisitsFromLastDays(String siteId, String pageUrl, long nbDay) throws MatomoException
368    {
369        String truncateURL = _truncateURL(pageUrl);
370
371        StringBuilder url = new StringBuilder(_matomoServerURL);
372        url.append("/?module=API&method=Actions.getPageUrl");
373        url.append("&format=JSON");
374        url.append("&period=day");
375        url.append("&date=previous");
376        url.append(nbDay);
377        url.append("&idSite=");
378        url.append(siteId);
379        url.append("&pageUrl=");
380        url.append(URIUtils.encodeParameter(truncateURL));
381        url.append("&token_auth=");
382        url.append(_matomoToken);
383        
384        try
385        {
386            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.LAST_DAYS, truncateURL);
387            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsFromLastDays(url.toString())));
388        }
389        catch (LambdaException e) 
390        {
391            if (e.getCause() instanceof RuntimeException re) 
392            {
393                throw re;
394            }
395            else if (e.getCause() instanceof MatomoException m) 
396            {
397                throw m;
398            }
399            else 
400            {
401                throw new RuntimeException(e.getCause());
402            }
403        }
404    }
405    
406    private Integer _requestComputedVisitsFromLastDays(String url) throws MatomoException
407    {
408        @SuppressWarnings("unchecked")
409        Map<String, Object> json = (Map<String, Object>) _executeMatomoRequest(url, "map");
410        return _getNbVisitsFromMap(json);
411    }
412    
413    private Object _executeMatomoRequest(String matomoUrl, String returnType) throws MatomoException
414    {
415        try
416        {
417            if (getLogger().isInfoEnabled())
418            {
419                getLogger().info("A request is send to Matomo with url '" + matomoUrl + "'");
420            }
421            
422            HttpPost post = new HttpPost(matomoUrl);
423            return _httpClient.execute(post, httpResponse -> {
424                if (httpResponse.getCode() != 200)
425                {
426                    throw new IllegalStateException("Could not join matomo for URL " + matomoUrl + ". Error code " + httpResponse.getCode());
427                }
428                
429                if ("map".equals(returnType))
430                {
431                    return _jsonUtils.convertJsonToMap(EntityUtils.toString(httpResponse.getEntity()));
432                }
433                else if ("list".equals(returnType))
434                {
435                    return _jsonUtils.convertJsonToList(EntityUtils.toString(httpResponse.getEntity()));
436                }
437                
438                throw new IllegalArgumentException("Invalid return type for request with url '" + matomoUrl + "'");
439            });
440        }
441        catch (Exception e)
442        {
443            throw new MatomoException("An error occurred requesting Matomo for url '" + matomoUrl + "'", e);
444        }
445    }
446    
447    @SuppressWarnings("unchecked")
448    private int _getNbVisitsFromMap(Map<String, Object> jsonAsMap)
449    {
450        int nbVisits = 0;
451        for (String key : jsonAsMap.keySet())
452        {
453            nbVisits += _getNbVisitsFromList((List<Map<String, Object>>) jsonAsMap.get(key));
454        }
455        
456        return nbVisits;
457    }
458    
459    private int _getNbVisitsFromList(List<Map<String, Object>> data)
460    {
461        if (!data.isEmpty())
462        {
463            return (Integer) data.get(0).getOrDefault("nb_visits", 0);
464        }
465        
466        return 0;
467    }
468    
469    private int _getTodayNbVisits(List<Map<String, Object>> jsonAsMap, String pageUrl)
470    {
471        int nbVisits = 0;
472        if (!jsonAsMap.isEmpty())
473        {
474            String truncateURL = _truncateURL(pageUrl);
475            
476            LocalDate today = LocalDate.now();
477            for (Map<String, Object> visitInfo : jsonAsMap)
478            {
479                String visitDateAsString = (String) visitInfo.get("serverDate");
480                LocalDate visitDate = LocalDate.parse(visitDateAsString);
481                if (visitDate.equals(today))
482                {
483                    @SuppressWarnings("unchecked")
484                    List<Map<String, Object>> actions = (List<Map<String, Object>>) visitInfo.get("actionDetails");
485                    Optional<String> pageVisits = actions.stream()
486                            .map(a -> (String) a.get("url"))
487                            .map(this::_truncateURL)
488                            .filter(url -> truncateURL.equals(url))
489                            .findAny();
490                    
491                    if (pageVisits.isPresent())
492                    {
493                        nbVisits++;
494                    }
495                }
496            }
497        }
498        return nbVisits;
499    }
500    
501    /**
502     * Truncate parameters and fragments of the URL
503     * @param pageUrl the page URL
504     * @return the page URL without parameters and fragments
505     */
506    protected String _truncateURL(String pageUrl)
507    {
508        return StringUtils.substringBefore(StringUtils.substringBefore(pageUrl, "?"), "#");
509    }
510    
511    private Cache<MatomoComputedCacheKey, Integer> _getComputedVisitsCache()
512    {
513        return _cacheManager.get(__CACHE_COMPUTED_VISITS);
514    }
515    
516    private Cache<String, List<Map<String, Object>>> _getLiveVisitsCache()
517    {
518        return _cacheManager.get(__CACHE_LIVE_VISITS);
519    }
520    
521    private static final class MatomoComputedCacheKey extends AbstractCacheKey
522    {
523        public MatomoComputedCacheKey(RequestType type, String pageUrl)
524        {
525            super(type, pageUrl);
526        }
527        
528        public static MatomoComputedCacheKey of(RequestType type, String pageUrl)
529        {
530            return new MatomoComputedCacheKey(type, pageUrl);
531        }
532    }
533    
534    public void dispose()
535    {
536        _httpClient.close(CloseMode.GRACEFUL);
537    }
538}