Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.webanalytics.matomo;
017
018import java.io.IOException;
019import java.time.Duration;
020import java.time.LocalDate;
021import java.util.List;
022import java.util.Map;
023import java.util.Optional;
024
025import org.apache.avalon.framework.activity.Disposable;
026import org.apache.avalon.framework.activity.Initializable;
027import org.apache.avalon.framework.component.Component;
028import org.apache.avalon.framework.service.ServiceException;
029import org.apache.avalon.framework.service.ServiceManager;
030import org.apache.avalon.framework.service.Serviceable;
031import org.apache.commons.lang.StringUtils;
032import org.apache.http.client.config.RequestConfig;
033import org.apache.http.client.methods.HttpPost;
034import org.apache.http.impl.client.CloseableHttpClient;
035import org.apache.http.impl.client.HttpClientBuilder;
036import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
037import org.apache.http.util.EntityUtils;
038
039import org.ametys.core.cache.AbstractCacheManager;
040import org.ametys.core.cache.Cache;
041import org.ametys.core.util.JSONUtils;
042import org.ametys.core.util.LambdaUtils;
043import org.ametys.core.util.LambdaUtils.LambdaException;
044import org.ametys.core.util.URIUtils;
045import org.ametys.plugins.core.impl.cache.AbstractCacheKey;
046import org.ametys.runtime.config.Config;
047import org.ametys.runtime.i18n.I18nizableText;
048import org.ametys.runtime.plugin.component.AbstractLogEnabled;
049import org.ametys.web.repository.site.Site;
050
051/**
052 * The matomo data helper
053 */
054public class MatomoDataHelper extends AbstractLogEnabled implements Component, Initializable, Serviceable, Disposable
055{
056    /** The component role. */
057    public static final String ROLE = MatomoDataHelper.class.getName();
058    
059    /** The Matomo URL config parameter name */
060    public static final String MATOMO_URL_CONFIG = "piwik-url";
061    
062    /** The Matomo site id config parameter name */
063    public static final String MATOMO_SITE_ID_SITE_CONFIG = "piwik-id";
064
065    private static final String __MATOMO_TOKEN_CONFIG = "matomo-token";
066    private static final String __MATOMO_SITE_LIVE_STATS_CONFIG = "matomo-live-stats";
067    private static final String __MATOMO_LAST_DAYS_SITE_CONFIG = "matomo-last-days";
068    private static final String __MATOMO_BEGIN_YEAR_SITE_CONFIG = "matomo-begin-year";
069    
070    private static final String __CACHE_COMPUTED_VISITS = "computed-visits";
071    private static final String __CACHE_LIVE_VISITS = "live-visits";
072    
073    /** The cache manager */
074    protected AbstractCacheManager _cacheManager;
075    
076    /** JSON Utils */
077    protected JSONUtils _jsonUtils;
078    
079    private CloseableHttpClient _httpClient;
080    private String _matomoServerURL; 
081    private String _matomoToken; 
082    
083    /**
084     * The computed matomo request type.
085     */
086    private enum RequestType
087    {
088        /** Computed visits from begin year */
089        BEGIN_YEAR,
090        /** Computed visits of the current day */
091        CURRENT_DAY,
092        /** Computed visits of from the last days */
093        LAST_DAYS;
094    }
095    
096    
097    public void service(ServiceManager manager) throws ServiceException
098    {
099        _cacheManager = (AbstractCacheManager) manager.lookup(AbstractCacheManager.ROLE);
100        _jsonUtils = (JSONUtils) manager.lookup(JSONUtils.ROLE);
101    }
102    
103    @Override
104    public void initialize() throws Exception
105    {
106        // The duration of 6 hours is an arbitrary value because computed data in Matomo are re-computed every 6 hours. But the cache need to be recalculate only once every day.
107        // But if the value is put in the cache at 23pm, we don't want the cache to be recalculate at 23pm the next day ... 6 hours seems to be fine ...
108        _cacheManager.createMemoryCache(__CACHE_COMPUTED_VISITS, 
109                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_COMPUTED_VISITS_LABEL"),
110                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_COMPUTED_VISITS_DESC"),
111                true,
112                Duration.ofHours(6)); 
113        
114        // The duration is fixed at 5 minutes. Live stats for pages will be computed every 5 minutes.
115        _cacheManager.createMemoryCache(__CACHE_LIVE_VISITS, 
116                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_LIVE_VISITS_LABEL"),
117                new I18nizableText("plugin.web-analytics", "PLUGINS_WEB_ANALTICS_CACHE_MATOMO_LIVE_VISITS_DESC"),
118                true,
119                Duration.ofMinutes(5)); 
120        
121        _httpClient = _createHttpClient(5, 5); // Max 25 attempts simultany and 5 sec of timeout
122        _matomoServerURL = Config.getInstance().getValue(MATOMO_URL_CONFIG);
123        _matomoToken = Config.getInstance().getValue(__MATOMO_TOKEN_CONFIG);
124    }
125    
126    private CloseableHttpClient _createHttpClient(int maxConnections, int timeout)
127    {
128        PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
129        connectionManager.setMaxTotal(maxConnections);
130        connectionManager.setDefaultMaxPerRoute(maxConnections);
131        
132        RequestConfig config = RequestConfig.custom()
133                .setConnectTimeout(timeout * 1000)
134                .setConnectionRequestTimeout(timeout * 1000)
135                .setSocketTimeout(timeout * 1000)
136                .build();
137        
138        return HttpClientBuilder.create()
139                .setDefaultRequestConfig(config)
140                .setConnectionManager(connectionManager)
141                .build();
142    }
143    
144    private void _checkMatomoConfig(Site site) throws MatomoException
145    {
146        if (!site.getValue(__MATOMO_SITE_LIVE_STATS_CONFIG, true, false))
147        {
148            throw new MatomoException("Can't request the matomo stats because the site does not enable it.");
149        }
150        
151        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
152        if (StringUtils.isBlank(_matomoServerURL) || StringUtils.isBlank(_matomoToken) || StringUtils.isBlank(siteId))
153        {
154            throw new MatomoException("One or more of these following parameters are empty: the matamo server URL, the matomo token or the matomo site id. Can't access to matomo data without these parameters");
155        }
156    }
157    
158    /**
159     * Get the number of days from now to computed the number of visits
160     * @param site the site
161     * @return the number of days
162     */
163    public long getMatomoSiteNbLastDays(Site site)
164    {
165        return site.getValue(__MATOMO_LAST_DAYS_SITE_CONFIG, true, 30L);
166    }
167    
168    /**
169     * Get the number of visits since the begin year for the given page and site
170     * @param site the site 
171     * @param pageUrl the page URL
172     * @return the number of visits
173     * @throws MatomoException if an error with Matomo occurred
174     */
175    public int getNbTotalVisits(Site site, String pageUrl) throws MatomoException
176    {
177        _checkMatomoConfig(site);
178
179        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
180        String beginYear = site.getValue(__MATOMO_BEGIN_YEAR_SITE_CONFIG);
181        
182        // Number of visits of the current day (Live from Matomo)
183        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl);
184        
185        // Number of visits of the current day (Computed from Matomo)
186        Integer nbComputedVisitsToday = _getComputedVisitsOfCurrentDay(siteId, pageUrl);
187        
188        // Number of visits from the begin years but with the computed current day
189        Integer nbComputedVisitsFromLastYears = _getComputedVisitsFromBeginYear(siteId, pageUrl, beginYear);
190        
191        return nbComputedVisitsFromLastYears - nbComputedVisitsToday + liveVisitsOfDay;
192    }
193    
194    /**
195     * Get the number of visits from the last days for the given page and site 
196     * @param site the site 
197     * @param pageUrl the page URL
198     * @return the number of visits
199     * @throws MatomoException if an error with Matomo occurred
200     */
201    public int getNbVisitsFromLastDays(Site site, String pageUrl) throws MatomoException
202    {
203        _checkMatomoConfig(site);
204        
205        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
206        long lastDays = getMatomoSiteNbLastDays(site);
207        
208        // Number of visits of the current day (Live from Matomo)
209        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl);
210
211        // Visits from the last n days (starting from yesterday)
212        Integer computedVisitsFromLastDays = _getComputedVisitsFromLastDays(siteId, pageUrl, lastDays);
213        
214        return computedVisitsFromLastDays + liveVisitsOfDay;
215    }
216    
217    /**
218     * Get the live number of visits of today for the given page and site
219     * @param siteId the site identifier in Matomo
220     * @param pageUrl the page URL
221     * @return the number of visits
222     * @throws MatomoException if an error with Matomo occurred
223     */
224    protected int _getLiveVisitsOfDay(String siteId, String pageUrl) throws MatomoException
225    {
226        StringBuilder url = new StringBuilder(_matomoServerURL);
227        url.append("/?module=API&method=Live.getLastVisitsDetails");
228        url.append("&format=JSON");
229        url.append("&period=day");
230        url.append("&date=today");
231        url.append("&idSite=");
232        url.append(siteId);
233        url.append("&token_auth=");
234        url.append(_matomoToken);
235        
236        try
237        {
238            List<Map<String, Object>> liveVisits = _getLiveVisitsCache().get(siteId, LambdaUtils.wrap(k -> _requestLiveVisitsOfDay(url.toString())));
239            return _getTodayNbVisits(liveVisits, pageUrl);
240        }
241        catch (LambdaException e) 
242        {
243            if (e.getCause() instanceof RuntimeException re) 
244            {
245                throw re;
246            }
247            else if (e.getCause() instanceof MatomoException m) 
248            {
249                throw m;
250            }
251            else 
252            {
253                throw new RuntimeException(e.getCause());
254            }
255        }
256    }
257    
258    @SuppressWarnings("unchecked")
259    private List<Map<String, Object>> _requestLiveVisitsOfDay(String url) throws MatomoException
260    {
261        return (List<Map<String, Object>>) _executeMatomoRequest(url, "list");
262    }
263    
264    /**
265     * Get the computed number of visits from the last years starting from the begin year for the given page and site
266     * @param siteId the site identifier in matomo
267     * @param pageUrl the page URL
268     * @param beginYear the begin year
269     * @return the number of visits
270     * @throws MatomoException if an error with Matomo occurred
271     */
272    protected int _getComputedVisitsFromBeginYear(String siteId, String pageUrl, String beginYear) throws MatomoException
273    {
274        String truncateURL = _truncateURL(pageUrl);
275
276        LocalDate beginDate = LocalDate.of(Integer.valueOf(beginYear), 1, 1);
277        LocalDate now = LocalDate.now();
278        
279        int nbYear = now.getYear() - beginDate.getYear() + 1;
280        
281        StringBuilder url = new StringBuilder(_matomoServerURL);
282        url.append("/?module=API&method=Actions.getPageUrl");
283        url.append("&format=JSON");
284        url.append("&period=year");
285        url.append("&date=last");
286        url.append(nbYear);
287        url.append("&idSite=");
288        url.append(siteId);
289        url.append("&pageUrl=");
290        url.append(URIUtils.encodeParameter(truncateURL));
291        url.append("&token_auth=");
292        url.append(_matomoToken);
293        
294        try
295        {
296            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.BEGIN_YEAR, truncateURL);
297            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsFromBeginYear(url.toString())));
298        }
299        catch (LambdaException e)
300        {
301            if (e.getCause() instanceof RuntimeException re) 
302            {
303                throw re;
304            }
305            else if (e.getCause() instanceof MatomoException m) 
306            {
307                throw m;
308            }
309            else 
310            {
311                throw new RuntimeException(e.getCause());
312            }
313        }
314    }
315    
316    private Integer _requestComputedVisitsFromBeginYear(String url) throws MatomoException
317    {
318        @SuppressWarnings("unchecked")
319        Map<String, Object> json = (Map<String, Object>) _executeMatomoRequest(url, "map");
320        return _getNbVisitsFromMap(json);
321    }
322    
323    /**
324     * Get the computed number of visits of today for the given page and site
325     * @param siteId the site identifier in matomo
326     * @param pageUrl the page URL
327     * @return the number of visits
328     * @throws MatomoException if an error with Matomo occurred
329     */
330    protected int _getComputedVisitsOfCurrentDay(String siteId, String pageUrl) throws MatomoException
331    {
332        String truncateURL = _truncateURL(pageUrl);
333
334        StringBuilder url = new StringBuilder(_matomoServerURL);
335        url.append("/?module=API&method=Actions.getPageUrl");
336        url.append("&format=JSON");
337        url.append("&period=day");
338        url.append("&date=today");
339        url.append("&idSite=");
340        url.append(siteId);
341        url.append("&pageUrl=");
342        url.append(URIUtils.encodeParameter(truncateURL));
343        url.append("&token_auth=");
344        url.append(_matomoToken);
345        
346        try
347        {
348            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.CURRENT_DAY, truncateURL);
349            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsOfCurrentDay(url.toString())));
350        }
351        catch (LambdaException e) 
352        {
353            if (e.getCause() instanceof RuntimeException re) 
354            {
355                throw re;
356            }
357            else if (e.getCause() instanceof MatomoException m) 
358            {
359                throw m;
360            }
361            else 
362            {
363                throw new RuntimeException(e.getCause());
364            }
365        }
366    }
367    
368    private Integer _requestComputedVisitsOfCurrentDay(String url) throws MatomoException
369    {
370        @SuppressWarnings("unchecked")
371        List<Map<String, Object>> json = (List<Map<String, Object>>) _executeMatomoRequest(url, "list");
372        return _getNbVisitsFromList(json);
373    }
374    
375    /**
376     * Get the computed number of visits from the last n days (starting from yesterday) for the given page and site
377     * @param siteId the site identifier in matomo
378     * @param pageUrl the page URL
379     * @param nbDay the number of day
380     * @return the number of visits
381     * @throws MatomoException if an error with Matomo occurred
382     */
383    protected int _getComputedVisitsFromLastDays(String siteId, String pageUrl, long nbDay) throws MatomoException
384    {
385        String truncateURL = _truncateURL(pageUrl);
386
387        StringBuilder url = new StringBuilder(_matomoServerURL);
388        url.append("/?module=API&method=Actions.getPageUrl");
389        url.append("&format=JSON");
390        url.append("&period=day");
391        url.append("&date=previous");
392        url.append(nbDay);
393        url.append("&idSite=");
394        url.append(siteId);
395        url.append("&pageUrl=");
396        url.append(URIUtils.encodeParameter(truncateURL));
397        url.append("&token_auth=");
398        url.append(_matomoToken);
399        
400        try
401        {
402            MatomoComputedCacheKey key = MatomoComputedCacheKey.of(RequestType.LAST_DAYS, truncateURL);
403            return _getComputedVisitsCache().get(key, LambdaUtils.wrap(k -> _requestComputedVisitsFromLastDays(url.toString())));
404        }
405        catch (LambdaException e) 
406        {
407            if (e.getCause() instanceof RuntimeException re) 
408            {
409                throw re;
410            }
411            else if (e.getCause() instanceof MatomoException m) 
412            {
413                throw m;
414            }
415            else 
416            {
417                throw new RuntimeException(e.getCause());
418            }
419        }
420    }
421    
422    private Integer _requestComputedVisitsFromLastDays(String url) throws MatomoException
423    {
424        @SuppressWarnings("unchecked")
425        Map<String, Object> json = (Map<String, Object>) _executeMatomoRequest(url, "map");
426        return _getNbVisitsFromMap(json);
427    }
428    
429    private Object _executeMatomoRequest(String matomoUrl, String returnType) throws MatomoException
430    {
431        try
432        {
433            if (getLogger().isInfoEnabled())
434            {
435                getLogger().info("A request is send to Matomo with url '" + matomoUrl + "'");
436            }
437            
438            HttpPost post = new HttpPost(matomoUrl);
439            return _httpClient.execute(post, httpResponse -> {
440                if (httpResponse.getStatusLine().getStatusCode() != 200)
441                {
442                    throw new IllegalStateException("Could not join matomo for URL " + matomoUrl + ". Error code " + httpResponse.getStatusLine().getStatusCode());
443                }
444                
445                if ("map".equals(returnType))
446                {
447                    return _jsonUtils.convertJsonToMap(EntityUtils.toString(httpResponse.getEntity()));
448                }
449                else if ("list".equals(returnType))
450                {
451                    return _jsonUtils.convertJsonToList(EntityUtils.toString(httpResponse.getEntity()));
452                }
453                
454                throw new IllegalArgumentException("Invalid return type for request with url '" + matomoUrl + "'");
455            });
456        }
457        catch (Exception e)
458        {
459            throw new MatomoException("An error occurred requesting Matomo for url '" + matomoUrl + "'", e);
460        }
461    }
462    
463    @SuppressWarnings("unchecked")
464    private int _getNbVisitsFromMap(Map<String, Object> jsonAsMap)
465    {
466        int nbVisits = 0;
467        for (String key : jsonAsMap.keySet())
468        {
469            nbVisits += _getNbVisitsFromList((List<Map<String, Object>>) jsonAsMap.get(key));
470        }
471        
472        return nbVisits;
473    }
474    
475    private int _getNbVisitsFromList(List<Map<String, Object>> data)
476    {
477        if (!data.isEmpty())
478        {
479            return (Integer) data.get(0).getOrDefault("nb_visits", 0);
480        }
481        
482        return 0;
483    }
484    
485    private int _getTodayNbVisits(List<Map<String, Object>> jsonAsMap, String pageUrl)
486    {
487        int nbVisits = 0;
488        if (!jsonAsMap.isEmpty())
489        {
490            String truncateURL = _truncateURL(pageUrl);
491            
492            LocalDate today = LocalDate.now();
493            for (Map<String, Object> visitInfo : jsonAsMap)
494            {
495                String visitDateAsString = (String) visitInfo.get("serverDate");
496                LocalDate visitDate = LocalDate.parse(visitDateAsString);
497                if (visitDate.equals(today))
498                {
499                    @SuppressWarnings("unchecked")
500                    List<Map<String, Object>> actions = (List<Map<String, Object>>) visitInfo.get("actionDetails");
501                    Optional<String> pageVisits = actions.stream()
502                            .map(a -> (String) a.get("url"))
503                            .map(this::_truncateURL)
504                            .filter(url -> truncateURL.equals(url))
505                            .findAny();
506                    
507                    if (pageVisits.isPresent())
508                    {
509                        nbVisits++;
510                    }
511                }
512            }
513        }
514        return nbVisits;
515    }
516    
517    /**
518     * Truncate parameters and fragments of the URL
519     * @param pageUrl the page URL
520     * @return the page URL without parameters and fragments
521     */
522    protected String _truncateURL(String pageUrl)
523    {
524        return StringUtils.substringBefore(StringUtils.substringBefore(pageUrl, "?"), "#");
525    }
526    
527    private Cache<MatomoComputedCacheKey, Integer> _getComputedVisitsCache()
528    {
529        return _cacheManager.get(__CACHE_COMPUTED_VISITS);
530    }
531    
532    private Cache<String, List<Map<String, Object>>> _getLiveVisitsCache()
533    {
534        return _cacheManager.get(__CACHE_LIVE_VISITS);
535    }
536    
537    private static final class MatomoComputedCacheKey extends AbstractCacheKey
538    {
539        public MatomoComputedCacheKey(RequestType type, String pageUrl)
540        {
541            super(type, pageUrl);
542        }
543        
544        public static MatomoComputedCacheKey of(RequestType type, String pageUrl)
545        {
546            return new MatomoComputedCacheKey(type, pageUrl);
547        }
548    }
549    
550    public void dispose()
551    {
552        try
553        {
554            _httpClient.close();
555        }
556        catch (IOException e)
557        {
558            getLogger().error("An error occurred closing the http client", e);
559        }
560    }
561}