Source code

001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.webanalytics.matomo;
017
018import java.io.InputStream;
019import java.net.HttpURLConnection;
020import java.net.URL;
021import java.time.LocalDate;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.concurrent.ExecutionException;
027import java.util.concurrent.TimeUnit;
028
029import org.apache.avalon.framework.activity.Initializable;
030import org.apache.avalon.framework.component.Component;
031import org.apache.commons.lang.StringUtils;
032
033import org.ametys.plugins.core.impl.cache.AbstractCacheKey;
034import org.ametys.runtime.config.Config;
035import org.ametys.runtime.plugin.component.AbstractLogEnabled;
036import org.ametys.web.repository.site.Site;
037
038import com.fasterxml.jackson.core.JsonFactory;
039import com.fasterxml.jackson.core.JsonParser;
040import com.fasterxml.jackson.databind.ObjectMapper;
041import com.google.common.cache.CacheBuilder;
042import com.google.common.cache.CacheLoader;
043import com.google.common.cache.LoadingCache;
044
045/**
046 * The matomo data helper
047 */
048public class MatomoDataHelper extends AbstractLogEnabled implements Component, Initializable
049{
050    /** The component role. */
051    public static final String ROLE = MatomoDataHelper.class.getName();
052    
053    /** The Matomo URL config parameter name */
054    public static final String MATOMO_URL_CONFIG = "piwik-url";
055    
056    /** The Matomo site id config parameter name */
057    public static final String MATOMO_SITE_ID_SITE_CONFIG = "piwik-id";
058
059    private static final String __MATOMO_TOKEN_CONFIG = "matomo-token";
060    private static final String __MATOMO_SITE_LIVE_STATS_CONFIG = "matomo-live-stats";
061    private static final String __MATOMO_LAST_DAYS_SITE_CONFIG = "matomo-last-days";
062    private static final String __MATOMO_BEGIN_YEAR_SITE_CONFIG = "matomo-begin-year";
063    private static final String __MATOMO_CACHE_LIFETIME_SITE_CONFIG = "matomo-cache-lifetime";
064    
065    private static final long __CACHE_LIFETIME_DEFAULT_VALUE = 5L;
066    
067    /** The user information cache. */
068    protected LoadingCache<MatomoCacheKey, NbVisits> _dataCache;
069    
070    private String _matomoServerURL; 
071    private String _matomoToken; 
072    
073    @Override
074    public void initialize() throws Exception
075    {
076        NbVisitsCacheLoader loader = new NbVisitsCacheLoader();
077        
078        CacheBuilder<Object, Object> computedDataCacheBuilder = CacheBuilder.newBuilder()
079                .expireAfterWrite(6, TimeUnit.HOURS);
080        _dataCache = computedDataCacheBuilder.build(loader);
081        
082        _matomoServerURL = Config.getInstance().getValue(MATOMO_URL_CONFIG);
083        _matomoToken = Config.getInstance().getValue(__MATOMO_TOKEN_CONFIG);
084    }
085    
086    private void _checkMatomoConfig(Site site) throws MatomoException
087    {
088        if (!site.getValue(__MATOMO_SITE_LIVE_STATS_CONFIG, true, false))
089        {
090            throw new MatomoException("Can't request the matomo stats because the site does not enable it.");
091        }
092        
093        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
094        if (StringUtils.isBlank(_matomoServerURL) || StringUtils.isBlank(_matomoToken) || StringUtils.isBlank(siteId))
095        {
096            throw new MatomoException("One or more of these following parameters are empty: the matamo server URL, the matomo token or the matomo site id. Can't access to matomo data without these parameters");
097        }
098    }
099    
100    /**
101     * Get the number of days from now to computed the number of visits
102     * @param site the site
103     * @return the number of days
104     */
105    public long getMatomoSiteNbLastDays(Site site)
106    {
107        return site.getValue(__MATOMO_LAST_DAYS_SITE_CONFIG, true, 30L);
108    }
109    
110    /**
111     * Get the number of visits since the begin year for the given page and site
112     * @param site the site 
113     * @param pageUrl the page URL
114     * @return the number of visits
115     * @throws MatomoException if an error with Matomo occurred
116     */
117    public int getNbTotalVisits(Site site, String pageUrl) throws MatomoException
118    {
119        _checkMatomoConfig(site);
120
121        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
122        long cacheLifeTime = site.getValue(__MATOMO_CACHE_LIFETIME_SITE_CONFIG, true, __CACHE_LIFETIME_DEFAULT_VALUE);
123        String beginYear = site.getValue(__MATOMO_BEGIN_YEAR_SITE_CONFIG);
124        
125        // Number of visits of the current day (Live from Matomo)
126        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl, cacheLifeTime);
127        
128        // Number of visits of the current day (Computed from Matomo)
129        Integer nbComputedVisitsToday = _getComputedVisitsOfCurrentDay(siteId, pageUrl);
130        
131        // Number of visits from the begin years but with the computed current day
132        Integer nbComputedVisitsFromLastYears = _getComputedVisitsFromBeginYear(siteId, pageUrl, beginYear);
133        
134        return nbComputedVisitsFromLastYears - nbComputedVisitsToday + liveVisitsOfDay;
135    }
136    
137    /**
138     * Get the number of visits from the last days for the given page and site 
139     * @param site the site 
140     * @param pageUrl the page URL
141     * @return the number of visits
142     * @throws MatomoException if an error with Matomo occurred
143     */
144    public int getNbVisitsFromLastDays(Site site, String pageUrl) throws MatomoException
145    {
146        _checkMatomoConfig(site);
147        
148        String siteId = site.getValue(MATOMO_SITE_ID_SITE_CONFIG);
149        long cacheLifeTime = site.getValue(__MATOMO_CACHE_LIFETIME_SITE_CONFIG, true, __CACHE_LIFETIME_DEFAULT_VALUE);
150        long lastDays = getMatomoSiteNbLastDays(site);
151        
152        // Number of visits of the current day (Live from Matomo)
153        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl, cacheLifeTime);
154
155        // Visits from the last n days (starting from yesterday)
156        Integer computedVisitsFromLastDays = _getComputedVisitsFromLastDays(siteId, pageUrl, lastDays);
157        
158        return computedVisitsFromLastDays + liveVisitsOfDay;
159    }
160    
161    /**
162     * Get the live number of visits of today for the given page and site
163     * @param siteId the site identifier in Matomo
164     * @param pageUrl the page URL
165     * @param cacheValidityInMinutes the number of minute of the cache validity
166     * @return the number of visits
167     * @throws MatomoException if an error with Matomo occurred
168     */
169    protected int _getLiveVisitsOfDay(String siteId, String pageUrl, Long cacheValidityInMinutes) throws MatomoException
170    {
171        StringBuilder url = new StringBuilder(_matomoServerURL);
172        url.append("/?module=API&method=Live.getLastVisitsDetails");
173        url.append("&format=JSON");
174        url.append("&period=day");
175        url.append("&date=today");
176        url.append("&idSite=");
177        url.append(siteId);
178        url.append("&token_auth=");
179        url.append(_matomoToken);
180        
181        MatomoCacheKey key = MatomoCacheKey.of(url.toString(), pageUrl);
182        try
183        {
184            NbVisits nbVisits = _dataCache.get(key);
185            // Clear the cache after nbMinutes
186            if (nbVisits.date().plusMinutes(cacheValidityInMinutes).isBefore(ZonedDateTime.now()))
187            {
188                _dataCache.invalidate(key);
189                nbVisits = _dataCache.get(key);
190            }
191            
192            return nbVisits.number();
193        }
194        catch (ExecutionException e)
195        {
196            throw new MatomoException("An error occurred getting the number of visits of today for page url '" + pageUrl + "'", e);
197        }
198    }
199    
200    /**
201     * Get the computed number of visits from the last years starting from the begin year for the given page and site
202     * @param siteId the site identifier in matomo
203     * @param pageUrl the page URL
204     * @param beginYear the begin year
205     * @return the number of visits
206     * @throws MatomoException if an error with Matomo occurred
207     */
208    protected int _getComputedVisitsFromBeginYear(String siteId, String pageUrl, String beginYear) throws MatomoException
209    {
210        LocalDate beginDate = LocalDate.of(Integer.valueOf(beginYear), 1, 1);
211        LocalDate now = LocalDate.now();
212        
213        int nbYear = now.getYear() - beginDate.getYear() + 1;
214        
215        StringBuilder url = new StringBuilder(_matomoServerURL);
216        url.append("/?module=API&method=Actions.getPageUrl");
217        url.append("&format=JSON");
218        url.append("&period=year");
219        url.append("&date=last");
220        url.append(nbYear);
221        url.append("&idSite=");
222        url.append(siteId);
223        url.append("&pageUrl=");
224        url.append(pageUrl);
225        url.append("&token_auth=");
226        url.append(_matomoToken);
227        
228        return _getComputedNbVisits(url, pageUrl);
229    }
230    
231    /**
232     * Get the computed number of visits of today for the given page and site
233     * @param siteId the site identifier in matomo
234     * @param pageUrl the page URL
235     * @return the number of visits
236     * @throws MatomoException if an error with Matomo occurred
237     */
238    protected int _getComputedVisitsOfCurrentDay(String siteId, String pageUrl) throws MatomoException
239    {
240        StringBuilder url = new StringBuilder(_matomoServerURL);
241        url.append("/?module=API&method=Actions.getPageUrl");
242        url.append("&format=JSON");
243        url.append("&period=day");
244        url.append("&date=today");
245        url.append("&idSite=");
246        url.append(siteId);
247        url.append("&pageUrl=");
248        url.append(pageUrl);
249        url.append("&token_auth=");
250        url.append(_matomoToken);
251        
252        return _getComputedNbVisits(url, pageUrl);
253    }
254    
255    /**
256     * Get the computed number of visits from the last n days (starting from yesterday) for the given page and site
257     * @param siteId the site identifier in matomo
258     * @param pageUrl the page URL
259     * @param nbDay the number of day
260     * @return the number of visits
261     * @throws MatomoException if an error with Matomo occurred
262     */
263    protected int _getComputedVisitsFromLastDays(String siteId, String pageUrl, long nbDay) throws MatomoException
264    {
265        StringBuilder url = new StringBuilder(_matomoServerURL);
266        url.append("/?module=API&method=Actions.getPageUrl");
267        url.append("&format=JSON");
268        url.append("&period=day");
269        url.append("&date=previous");
270        url.append(nbDay);
271        url.append("&idSite=");
272        url.append(siteId);
273        url.append("&pageUrl=");
274        url.append(pageUrl);
275        url.append("&token_auth=");
276        url.append(_matomoToken);
277        
278        return _getComputedNbVisits(url, pageUrl);
279    }
280    
281    private int _getComputedNbVisits(StringBuilder url, String pageUrl) throws MatomoException
282    {
283        MatomoCacheKey key = MatomoCacheKey.of(url.toString(), pageUrl);
284        try
285        {
286            NbVisits nbVisits = _dataCache.get(key);
287            return nbVisits.number();
288        }
289        catch (ExecutionException e)
290        {
291            throw new MatomoException("An error occurred getting the number of visits for page url '" + pageUrl + "'", e);
292        }
293    }
294    
295    private NbVisits _getNbVisitsFromMatomo(MatomoCacheKey key) throws MatomoException
296    {
297        String matomoUrl = (String) key.getFields().get(0);
298        String pageUrl = (String) key.getFields().get(1);
299        try
300        {
301            int nbVisits = 0;
302            
303            URL url = new URL(matomoUrl);
304            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
305            connection.setConnectTimeout(1000);
306            connection.setReadTimeout(5000);
307            try (InputStream is = connection.getInputStream())
308            {
309                JsonParser parser = new JsonFactory().createParser(is);
310                if (StringUtils.contains(matomoUrl, "Live.getLastVisitsDetails"))
311                {
312                    nbVisits = _getTodayNbVisits(new ObjectMapper().readValue(parser, List.class), pageUrl);
313                }
314                else if (StringUtils.contains(matomoUrl, "date=today")) 
315                {
316                    nbVisits = _getNbVisitsFromList(new ObjectMapper().readValue(parser, List.class));
317                }
318                else
319                {
320                    nbVisits = _getNbVisitsFromMap(new ObjectMapper().readValue(parser, HashMap.class));
321                }
322            }
323            
324            return new NbVisits(nbVisits, ZonedDateTime.now());
325        }
326        catch (Exception e)
327        {
328            throw new MatomoException("Can't parse matomo JSON result for url '" + matomoUrl + "'", e);
329        }
330    }
331    
332    @SuppressWarnings("unchecked")
333    private int _getNbVisitsFromMap(Map<String, Object> jsonAsMap)
334    {
335        int nbVisits = 0;
336        for (String key : jsonAsMap.keySet())
337        {
338            nbVisits += _getNbVisitsFromList((List<Map<String, Object>>) jsonAsMap.get(key));
339        }
340        
341        return nbVisits;
342    }
343    
344    private int _getNbVisitsFromList(List<Map<String, Object>> data)
345    {
346        if (!data.isEmpty())
347        {
348            return (Integer) data.get(0).getOrDefault("nb_visits", 0);
349        }
350        
351        return 0;
352    }
353    
354    private int _getTodayNbVisits(List<Map<String, Object>> jsonAsMap, String pageUrl)
355    {
356        int nbVisits = 0;
357        if (!jsonAsMap.isEmpty())
358        {
359            LocalDate today = LocalDate.now();
360            for (Map<String, Object> visitInfo : jsonAsMap)
361            {
362                @SuppressWarnings("unchecked")
363                List<Map<String, Object>> actions = (List<Map<String, Object>>) visitInfo.get("actionDetails");
364                boolean present = actions.stream()
365                    .filter(a -> pageUrl.equals(a.get("url")))
366                    .findAny()
367                    .isPresent();
368                
369                if (present)
370                {
371                    String visitDateAsString = (String) visitInfo.get("serverDate");
372                    LocalDate visitDate = LocalDate.parse(visitDateAsString);
373                    if (visitDate.equals(today))
374                    {
375                        nbVisits++;
376                    }
377                }
378            }
379        }
380        return nbVisits;
381    }
382    
383    /**
384     * The cache loader for the number of visits
385     */
386    protected class NbVisitsCacheLoader extends CacheLoader<MatomoCacheKey, NbVisits>
387    {
388        @Override
389        public NbVisits load(MatomoCacheKey url) throws Exception
390        {
391            return _getNbVisitsFromMatomo(url);
392        }
393    }
394    
395    private static final class MatomoCacheKey extends AbstractCacheKey
396    {
397        public MatomoCacheKey(String requestUrl, String pageUrl)
398        {
399            super(requestUrl, pageUrl);
400        }
401        
402        public static MatomoCacheKey of(String requestUrl, String pageUrl)
403        {
404            return new MatomoCacheKey(requestUrl, pageUrl);
405        }
406    }
407    
408    /**
409     * Record representing the number of visits at the given local date
410     * @param number the number of visits
411     * @param date the date when the the number of visits has been computed
412     */
413    public record NbVisits(int number, ZonedDateTime date) { /* */ }
414}