001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.webanalytics.matomo;
017
018import java.io.InputStream;
019import java.net.HttpURLConnection;
020import java.net.URL;
021import java.time.LocalDate;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.concurrent.ExecutionException;
027import java.util.concurrent.TimeUnit;
028
029import org.apache.avalon.framework.activity.Initializable;
030import org.apache.avalon.framework.component.Component;
031import org.apache.commons.lang.StringUtils;
032
033import org.ametys.plugins.core.impl.cache.AbstractCacheKey;
034import org.ametys.runtime.config.Config;
035import org.ametys.runtime.plugin.component.AbstractLogEnabled;
036import org.ametys.web.repository.site.Site;
037
038import com.fasterxml.jackson.core.JsonFactory;
039import com.fasterxml.jackson.core.JsonParser;
040import com.fasterxml.jackson.databind.ObjectMapper;
041import com.google.common.cache.CacheBuilder;
042import com.google.common.cache.CacheLoader;
043import com.google.common.cache.LoadingCache;
044
045/**
046 * The matomo data helper
047 */
048public class MatomoDataHelper extends AbstractLogEnabled implements Component, Initializable
049{
050    private static final String __MATOMO_URL_CONFIG = "piwik-url";
051    private static final String __MATOMO_TOKEN_CONFIG = "matomo-token";
052    private static final String __MATOMO_SITE_ID_SITE_CONFIG = "piwik-id";
053    private static final String __MATOMO_SITE_LIVE_STATS_CONFIG = "matomo-live-stats";
054    private static final String __MATOMO_LAST_DAYS_SITE_CONFIG = "matomo-last-days";
055    private static final String __MATOMO_BEGIN_YEAR_SITE_CONFIG = "matomo-begin-year";
056    private static final String __MATOMO_CACHE_LIFETIME_SITE_CONFIG = "matomo-cache-lifetime";
057    
058    private static final long __CACHE_LIFETIME_DEFAULT_VALUE = 5L;
059    
060    /** The component role. */
061    public static final String ROLE = MatomoDataHelper.class.getName();
062    
063    /** The user information cache. */
064    protected LoadingCache<MatomoCacheKey, NbVisits> _dataCache;
065    
066    private String _matomoServerURL; 
067    private String _matomoToken; 
068    
069    @Override
070    public void initialize() throws Exception
071    {
072        NbVisitsCacheLoader loader = new NbVisitsCacheLoader();
073        
074        CacheBuilder<Object, Object> computedDataCacheBuilder = CacheBuilder.newBuilder()
075                .expireAfterWrite(6, TimeUnit.HOURS);
076        _dataCache = computedDataCacheBuilder.build(loader);
077        
078        _matomoServerURL = Config.getInstance().getValue(__MATOMO_URL_CONFIG);
079        _matomoToken = Config.getInstance().getValue(__MATOMO_TOKEN_CONFIG);
080    }
081    
082    private void _checkMatomoConfig(Site site) throws MatomoException
083    {
084        if (!site.getValue(__MATOMO_SITE_LIVE_STATS_CONFIG, true, false))
085        {
086            throw new MatomoException("Can't request the matomo stats because the site does not enable it.");
087        }
088        
089        String siteId = site.getValue(__MATOMO_SITE_ID_SITE_CONFIG);
090        if (StringUtils.isBlank(_matomoServerURL) || StringUtils.isBlank(_matomoToken) || StringUtils.isBlank(siteId))
091        {
092            throw new MatomoException("One or more of these following parameters are empty: the matamo server URL, the matomo token or the matomo site id. Can't access to matomo data without these parameters");
093        }
094    }
095    
096    /**
097     * Get the number of days from now to computed the number of visits
098     * @param site the site
099     * @return the number of days
100     */
101    public long getMatomoSiteNbLastDays(Site site)
102    {
103        return site.getValue(__MATOMO_LAST_DAYS_SITE_CONFIG, true, 30L);
104    }
105    
106    /**
107     * Get the number of visits since the begin year for the given page and site
108     * @param site the site 
109     * @param pageUrl the page URL
110     * @return the number of visits
111     * @throws MatomoException if an error with Matomo occurred
112     */
113    public int getNbTotalVisits(Site site, String pageUrl) throws MatomoException
114    {
115        _checkMatomoConfig(site);
116
117        String siteId = site.getValue(__MATOMO_SITE_ID_SITE_CONFIG);
118        long cacheLifeTime = site.getValue(__MATOMO_CACHE_LIFETIME_SITE_CONFIG, true, __CACHE_LIFETIME_DEFAULT_VALUE);
119        String beginYear = site.getValue(__MATOMO_BEGIN_YEAR_SITE_CONFIG);
120        
121        // Number of visits of the current day (Live from Matomo)
122        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl, cacheLifeTime);
123        
124        // Number of visits of the current day (Computed from Matomo)
125        Integer nbComputedVisitsToday = _getComputedVisitsOfCurrentDay(siteId, pageUrl);
126        
127        // Number of visits from the begin years but with the computed current day
128        Integer nbComputedVisitsFromLastYears = _getComputedVisitsFromBeginYear(siteId, pageUrl, beginYear);
129        
130        return nbComputedVisitsFromLastYears - nbComputedVisitsToday + liveVisitsOfDay;
131    }
132    
133    /**
134     * Get the number of visits from the last days for the given page and site 
135     * @param site the site 
136     * @param pageUrl the page URL
137     * @return the number of visits
138     * @throws MatomoException if an error with Matomo occurred
139     */
140    public int getNbVisitsFromLastDays(Site site, String pageUrl) throws MatomoException
141    {
142        _checkMatomoConfig(site);
143        
144        String siteId = site.getValue(__MATOMO_SITE_ID_SITE_CONFIG);
145        long cacheLifeTime = site.getValue(__MATOMO_CACHE_LIFETIME_SITE_CONFIG, true, __CACHE_LIFETIME_DEFAULT_VALUE);
146        long lastDays = getMatomoSiteNbLastDays(site);
147        
148        // Number of visits of the current day (Live from Matomo)
149        Integer liveVisitsOfDay = _getLiveVisitsOfDay(siteId, pageUrl, cacheLifeTime);
150
151        // Visits from the last n days (starting from yesterday)
152        Integer computedVisitsFromLastDays = _getComputedVisitsFromLastDays(siteId, pageUrl, lastDays);
153        
154        return computedVisitsFromLastDays + liveVisitsOfDay;
155    }
156    
157    /**
158     * Get the live number of visits of today for the given page and site
159     * @param siteId the site identifier in Matomo
160     * @param pageUrl the page URL
161     * @param cacheValidityInMinutes the number of minute of the cache validity
162     * @return the number of visits
163     * @throws MatomoException if an error with Matomo occurred
164     */
165    protected int _getLiveVisitsOfDay(String siteId, String pageUrl, Long cacheValidityInMinutes) throws MatomoException
166    {
167        StringBuilder url = new StringBuilder(_matomoServerURL);
168        url.append("/?module=API&method=Live.getLastVisitsDetails");
169        url.append("&format=JSON");
170        url.append("&period=day");
171        url.append("&date=today");
172        url.append("&idSite=");
173        url.append(siteId);
174        url.append("&token_auth=");
175        url.append(_matomoToken);
176        
177        MatomoCacheKey key = MatomoCacheKey.of(url.toString(), pageUrl);
178        try
179        {
180            NbVisits nbVisits = _dataCache.get(key);
181            // Clear the cache after nbMinutes
182            if (nbVisits.date().plusMinutes(cacheValidityInMinutes).isBefore(ZonedDateTime.now()))
183            {
184                _dataCache.invalidate(key);
185                nbVisits = _dataCache.get(key);
186            }
187            
188            return nbVisits.number();
189        }
190        catch (ExecutionException e)
191        {
192            throw new MatomoException("An error occurred getting the number of visits of today for page url '" + pageUrl + "'", e);
193        }
194    }
195    
196    /**
197     * Get the computed number of visits from the last years starting from the begin year for the given page and site
198     * @param siteId the site identifier in matomo
199     * @param pageUrl the page URL
200     * @param beginYear the begin year
201     * @return the number of visits
202     * @throws MatomoException if an error with Matomo occurred
203     */
204    protected int _getComputedVisitsFromBeginYear(String siteId, String pageUrl, String beginYear) throws MatomoException
205    {
206        LocalDate beginDate = LocalDate.of(Integer.valueOf(beginYear), 1, 1);
207        LocalDate now = LocalDate.now();
208        
209        int nbYear = now.getYear() - beginDate.getYear() + 1;
210        
211        StringBuilder url = new StringBuilder(_matomoServerURL);
212        url.append("/?module=API&method=Actions.getPageUrl");
213        url.append("&format=JSON");
214        url.append("&period=year");
215        url.append("&date=last");
216        url.append(nbYear);
217        url.append("&idSite=");
218        url.append(siteId);
219        url.append("&pageUrl=");
220        url.append(pageUrl);
221        url.append("&token_auth=");
222        url.append(_matomoToken);
223        
224        return _getComputedNbVisits(url, pageUrl);
225    }
226    
227    /**
228     * Get the computed number of visits of today for the given page and site
229     * @param siteId the site identifier in matomo
230     * @param pageUrl the page URL
231     * @return the number of visits
232     * @throws MatomoException if an error with Matomo occurred
233     */
234    protected int _getComputedVisitsOfCurrentDay(String siteId, String pageUrl) throws MatomoException
235    {
236        StringBuilder url = new StringBuilder(_matomoServerURL);
237        url.append("/?module=API&method=Actions.getPageUrl");
238        url.append("&format=JSON");
239        url.append("&period=day");
240        url.append("&date=today");
241        url.append("&idSite=");
242        url.append(siteId);
243        url.append("&pageUrl=");
244        url.append(pageUrl);
245        url.append("&token_auth=");
246        url.append(_matomoToken);
247        
248        return _getComputedNbVisits(url, pageUrl);
249    }
250    
251    /**
252     * Get the computed number of visits from the last n days (starting from yesterday) for the given page and site
253     * @param siteId the site identifier in matomo
254     * @param pageUrl the page URL
255     * @param nbDay the number of day
256     * @return the number of visits
257     * @throws MatomoException if an error with Matomo occurred
258     */
259    protected int _getComputedVisitsFromLastDays(String siteId, String pageUrl, long nbDay) throws MatomoException
260    {
261        StringBuilder url = new StringBuilder(_matomoServerURL);
262        url.append("/?module=API&method=Actions.getPageUrl");
263        url.append("&format=JSON");
264        url.append("&period=day");
265        url.append("&date=previous");
266        url.append(nbDay);
267        url.append("&idSite=");
268        url.append(siteId);
269        url.append("&pageUrl=");
270        url.append(pageUrl);
271        url.append("&token_auth=");
272        url.append(_matomoToken);
273        
274        return _getComputedNbVisits(url, pageUrl);
275    }
276    
277    private int _getComputedNbVisits(StringBuilder url, String pageUrl) throws MatomoException
278    {
279        MatomoCacheKey key = MatomoCacheKey.of(url.toString(), pageUrl);
280        try
281        {
282            NbVisits nbVisits = _dataCache.get(key);
283            return nbVisits.number();
284        }
285        catch (ExecutionException e)
286        {
287            throw new MatomoException("An error occurred getting the number of visits for page url '" + pageUrl + "'", e);
288        }
289    }
290    
291    private NbVisits _getNbVisitsFromMatomo(MatomoCacheKey key) throws MatomoException
292    {
293        String matomoUrl = (String) key.getFields().get(0);
294        String pageUrl = (String) key.getFields().get(1);
295        try
296        {
297            int nbVisits = 0;
298            
299            URL url = new URL(matomoUrl);
300            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
301            connection.setConnectTimeout(1000);
302            connection.setReadTimeout(5000);
303            try (InputStream is = connection.getInputStream())
304            {
305                JsonParser parser = new JsonFactory().createParser(is);
306                if (StringUtils.contains(matomoUrl, "Live.getLastVisitsDetails"))
307                {
308                    nbVisits = _getTodayNbVisits(new ObjectMapper().readValue(parser, List.class), pageUrl);
309                }
310                else if (StringUtils.contains(matomoUrl, "date=today")) 
311                {
312                    nbVisits = _getNbVisitsFromList(new ObjectMapper().readValue(parser, List.class));
313                }
314                else
315                {
316                    nbVisits = _getNbVisitsFromMap(new ObjectMapper().readValue(parser, HashMap.class));
317                }
318            }
319            
320            return new NbVisits(nbVisits, ZonedDateTime.now());
321        }
322        catch (Exception e)
323        {
324            throw new MatomoException("Can't parse matomo JSON result for url '" + matomoUrl + "'", e);
325        }
326    }
327    
328    @SuppressWarnings("unchecked")
329    private int _getNbVisitsFromMap(Map<String, Object> jsonAsMap)
330    {
331        int nbVisits = 0;
332        for (String key : jsonAsMap.keySet())
333        {
334            nbVisits += _getNbVisitsFromList((List<Map<String, Object>>) jsonAsMap.get(key));
335        }
336        
337        return nbVisits;
338    }
339    
340    private int _getNbVisitsFromList(List<Map<String, Object>> data)
341    {
342        if (!data.isEmpty())
343        {
344            return (Integer) data.get(0).getOrDefault("nb_visits", 0);
345        }
346        
347        return 0;
348    }
349    
350    private int _getTodayNbVisits(List<Map<String, Object>> jsonAsMap, String pageUrl)
351    {
352        int nbVisits = 0;
353        if (!jsonAsMap.isEmpty())
354        {
355            LocalDate today = LocalDate.now();
356            for (Map<String, Object> visitInfo : jsonAsMap)
357            {
358                @SuppressWarnings("unchecked")
359                List<Map<String, Object>> actions = (List<Map<String, Object>>) visitInfo.get("actionDetails");
360                boolean present = actions.stream()
361                    .filter(a -> a.get("url").equals(pageUrl))
362                    .findAny()
363                    .isPresent();
364                
365                if (present)
366                {
367                    String visitDateAsString = (String) visitInfo.get("serverDate");
368                    LocalDate visitDate = LocalDate.parse(visitDateAsString);
369                    if (visitDate.equals(today))
370                    {
371                        nbVisits++;
372                    }
373                }
374            }
375        }
376        return nbVisits;
377    }
378    
379    /**
380     * The cache loader for the number of visits
381     */
382    protected class NbVisitsCacheLoader extends CacheLoader<MatomoCacheKey, NbVisits>
383    {
384        @Override
385        public NbVisits load(MatomoCacheKey url) throws Exception
386        {
387            return _getNbVisitsFromMatomo(url);
388        }
389    }
390    
391    private static final class MatomoCacheKey extends AbstractCacheKey
392    {
393        public MatomoCacheKey(String requestUrl, String pageUrl)
394        {
395            super(requestUrl, pageUrl);
396        }
397        
398        public static MatomoCacheKey of(String requestUrl, String pageUrl)
399        {
400            return new MatomoCacheKey(requestUrl, pageUrl);
401        }
402    }
403    
404    /**
405     * Record representing the number of visits at the given local date
406     * @param number the number of visits
407     * @param date the date when the the number of visits has been computed
408     */
409    public record NbVisits(int number, ZonedDateTime date) { /* */ }
410}