001/*
002 *  Copyright 2022 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.forms.helper;
017
018import java.util.ArrayList;
019import java.util.Arrays;
020import java.util.HashMap;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.Map.Entry;
025import java.util.stream.Collectors;
026
027import org.apache.avalon.framework.component.Component;
028import org.apache.avalon.framework.service.ServiceException;
029import org.apache.avalon.framework.service.ServiceManager;
030import org.apache.avalon.framework.service.Serviceable;
031import org.apache.commons.lang3.StringUtils;
032
033import org.ametys.core.right.RightManager;
034import org.ametys.core.ui.Callable;
035import org.ametys.core.util.I18nUtils;
036import org.ametys.plugins.forms.dao.FormEntryDAO;
037import org.ametys.plugins.forms.question.FormQuestionType;
038import org.ametys.plugins.forms.question.sources.AbstractSourceType;
039import org.ametys.plugins.forms.question.sources.ChoiceOption;
040import org.ametys.plugins.forms.question.sources.ChoiceSourceType;
041import org.ametys.plugins.forms.question.types.CheckBoxQuestionType;
042import org.ametys.plugins.forms.question.types.ChoicesListQuestionType;
043import org.ametys.plugins.forms.question.types.MatrixQuestionType;
044import org.ametys.plugins.forms.repository.Form;
045import org.ametys.plugins.forms.repository.FormEntry;
046import org.ametys.plugins.forms.repository.FormQuestion;
047import org.ametys.plugins.forms.repository.type.Matrix;
048import org.ametys.plugins.repository.AmetysObjectResolver;
049import org.ametys.plugins.repository.query.expression.AndExpression;
050import org.ametys.plugins.repository.query.expression.BooleanExpression;
051import org.ametys.plugins.repository.query.expression.Expression;
052import org.ametys.plugins.repository.query.expression.Expression.Operator;
053import org.ametys.plugins.repository.query.expression.MetadataExpression;
054import org.ametys.plugins.repository.query.expression.NotExpression;
055import org.ametys.plugins.repository.query.expression.OrExpression;
056import org.ametys.plugins.repository.query.expression.StringExpression;
057import org.ametys.runtime.i18n.I18nizableText;
058import org.ametys.runtime.plugin.component.AbstractLogEnabled;
059
060/**
061 * The helper to handle admin emails
062 */
063public class FormStatisticsHelper extends AbstractLogEnabled implements Serviceable, Component
064{
065    /** Avalon ROLE. */
066    public static final String ROLE = FormStatisticsHelper.class.getName();
067    
068    /** The Ametys Object resolver */
069    protected AmetysObjectResolver _resolver;
070
071    /** The right manager */
072    protected RightManager _rightManager;
073    
074    /** The form entry DAO */
075    protected FormEntryDAO _formEntryDAO;
076
077    /** The I18n utils */
078    protected I18nUtils _i18nUtils;
079    
080    public void service(ServiceManager manager) throws ServiceException
081    {
082        _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
083        _rightManager = (RightManager) manager.lookup(RightManager.ROLE);
084        _formEntryDAO = (FormEntryDAO) manager.lookup(FormEntryDAO.ROLE);
085        _i18nUtils = (I18nUtils) manager.lookup(I18nUtils.ROLE);
086    }
087    
088    /**
089     * Generates statistics on each question of a form.
090     * @param id The form id
091     * @return A map containing the statistics
092     */
093    @Callable (rights = Callable.SKIP_BUILTIN_CHECK)
094    public Map<String, Object> getStatistics(String id)
095    {
096        Form form = _resolver.resolveById(id);
097        _formEntryDAO.checkHandleDataRight(form);
098        
099        return _getStatistics(form);
100    }
101
102    /**
103     * Generate statistics of a mini-survey
104     * No rights will be checked. Only a check that it is indeed a mini-survey will be done.
105     * @param id the form id
106     * @return a JSON map of the statistics
107     */
108    public Map<String, Object> getMiniSurveyStatistics(String id)
109    {
110        Form form = _resolver.resolveById(id);
111        if (form.isMiniSurvey())
112        {
113            return _getStatistics(form);
114        }
115        else
116        {
117            return Map.of("error", "not-a-mini-survey");
118        }
119    }
120    
121    /**
122     * Compute the statistics for each questions of the form
123     * @param form the form
124     * @return A JSON map representing the statistics
125     */
126    protected Map<String, Object> _getStatistics(Form form)
127    {
128        Map<String, Object> statistics = new HashMap<>();
129        
130        statistics.put("id", form.getId());
131        statistics.put("title", form.getTitle());
132        statistics.put("nbEntries", form.getEntries().size());
133        statistics.put("questions", getStatsToArray(form));
134        
135        return statistics;
136    }
137    
138    /**
139     * Create a map with count of all answers per question
140     * @param form current form
141     * @return the statsMap
142     */
143    public Map<String, Map<String, Map<String, Object>>> getStatsMap(Form form)
144    {
145        Map<String, Map<String, Map<String, Object>>> statsMap = new LinkedHashMap<>();
146        List<FormQuestion> questions = form.getQuestions()
147                .stream()
148                .filter(this::_displayField)
149                .collect(Collectors.toList());
150        
151        List<FormEntry> entries = form.getEntries();
152        
153        for (FormQuestion question : questions)
154        {
155            Map<String, Map<String, Object>> questionValues = new LinkedHashMap<>();
156            statsMap.put(question.getNameForForm(), questionValues);
157            
158            if (question.getType() instanceof MatrixQuestionType)
159            {
160                _dispatchMatrixStats(entries, question, questionValues);
161            }
162            else if (question.getType() instanceof ChoicesListQuestionType type)
163            {
164                if (type.getSourceType(question).remoteData())
165                {
166                    _dispatchChoicesWithRemoteDataStats(form, question, questionValues);
167                }
168                else
169                {
170                    _dispatchChoicesStats(form, question, questionValues);
171                }
172            }
173            else if (question.getType() instanceof CheckBoxQuestionType)
174            {
175                _dispatchBooleanStats(form, entries, question, questionValues);
176            }
177            else
178            {
179                _dispatchStats(form, entries, question, questionValues);
180            }
181        }
182        
183        return statsMap;
184    }
185    
186    /**
187     * Transforms the statistics map into an array with some info.
188     * @param form The form
189     * @return A list of statistics.
190     */
191    public List<Map<String, Object>> getStatsToArray (Form form)
192    {
193        Map<String, Map<String, Map<String, Object>>> stats = getStatsMap(form);
194        
195        List<Map<String, Object>> result = new ArrayList<>();
196        
197        for (String questionNameForForm : stats.keySet())
198        {
199            Map<String, Object> questionMap = new HashMap<>();
200            
201            FormQuestion question = form.getQuestion(questionNameForForm);
202            Map<String, Map<String, Object>> questionStats = stats.get(questionNameForForm);
203            
204            questionMap.put("id", questionNameForForm);
205            questionMap.put("title", question.getTitle());
206            questionMap.put("type", question.getType().getStorageType(question));
207            questionMap.put("typeId", question.getType().getId());
208            questionMap.put("mandatory", question.isMandatory());
209            
210            List<Object> options = new ArrayList<>();
211            for (String optionId : questionStats.keySet())
212            {
213                Map<String, Object> option = new HashMap<>();
214
215                option.put("id", optionId);
216
217                if (question.getType() instanceof MatrixQuestionType)
218                {
219                    MatrixQuestionType type = (MatrixQuestionType) question.getType();
220                    option.put("label", type.getRows(question).get(optionId));
221                }
222                
223                questionStats.get(optionId).entrySet();
224                List<Object> choices = new ArrayList<>();
225                for (Entry<String, Object> choice : questionStats.get(optionId).entrySet())
226                {
227                    Map<String, Object> choiceMap = new HashMap<>();
228                    
229                    String choiceId = choice.getKey();
230                    Object choiceOb = choice.getValue();
231                    choiceMap.put("value", choiceId);
232                    choiceMap.put("label", choiceOb instanceof Option ? ((Option) choiceOb).label() : choiceOb);
233
234                    if (question.getType() instanceof MatrixQuestionType)
235                    {
236                        MatrixQuestionType type = (MatrixQuestionType) question.getType();
237                        choiceMap.put("label", type.getColumns(question).get(choiceId));
238                    }
239                    
240                    
241                    choiceMap.put("count", choiceOb instanceof Option ? ((Option) choiceOb).count() : choiceOb);
242                    
243                    choices.add(choiceMap);
244                }
245                option.put("choices", choices);
246                
247                options.add(option);
248            }
249            questionMap.put("options", options);
250            
251            result.add(questionMap);
252        }
253        
254        return result;
255    }
256
257    /**
258     * Dispatch matrix stats
259     * @param entries the entries
260     * @param question the question
261     * @param questionValues the values to fill
262     */
263    protected void _dispatchMatrixStats(List<FormEntry> entries, FormQuestion question, Map<String, Map<String, Object>> questionValues)
264    {
265        MatrixQuestionType matrixType = (MatrixQuestionType) question.getType();
266        Map<String, String> rows = matrixType.getRows(question);
267        if (rows != null)
268        {
269            for (String option : rows.keySet())
270            {
271                Map<String, Object> values = new LinkedHashMap<>();
272                questionValues.put(option, values);
273                
274                Map<String, String> columns = matrixType.getColumns(question);
275                if (columns != null)
276                {
277                    for (String column : columns.keySet())
278                    {
279                        values.put(column, 0);
280                        _setOptionCount(question.getNameForForm(), entries, values, option, column);
281                    }
282                }
283            }
284        }
285    }
286    
287    private void _setOptionCount(String questionId, List<FormEntry> entries, Map<String, Object> values, String rowValue, String columnValue)
288    {
289        int columnCount = (int) values.get(columnValue);
290        for (FormEntry entry : entries)
291        {
292            Matrix matrix = entry.getValue(questionId);
293            if (matrix != null)
294            {
295                List<String> options = matrix.get(rowValue);
296                if (options != null && options.contains(columnValue))
297                {
298                    columnCount++;
299                }
300            }
301        }
302        values.put(columnValue, columnCount);
303    }
304
305    /**
306     * Dispatch choices list stats
307     * @param form the form
308     * @param question the question
309     * @param questionValues the values to fill
310     */
311    protected void _dispatchChoicesStats(Form form, FormQuestion question, Map<String, Map<String, Object>> questionValues)
312    {
313        Map<String, Object> values = new LinkedHashMap<>();
314        questionValues.put("values", values);
315        
316        ChoicesListQuestionType type = (ChoicesListQuestionType) question.getType();
317        ChoiceSourceType sourceType = type.getSourceType(question);
318        Map<ChoiceOption, I18nizableText> options;
319        try
320        {
321            Map<String, Object> enumParam = new HashMap<>();
322            enumParam.put(AbstractSourceType.QUESTION_PARAM_KEY, question);
323            options = sourceType.getTypedEntries(enumParam);
324            
325            for (ChoiceOption option : options.keySet())
326            {
327                String optionValue = (String) option.getValue();
328                StringExpression expr = new StringExpression(question.getNameForForm(), Operator.EQ, optionValue);
329                long countOption = _formEntryDAO.getFormEntries(form, false, expr, List.of()).size();
330                Option choiceAttributes = new Option(options.get(option).getLabel(), countOption);
331                values.put(optionValue, choiceAttributes);
332            }
333            
334            if (type.hasOtherOption(question))
335            {
336                // Add other option
337                Expression expr = new AndExpression(
338                    new MetadataExpression(ChoicesListQuestionType.OTHER_PREFIX_DATA_NAME + question.getNameForForm()),
339                    new NotExpression(new StringExpression(ChoicesListQuestionType.OTHER_PREFIX_DATA_NAME + question.getNameForForm(), Operator.EQ, StringUtils.EMPTY))
340                );
341                long countOtherOption = _formEntryDAO.getFormEntries(form, false, expr, List.of()).size();
342                Option choiceAttributes = new Option(ChoicesListQuestionType.OTHER_OPTION_VALUE, countOtherOption);
343                values.put(ChoicesListQuestionType.OTHER_OPTION_VALUE, choiceAttributes);
344            }
345            
346            if (!type.isMandatory(question))
347            {
348                List<Expression> exprs = new ArrayList<>();
349                
350                exprs.add(new OrExpression(
351                    new NotExpression(new MetadataExpression(question.getNameForForm())),
352                    new StringExpression(question.getNameForForm(), Operator.EQ, StringUtils.EMPTY)
353                ));
354                
355                if (type.hasOtherOption(question))
356                {
357                    exprs.add(new OrExpression(
358                            new NotExpression(new MetadataExpression(ChoicesListQuestionType.OTHER_PREFIX_DATA_NAME + question.getNameForForm())),
359                            new StringExpression(ChoicesListQuestionType.OTHER_PREFIX_DATA_NAME + question.getNameForForm(), Operator.EQ, StringUtils.EMPTY)
360                        ));
361                }
362                
363                long countOtherOption = _formEntryDAO.getFormEntries(form, false, new AndExpression(exprs), List.of()).size();
364                Option choiceAttributes = new Option("__internal_not_answered", countOtherOption);
365                values.put("_no_answer", choiceAttributes);
366            }
367        }
368        catch (Exception e)
369        {
370            getLogger().error("An error occurred while trying to get choices options for question " + question.getId(), e);
371        }
372    }
373    
374    /**
375     * Dispatch choices list with remote data stats
376     * @param form the form
377     * @param question the question
378     * @param questionValues the values to fill
379     */
380    protected void _dispatchChoicesWithRemoteDataStats(Form form, FormQuestion question, Map<String, Map<String, Object>> questionValues)
381    {
382        Map<String, Object> values = new LinkedHashMap<>();
383        questionValues.put("values", values);
384        
385        ChoicesListQuestionType type = (ChoicesListQuestionType) question.getType();
386        ChoiceSourceType sourceType = type.getSourceType(question);
387
388        try
389        {
390            long otherCount = 0;
391            long noAnswer = 0;
392            String nameForForm = question.getNameForForm();
393            Map<Object, Long> stats = new LinkedHashMap<>();
394            for (FormEntry entry : form.getEntries())
395            {
396                @SuppressWarnings("cast")
397                List<Object> vals = entry.getValue(nameForForm) != null
398                        ? entry.isMultiple(nameForForm)
399                                ? Arrays.asList(entry.getValue(nameForForm))
400                                : List.of((Object) entry.getValue(nameForForm)) // Need to cast in object because List.of want object and entry.getValue return typed value as string for exemple
401                        : List.of();
402                
403                for (Object value : vals)
404                {
405                    Long count = stats.getOrDefault(value, 0L);
406                    stats.put(value, count + 1);
407                }
408
409                if (type.hasOtherOption(question) && StringUtils.isNotBlank(entry.getValue(ChoicesListQuestionType.OTHER_PREFIX_DATA_NAME + nameForForm)))
410                {
411                    otherCount++;
412                }
413                else if (vals.isEmpty())
414                {
415                    noAnswer++;
416                }
417            }
418        
419            Map<String, Object> enumParam = new HashMap<>();
420            enumParam.put(AbstractSourceType.QUESTION_PARAM_KEY, question);
421            
422            for (Object value : stats.keySet())
423            {
424                I18nizableText entry = sourceType.getEntry(new ChoiceOption(value), enumParam);
425                if (entry != null)
426                {
427                    Option choiceAttributes = new Option(_i18nUtils.translate(entry), stats.get(value));
428                    values.put(value.toString(), choiceAttributes);
429                }
430            }
431            
432            if (otherCount > 0)
433            {
434                Option choiceAttributes = new Option(ChoicesListQuestionType.OTHER_OPTION_VALUE, otherCount);
435                values.put(ChoicesListQuestionType.OTHER_OPTION_VALUE, choiceAttributes);
436            }
437            
438            if (noAnswer > 0)
439            {
440                Option choiceAttributes = new Option("__internal_not_answered", noAnswer);
441                values.put("_no_answer", choiceAttributes);
442            }
443        }
444        catch (Exception e)
445        {
446            getLogger().error("An error occurred while trying to get choices options for question " + question.getId(), e);
447        }
448    }
449    
450    /**
451     * Dispatch boolean stats
452     * @param form the form
453     * @param entries the entries
454     * @param question the question
455     * @param questionValues the values to fill
456     */
457    protected void _dispatchBooleanStats(Form form, List<FormEntry> entries, FormQuestion question, Map<String, Map<String, Object>> questionValues)
458    {
459        Map<String, Object> values = new LinkedHashMap<>();
460        questionValues.put("values", values);
461
462        BooleanExpression booleanExpr = new BooleanExpression(question.getNameForForm(), true);
463        int totalTrue = _formEntryDAO.getFormEntries(form, false, booleanExpr, List.of()).size();
464        
465        values.put("true", totalTrue);
466        values.put("false", entries.size() - totalTrue);
467    }
468    
469    /**
470     * Dispatch default stats
471     * @param form the form
472     * @param entries the entries
473     * @param question the question
474     * @param questionValues the values to fill
475     */
476    protected void _dispatchStats(Form form, List<FormEntry> entries, FormQuestion question, Map<String, Map<String, Object>> questionValues)
477    {
478        Map<String, Object> values = new LinkedHashMap<>();
479        questionValues.put("values", values);
480        
481        Expression expr = new AndExpression(
482            new MetadataExpression(question.getNameForForm()),
483            new NotExpression(new StringExpression(question.getNameForForm(), Operator.EQ, StringUtils.EMPTY))
484        );
485        
486        int totalAnswered = _formEntryDAO.getFormEntries(form, false, expr, List.of()).size();
487        values.put("answered", totalAnswered);
488        values.put("empty", entries.size() - totalAnswered);
489    }
490    
491    private boolean _displayField(FormQuestion question)
492    {
493        FormQuestionType type = question.getType();
494        return type.canBeAnsweredByUser(question);
495    }
496    
497    /**
498     * Record representing a choice list option
499     * @param label the label
500     * @param count the number of time the option is selected
501     */
502    public record Option(String label, long count) { /* empty */ }
503}