Source code

001/*
002 *  Copyright 2019 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.odf.skill;
017
018import java.io.BufferedReader;
019import java.io.IOException;
020import java.nio.charset.StandardCharsets;
021import java.nio.file.Files;
022import java.nio.file.Paths;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.LinkedHashMap;
027import java.util.List;
028import java.util.Map;
029import java.util.Objects;
030import java.util.Optional;
031import java.util.Set;
032import java.util.function.Predicate;
033import java.util.stream.Collectors;
034
035import org.apache.avalon.framework.component.Component;
036import org.apache.avalon.framework.service.ServiceException;
037import org.apache.avalon.framework.service.ServiceManager;
038import org.apache.avalon.framework.service.Serviceable;
039import org.apache.commons.lang.ArrayUtils;
040import org.apache.commons.lang3.StringUtils;
041import org.supercsv.io.CsvListReader;
042import org.supercsv.io.ICsvListReader;
043import org.supercsv.prefs.CsvPreference;
044
045import org.ametys.cms.ObservationConstants;
046import org.ametys.cms.data.ContentValue;
047import org.ametys.cms.indexing.solr.SolrIndexHelper;
048import org.ametys.cms.repository.Content;
049import org.ametys.cms.repository.ContentQueryHelper;
050import org.ametys.cms.repository.ContentTypeExpression;
051import org.ametys.cms.repository.ModifiableContent;
052import org.ametys.cms.repository.WorkflowAwareContent;
053import org.ametys.cms.workflow.AbstractContentWorkflowComponent;
054import org.ametys.cms.workflow.ContentWorkflowHelper;
055import org.ametys.core.observation.Event;
056import org.ametys.core.observation.ObservationManager;
057import org.ametys.core.ui.Callable;
058import org.ametys.core.user.CurrentUserProvider;
059import org.ametys.odf.ODFHelper;
060import org.ametys.odf.ProgramItem;
061import org.ametys.odf.course.Course;
062import org.ametys.odf.enumeration.OdfReferenceTableEntry;
063import org.ametys.odf.enumeration.OdfReferenceTableHelper;
064import org.ametys.odf.program.AbstractProgram;
065import org.ametys.odf.program.Container;
066import org.ametys.plugins.repository.AmetysObjectIterable;
067import org.ametys.plugins.repository.AmetysObjectIterator;
068import org.ametys.plugins.repository.AmetysObjectResolver;
069import org.ametys.plugins.repository.AmetysRepositoryException;
070import org.ametys.plugins.repository.ModifiableAmetysObject;
071import org.ametys.plugins.repository.data.holder.group.ModifiableModelAwareRepeater;
072import org.ametys.plugins.repository.data.holder.group.ModifiableModelAwareRepeaterEntry;
073import org.ametys.plugins.repository.query.expression.AndExpression;
074import org.ametys.plugins.repository.query.expression.Expression.Operator;
075import org.ametys.plugins.repository.query.expression.StringExpression;
076import org.ametys.runtime.plugin.component.AbstractLogEnabled;
077
078import com.opensymphony.workflow.WorkflowException;
079
080/**
081 * ODF skills helper
082 */
083public class ODFSkillsHelper extends AbstractLogEnabled implements Serviceable, Component
084{
085    /** The avalon role. */
086    public static final String ROLE = ODFSkillsHelper.class.getName();
087    
088    /** The skills other names attribute name */
089    public static final String SKILL_OTHER_NAMES_ATTRIBUTE_NAME = "otherNames";
090    
091    /** The internal attribute name to excluded from skills */
092    public static final String SKILLS_EXCLUDED_INTERNAL_ATTRIBUTE_NAME = "excluded";
093    
094    /** The content workflow helper */
095    protected ContentWorkflowHelper _contentWorkflowHelper;
096
097    /** The ametys object resolver */
098    protected AmetysObjectResolver _resolver;
099
100    /** The ODF helper */
101    protected ODFHelper _odfHelper;
102    
103    /** The observation manager */
104    protected ObservationManager _observationManager;
105
106    /** The Solr index helper */
107    protected SolrIndexHelper _solrIndexHelper;
108    
109    /** The current user provider */
110    protected CurrentUserProvider _currentUserProvider;
111    
112    public void service(ServiceManager manager) throws ServiceException
113    {
114        _contentWorkflowHelper = (ContentWorkflowHelper) manager.lookup(ContentWorkflowHelper.ROLE);
115        _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE);
116        _odfHelper = (ODFHelper) manager.lookup(ODFHelper.ROLE);
117        _observationManager = (ObservationManager) manager.lookup(ObservationManager.ROLE);
118        _currentUserProvider = (CurrentUserProvider) manager.lookup(CurrentUserProvider.ROLE);
119    }
120
121    /**
122     * Exclude or include the program items from skills display
123     * @param programItemIds the list of program item ids
124     * @param excluded <code>true</code> if the program items need to be excluded.
125     * @return the map of changed program items properties
126     */
127    @Callable
128    public Map<String, Object> setProgramItemsExclusion(List<String> programItemIds, boolean excluded)
129    {
130        Map<String, Object> results = new HashMap<>();
131        results.put("allright-program-items", new ArrayList<>());
132        
133        for (String programItemId : programItemIds)
134        {
135            ProgramItem programItem = _resolver.resolveById(programItemId);
136            if (programItem instanceof AbstractProgram || programItem instanceof Container)
137            {
138                ((Content) programItem).getInternalDataHolder().setValue(SKILLS_EXCLUDED_INTERNAL_ATTRIBUTE_NAME, excluded);
139                ((ModifiableAmetysObject) programItem).saveChanges();
140
141                Map<String, Object> programItem2Json = new HashMap<>();
142                programItem2Json.put("id", programItem.getId());
143                programItem2Json.put("title", ((Content) programItem).getTitle());
144                
145                @SuppressWarnings("unchecked")
146                List<Map<String, Object>> allRightProgramItems = (List<Map<String, Object>>) results.get("allright-program-items");
147                allRightProgramItems.add(programItem2Json);
148                
149                Map<String, Object> eventParams = new HashMap<>();
150                eventParams.put(ObservationConstants.ARGS_CONTENT, programItem);
151                eventParams.put(ObservationConstants.ARGS_CONTENT_ID, programItem.getId());
152                eventParams.put(org.ametys.odf.observation.OdfObservationConstants.ODF_CONTENT_SKILLS_EXCLUSION_ARG, excluded);
153                _observationManager.notify(new Event(org.ametys.odf.observation.OdfObservationConstants.ODF_CONTENT_SKILLS_EXCLUSION_CHANGED, _currentUserProvider.getUser(), eventParams));
154            }
155        }
156        
157        return results;
158        
159    }
160    
161    /**
162     * <code>true</code> if the program item is excluded from skills display
163     * @param programItem the program item
164     * @return <code>true</code> if the program item is excluded from skills display
165     */
166    public boolean isExcluded(ProgramItem programItem)
167    {
168        if (programItem instanceof AbstractProgram || programItem instanceof Container)
169        {
170            return ((Content) programItem).getInternalDataHolder().getValue(SKILLS_EXCLUDED_INTERNAL_ATTRIBUTE_NAME, false);
171        }
172        
173        return false;
174    }
175    
176    /**
177     * Get the computed skill values for a given {@link AbstractProgram}
178     * @param abstractProgram the abstract program
179     * @param maxDepth the max depth of courses. For example, set to 1 to compute skills over UE only, set 2 to compute skills over UE and EC, ...
180     * @return the skill values computed from the attached courses
181     */
182    public Set<ContentValue> getComputedSkills(AbstractProgram abstractProgram, int maxDepth)
183    {
184        return _computeSkills(abstractProgram, 1, maxDepth);
185    }
186    
187    private Set<ContentValue> _computeSkills(ProgramItem programItem, int depth, int maxDepth)
188    {
189        Set<ContentValue> skills = new HashSet<>();
190        
191        if (programItem instanceof Course)
192        {
193            // Get skills of current course
194            ModifiableModelAwareRepeater repeaterSkills = ((Course) programItem).getRepeater(Course.ACQUIRED_SKILLS);
195            if (repeaterSkills != null)
196            {
197                List< ? extends ModifiableModelAwareRepeaterEntry> entries = repeaterSkills.getEntries();
198                for (ModifiableModelAwareRepeaterEntry entry : entries)
199                {
200                    ModifiableModelAwareRepeater repeater = entry.getRepeater(Course.ACQUIRED_SKILLS_SKILLS);
201                    if (repeater != null)
202                    {
203                        skills.addAll(repeater.getEntries().stream()
204                            .map(e -> (ContentValue) e.getValue(Course.ACQUIRED_SKILLS_SKILLS_SKILL, false, null))
205                            .filter(Objects::nonNull)
206                            .collect(Collectors.toSet()));
207                    }
208                }
209            }
210            
211            if (depth < maxDepth)
212            {
213                ((Course) programItem).getCourseLists()
214                    .stream()
215                    .forEach(cl -> 
216                    {
217                        cl.getCourses()
218                            .stream().forEach(c -> 
219                            {
220                                skills.addAll(_computeSkills(c, depth + 1, maxDepth));
221                            });
222                    });
223            }
224        }
225        else 
226        {
227            List<ProgramItem> childProgramItems = _odfHelper.getChildProgramItems(programItem);
228            for (ProgramItem childProgramItem : childProgramItems)
229            {
230                skills.addAll(_computeSkills(childProgramItem, depth, maxDepth));
231            }
232        }
233        
234        return skills;
235    }
236    
237    /**
238     * Get the skills distribution by courses over a {@link ProgramItem}
239     * Distribution is computed over the course of first level only
240     * @param programItem the program item
241     * @return the skills distribution
242     */
243    public Map<Content, Map<Content, Map<Content, Content>>> getSkillsDistribution(ProgramItem programItem)
244    {
245        return getSkillsDistribution(programItem, 1);
246    }
247    
248    /**
249     * Get the skills distribution by courses over a {@link ProgramItem}
250     * @param programItem the program item
251     * @param maxDepth the max depth of courses. For example, set to 1 to compute distribution over UE only, set 2 to compute distribution over UE and EC, ...
252     * @return the skills distribution as Map&lt;SkillSet, Map&lt;Skill, Map&lt;Course, AcquisitionLevel&gt;&gt;&gt;
253     */
254    public Map<Content, Map<Content, Map<Content, Content>>> getSkillsDistribution(ProgramItem programItem, int maxDepth)
255    {
256        // Map<SkillSet, Map<Skill, Map<Course, AcquisitionLevel>>>
257        Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution = new LinkedHashMap<>();
258        
259        if (!isExcluded(programItem))
260        {
261            _buildSkillsDistribution(programItem, skillsDistribution, maxDepth);
262        }
263        
264        return skillsDistribution;
265    }
266    
267    
268    
269    private void _buildSkillsDistribution(ProgramItem programItem, Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution, int maxDepth)
270    {
271        if (programItem instanceof Course)
272        {
273            _buildSkillsDistribution((Course) programItem, (Course) programItem, skillsDistribution, 1, maxDepth);
274        }
275        else 
276        {
277            List<ProgramItem> children = _odfHelper.getChildProgramItems(programItem)
278                .stream()
279                .filter(Predicate.not(this::isExcluded))
280                .collect(Collectors.toList());
281            for (ProgramItem childProgramItem : children)
282            {
283                _buildSkillsDistribution(childProgramItem, skillsDistribution, maxDepth);
284            }
285        }
286    }
287    
288    private void _buildSkillsDistribution(Course course, Course parentCourse, Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution, int depth, int maxDepth)
289    {
290        List<? extends ModifiableModelAwareRepeaterEntry> acquiredSkillEntries = Optional.of(course)
291                .map(e -> e.getRepeater(Course.ACQUIRED_SKILLS))
292                .map(ModifiableModelAwareRepeater::getEntries)
293                .orElse(List.of());
294        
295        for (ModifiableModelAwareRepeaterEntry acquiredSkillEntry : acquiredSkillEntries)
296        {
297            Optional.of(acquiredSkillEntry)
298                .map(e -> e.<ContentValue>getValue(Course.ACQUIRED_SKILLS_SKILLSET))
299                .flatMap(ContentValue::getContentIfExists)
300                .ifPresent(
301                    skillSet ->
302                    {
303                        Map<Content, Map<Content, Content>> skills = skillsDistribution.computeIfAbsent(skillSet, __ -> new LinkedHashMap<>());
304                        
305                        List<? extends ModifiableModelAwareRepeaterEntry> skillEntries = Optional.of(acquiredSkillEntry)
306                            .map(e -> e.getRepeater(Course.ACQUIRED_SKILLS_SKILLS))
307                            .map(ModifiableModelAwareRepeater::getEntries)
308                            .orElse(List.of());
309                        
310                        for (ModifiableModelAwareRepeaterEntry skillEntry : skillEntries)
311                        {
312                            Content skill = Optional.of(skillEntry)
313                                .map(entry -> entry.<ContentValue>getValue(Course.ACQUIRED_SKILLS_SKILLS_SKILL))
314                                .flatMap(ContentValue::getContentIfExists)
315                                .orElse(null);
316                            
317                            if (skill != null)
318                            {
319                                Content acquisitionLevel = 
320                                        Optional.of(skillEntry)
321                                        .map(entry -> entry.<ContentValue>getValue(Course.ACQUIRED_SKILLS_SKILLS_ACQUISITION_LEVEL))
322                                        .flatMap(ContentValue::getContentIfExists)
323                                        .orElse(null);
324                                
325                                Map<Content, Content> courses = skills.computeIfAbsent(skill, s -> new LinkedHashMap<>());
326                                courses.put(parentCourse, _getMaxAcquisitionLevel(acquisitionLevel, courses.get(parentCourse)));
327                            }
328                        }
329                    }
330                );
331        }
332        
333        if (depth < maxDepth)
334        {
335            // Get skills distribution over child courses
336            course.getCourseLists()
337                .stream()
338                .forEach(cl -> 
339                {
340                    cl.getCourses()
341                        .stream().forEach(c -> 
342                        {
343                            _buildSkillsDistribution(c, parentCourse, skillsDistribution, depth + 1, maxDepth);
344                        });
345                });
346        }
347    }
348    
349    private Content _getMaxAcquisitionLevel(Content level1, Content level2)
350    {
351        if (level1 == null)
352        {
353            return level2;
354        }
355        
356        if (level2 == null)
357        {
358            return level1;
359        }
360        
361        long order1 = level1.getValue(OdfReferenceTableEntry.ORDER, false, -1L);
362        long order2 = level2.getValue(OdfReferenceTableEntry.ORDER, false, -1L);
363        
364        if (order1 >= order2)
365        {
366            return level1;
367        }
368        else
369        {
370            return level2;
371        }
372    }
373    
374    /**
375     * Create all skills from ESCO file
376     * @param skillsCSVFilePath the skills CSV file path
377     */
378    public void createSkillsFromESCOFileCSV(String skillsCSVFilePath)
379    {
380        String[] handledEvents = new String[] {
381            ObservationConstants.EVENT_CONTENT_ADDED, 
382            ObservationConstants.EVENT_CONTENT_MODIFIED,  
383            ObservationConstants.EVENT_CONTENT_WORKFLOW_CHANGED,
384            ObservationConstants.EVENT_CONTENT_TAGGED,
385            ObservationConstants.EVENT_CONTENT_DELETED,
386        };
387        
388        try
389        {
390            _solrIndexHelper.pauseSolrCommitForEvents(handledEvents);
391            for (Skill skill : _getSkillsFromCSVFile(skillsCSVFilePath))
392            {
393                try
394                {
395                    _createSkillTableRef(skill);
396                }
397                catch (AmetysRepositoryException | WorkflowException e) 
398                {
399                    getLogger().warn("An error occurred creating skill with label {}", skill.getLabel(), e);
400                }
401            }
402        }
403        finally
404        {
405            _solrIndexHelper.restartSolrCommitForEvents(handledEvents);
406        }
407    }
408    
409    /**
410     * Get the list of skills from the csv file
411     * @param skillsCSVFilePath the skills CSV file path
412     * @return the list of skills
413     */
414    protected List<Skill> _getSkillsFromCSVFile(String skillsCSVFilePath)
415    {
416        List<Skill> skills = new ArrayList<>();
417        try (BufferedReader reader = Files.newBufferedReader(Paths.get(skillsCSVFilePath), StandardCharsets.UTF_8);
418             ICsvListReader listReader = new CsvListReader(reader, CsvPreference.STANDARD_PREFERENCE))
419        {
420            listReader.getHeader(true); //Skip header
421
422            List<String> read = listReader.read();
423            while (read != null)
424            {
425                String conceptUri = read.get(1); // Uri
426                String label = read.get(4); // Get label
427                if (StringUtils.isNotBlank(label))
428                {
429                    String otherNamesAsString = read.get(5); // Get other names
430                    String[] otherNames = StringUtils.isNotBlank(otherNamesAsString) ? StringUtils.split(otherNamesAsString, "\n") : ArrayUtils.EMPTY_STRING_ARRAY;
431                    skills.add(new Skill(label, otherNames, conceptUri));
432                }
433                read = listReader.read();
434            }
435        }
436        catch (IOException e)
437        {
438            getLogger().warn("An error occurred parsing file {}", skillsCSVFilePath, e);
439        }
440        
441        getLogger().info("Find {} skills into file {}", skills.size(), skillsCSVFilePath);
442        
443        return skills;
444    }
445    
446    /**
447     * Create a skill table ref content from the skill object
448     * @param skill the skill object
449     * @throws AmetysRepositoryException if a repository error occurred
450     * @throws WorkflowException if a workflow error occurred
451     */
452    protected void _createSkillTableRef(Skill skill) throws AmetysRepositoryException, WorkflowException
453    {
454        String uri = skill.getConceptUri();
455        String titleFR = skill.getLabel();
456        String[] otherNames = skill.getOtherNames();
457        
458        ContentTypeExpression cTypeExpr = new ContentTypeExpression(Operator.EQ, OdfReferenceTableHelper.SKILL);
459        StringExpression codeExpr = new StringExpression(OdfReferenceTableEntry.CODE, Operator.EQ, uri);
460        
461        String xpathQuery = ContentQueryHelper.getContentXPathQuery(new AndExpression(cTypeExpr, codeExpr));
462        AmetysObjectIterable<ModifiableContent> contents = _resolver.query(xpathQuery);
463        AmetysObjectIterator<ModifiableContent> it = contents.iterator();
464        
465        if (!it.hasNext())
466        {
467            Map<String, String> titleVariants = new HashMap<>();
468            titleVariants.put("fr", titleFR);
469            
470            Map<String, Object> result = _contentWorkflowHelper.createContent("reference-table", 1, titleFR, titleVariants, new String[] {OdfReferenceTableHelper.SKILL}, new String[0]);
471            ModifiableContent content = (ModifiableContent) result.get(AbstractContentWorkflowComponent.CONTENT_KEY);
472            
473            content.setValue(OdfReferenceTableEntry.CODE, uri);
474            
475            if (otherNames.length > 0)
476            {
477                content.setValue(SKILL_OTHER_NAMES_ATTRIBUTE_NAME, otherNames);
478            }
479            
480            content.saveChanges();
481            _contentWorkflowHelper.doAction((WorkflowAwareContent) content, 22);
482            
483            getLogger().info("Skill's content \"{}\" ({}) was successfully created", titleFR, content.getId());
484        }
485    }
486    
487    private static class Skill
488    {
489        private String _label;
490        private String[] _otherNames;
491        private String _conceptUri;
492        
493        public Skill(String label, String[] otherNames, String conceptUri)
494        {
495            _label = label;
496            _otherNames = otherNames;
497            _conceptUri = conceptUri;
498        }
499        
500        public String getLabel()
501        {
502            return _label;
503        }
504
505        public String[] getOtherNames()
506        {
507            return _otherNames;
508        }
509        
510        public String getConceptUri()
511        {
512            return _conceptUri;
513        }
514    }
515}