001/* 002 * Copyright 2019 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.odf.skill; 017 018import java.io.File; 019import java.io.FileInputStream; 020import java.io.IOException; 021import java.io.InputStreamReader; 022import java.util.ArrayList; 023import java.util.HashMap; 024import java.util.HashSet; 025import java.util.LinkedHashMap; 026import java.util.List; 027import java.util.Map; 028import java.util.Objects; 029import java.util.Optional; 030import java.util.Set; 031import java.util.stream.Collectors; 032 033import org.apache.avalon.framework.component.Component; 034import org.apache.avalon.framework.service.ServiceException; 035import org.apache.avalon.framework.service.ServiceManager; 036import org.apache.avalon.framework.service.Serviceable; 037import org.apache.commons.lang.ArrayUtils; 038import org.apache.commons.lang3.StringUtils; 039import org.supercsv.io.CsvListReader; 040import org.supercsv.prefs.CsvPreference; 041 042import org.ametys.cms.ObservationConstants; 043import org.ametys.cms.data.ContentValue; 044import org.ametys.cms.repository.Content; 045import org.ametys.cms.repository.ContentQueryHelper; 046import org.ametys.cms.repository.ContentTypeExpression; 047import org.ametys.cms.repository.ModifiableContent; 048import org.ametys.cms.repository.WorkflowAwareContent; 049import org.ametys.cms.workflow.AbstractContentWorkflowComponent; 050import org.ametys.cms.workflow.ContentWorkflowHelper; 051import org.ametys.core.observation.ObservationManager; 052import org.ametys.odf.ODFHelper; 053import org.ametys.odf.ProgramItem; 054import org.ametys.odf.course.Course; 055import org.ametys.odf.enumeration.OdfReferenceTableEntry; 056import org.ametys.odf.program.AbstractProgram; 057import org.ametys.plugins.repository.AmetysObjectIterable; 058import org.ametys.plugins.repository.AmetysObjectIterator; 059import org.ametys.plugins.repository.AmetysObjectResolver; 060import org.ametys.plugins.repository.AmetysRepositoryException; 061import org.ametys.plugins.repository.data.holder.group.impl.ModifiableModelAwareRepeater; 062import org.ametys.plugins.repository.data.holder.group.impl.ModifiableModelAwareRepeaterEntry; 063import org.ametys.plugins.repository.query.expression.AndExpression; 064import org.ametys.plugins.repository.query.expression.Expression.Operator; 065import org.ametys.plugins.repository.query.expression.StringExpression; 066import org.ametys.runtime.plugin.component.AbstractLogEnabled; 067 068import com.opensymphony.workflow.WorkflowException; 069 070/** 071 * ODF skills helper 072 */ 073public class ODFSkillsHelper extends AbstractLogEnabled implements Serviceable, Component 074{ 075 /** The avalon role. */ 076 public static final String ROLE = ODFSkillsHelper.class.getName(); 077 078 /** The name of repeater for acquired skills */ 079 public static final String COURSES_REPEATER_ACQUIRED_SKILLS = "acquiredSkills"; 080 081 /** The skill content type id */ 082 public static final String SKILL_CONTENT_TYPE = "odf-enumeration.Skill"; 083 084 /** The attribute name of the skills */ 085 public static final String SKILLS_ATTRIBUTE_NAME = "skills"; 086 087 /** The skills other names attribute name */ 088 public static final String SKILL_OTHER_NAMES_ATTRIBUTE_NAME = "otherNames"; 089 090 /** The content workflow helper */ 091 protected ContentWorkflowHelper _contentWorkflowHelper; 092 093 /** The ametys object resolver */ 094 protected AmetysObjectResolver _resolver; 095 096 /** The ODF helper */ 097 protected ODFHelper _odfHelper; 098 099 /** The observation manager */ 100 protected ObservationManager _observationManager; 101 102 public void service(ServiceManager manager) throws ServiceException 103 { 104 _contentWorkflowHelper = (ContentWorkflowHelper) manager.lookup(ContentWorkflowHelper.ROLE); 105 _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE); 106 _odfHelper = (ODFHelper) manager.lookup(ODFHelper.ROLE); 107 _observationManager = (ObservationManager) manager.lookup(ObservationManager.ROLE); 108 } 109 110 /** 111 * Get the computed skill values for a given {@link AbstractProgram} 112 * @param abstractProgram the abstract program 113 * @param maxDepth the max depth of courses. For example, set to 1 to compute skills over UE only, set 2 to compute skills over UE and EC, ... 114 * @return the skill values computed from the attached courses 115 */ 116 public Set<ContentValue> getComputedSkills(AbstractProgram abstractProgram, int maxDepth) 117 { 118 return _computeSkills(abstractProgram, 1, maxDepth); 119 } 120 121 private Set<ContentValue> _computeSkills(ProgramItem programItem, int depth, int maxDepth) 122 { 123 Set<ContentValue> skills = new HashSet<>(); 124 125 if (programItem instanceof Course) 126 { 127 // Get skills of current course 128 ModifiableModelAwareRepeater repeaterSkills = ((Course) programItem).getRepeater(COURSES_REPEATER_ACQUIRED_SKILLS); 129 if (repeaterSkills != null) 130 { 131 List< ? extends ModifiableModelAwareRepeaterEntry> entries = repeaterSkills.getEntries(); 132 for (ModifiableModelAwareRepeaterEntry entry : entries) 133 { 134 ModifiableModelAwareRepeater repeater = entry.getRepeater("skills"); 135 if (repeater != null) 136 { 137 skills.addAll(repeater.getEntries().stream() 138 .map(e -> (ContentValue) e.getValue("skill", false, null)) 139 .filter(Objects::nonNull) 140 .collect(Collectors.toSet())); 141 } 142 } 143 } 144 145 if (depth < maxDepth) 146 { 147 ((Course) programItem).getCourseLists() 148 .stream() 149 .forEach(cl -> 150 { 151 cl.getCourses() 152 .stream().forEach(c -> 153 { 154 skills.addAll(_computeSkills(c, depth + 1, maxDepth)); 155 }); 156 }); 157 } 158 } 159 else 160 { 161 List<ProgramItem> childProgramItems = _odfHelper.getChildProgramItems(programItem); 162 for (ProgramItem childProgramItem : childProgramItems) 163 { 164 skills.addAll(_computeSkills(childProgramItem, depth, maxDepth)); 165 } 166 } 167 168 return skills; 169 } 170 171 /** 172 * Get the skills distribution by courses over a {@link ProgramItem} 173 * Distribution is computed over the course of first level only 174 * @param programItem the program item 175 * @return the skills distribution 176 */ 177 public Map<Content, Map<Content, Map<Content, Content>>> getSkillsDistribution(ProgramItem programItem) 178 { 179 return getSkillsDistribution(programItem, 1); 180 } 181 182 /** 183 * Get the skills distribution by courses over a {@link ProgramItem} 184 * @param programItem the program item 185 * @param maxDepth the max depth of courses. For example, set to 1 to compute distribution over UE only, set 2 to compute distribution over UE and EC, ... 186 * @return the skills distribution as Map<SkillSet, Map<Skill, Map<Course, AcquisitionLevel>>> 187 */ 188 public Map<Content, Map<Content, Map<Content, Content>>> getSkillsDistribution(ProgramItem programItem, int maxDepth) 189 { 190 // Map<SkillSet, Map<Skill, Map<Course, AcquisitionLevel>>> 191 Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution = new LinkedHashMap<>(); 192 193 _buildSkillsDistribution(programItem, skillsDistribution, maxDepth); 194 195 return skillsDistribution; 196 } 197 198 199 200 private void _buildSkillsDistribution(ProgramItem programItem, Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution, int maxDepth) 201 { 202 if (programItem instanceof Course) 203 { 204 _buildSkillsDistribution((Course) programItem, (Course) programItem, skillsDistribution, 1, maxDepth); 205 } 206 else 207 { 208 List<ProgramItem> childProgramItems = _odfHelper.getChildProgramItems(programItem); 209 for (ProgramItem childProgramItem : childProgramItems) 210 { 211 _buildSkillsDistribution(childProgramItem, skillsDistribution, maxDepth); 212 } 213 } 214 } 215 216 private void _buildSkillsDistribution(Course course, Course parentCourse, Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution, int depth, int maxDepth) 217 { 218 List<? extends ModifiableModelAwareRepeaterEntry> acquiredSkillEntries = Optional.of(course) 219 .map(e -> e.getRepeater(COURSES_REPEATER_ACQUIRED_SKILLS)) 220 .map(ModifiableModelAwareRepeater::getEntries) 221 .orElse(List.of()); 222 223 for (ModifiableModelAwareRepeaterEntry acquiredSkillEntry : acquiredSkillEntries) 224 { 225 Optional.of(acquiredSkillEntry) 226 .map(e -> e.<ContentValue>getValue("skillSet")) 227 .flatMap(ContentValue::getContentIfExists) 228 .ifPresent( 229 skillSet -> 230 { 231 Map<Content, Map<Content, Content>> skills = skillsDistribution.computeIfAbsent(skillSet, __ -> new LinkedHashMap<>()); 232 233 List<? extends ModifiableModelAwareRepeaterEntry> skillEntries = Optional.of(acquiredSkillEntry) 234 .map(e -> e.getRepeater("skills")) 235 .map(ModifiableModelAwareRepeater::getEntries) 236 .orElse(List.of()); 237 238 for (ModifiableModelAwareRepeaterEntry skillEntry : skillEntries) 239 { 240 Content skill = Optional.of(skillEntry) 241 .map(entry -> entry.<ContentValue>getValue("skill")) 242 .flatMap(ContentValue::getContentIfExists) 243 .orElse(null); 244 245 if (skill != null) 246 { 247 Content acquisitionLevel = 248 Optional.of(skillEntry) 249 .map(entry -> entry.<ContentValue>getValue("acquisitionLevel")) 250 .flatMap(ContentValue::getContentIfExists) 251 .orElse(null); 252 253 Map<Content, Content> courses = skills.computeIfAbsent(skill, s -> new LinkedHashMap<>()); 254 courses.put(parentCourse, _getMaxAcquisitionLevel(acquisitionLevel, courses.get(parentCourse))); 255 } 256 } 257 } 258 ); 259 } 260 261 if (depth < maxDepth) 262 { 263 // Get skills distribution over child courses 264 course.getCourseLists() 265 .stream() 266 .forEach(cl -> 267 { 268 cl.getCourses() 269 .stream().forEach(c -> 270 { 271 _buildSkillsDistribution(c, parentCourse, skillsDistribution, depth + 1, maxDepth); 272 }); 273 }); 274 } 275 } 276 277 private Content _getMaxAcquisitionLevel(Content level1, Content level2) 278 { 279 if (level1 == null) 280 { 281 return level2; 282 } 283 284 if (level2 == null) 285 { 286 return level1; 287 } 288 289 long order1 = level1.getValue("order", false, -1L); 290 long order2 = level2.getValue("order", false, -1L); 291 292 if (order1 >= order2) 293 { 294 return level1; 295 } 296 else 297 { 298 return level2; 299 } 300 } 301 302 /** 303 * Create all skills from ESCO file 304 * @param skillsCSVFilePath the skills CSV file path 305 */ 306 public void createSkillsFromESCOFileCSV(String skillsCSVFilePath) 307 { 308 String[] events = new String[] { 309 ObservationConstants.EVENT_CONTENT_ADDED, 310 ObservationConstants.EVENT_CONTENT_MODIFIED, 311 ObservationConstants.EVENT_CONTENT_WORKFLOW_CHANGED, 312 ObservationConstants.EVENT_CONTENT_TAGGED, 313 ObservationConstants.EVENT_CONTENT_DELETED, 314 }; 315 _observationManager.addArgumentForEvents(events, ObservationConstants.ARGS_CONTENT_COMMIT, false); 316 317 try 318 { 319 for (Skill skill : _getSkillsFromCSVFile(skillsCSVFilePath)) 320 { 321 try 322 { 323 _createSkillTableRef(skill); 324 } 325 catch (AmetysRepositoryException | WorkflowException e) 326 { 327 getLogger().warn("An error occurred creating skill with label {}", skill.getLabel(), e); 328 } 329 } 330 } 331 finally 332 { 333 _observationManager.removeArgumentForEvents(events, ObservationConstants.ARGS_CONTENT_COMMIT); 334 } 335 } 336 337 /** 338 * Get the list of skills from the csv file 339 * @param skillsCSVFilePath the skills CSV file path 340 * @return the list of skills 341 */ 342 protected List<Skill> _getSkillsFromCSVFile(String skillsCSVFilePath) 343 { 344 List<Skill> skills = new ArrayList<>(); 345 try (CsvListReader listReader = new CsvListReader(new InputStreamReader(new FileInputStream(new File(skillsCSVFilePath)), "UTF-8"), CsvPreference.STANDARD_PREFERENCE)) 346 { 347 listReader.getHeader(true); //Skip header 348 349 List<String> read = listReader.read(); 350 while (read != null) 351 { 352 String conceptUri = read.get(1); // Uri 353 String label = read.get(4); // Get label 354 if (StringUtils.isNotBlank(label)) 355 { 356 String otherNamesAsString = read.get(5); // Get other names 357 String[] otherNames = StringUtils.isNotBlank(otherNamesAsString) ? StringUtils.split(otherNamesAsString, "\n") : ArrayUtils.EMPTY_STRING_ARRAY; 358 skills.add(new Skill(label, otherNames, conceptUri)); 359 } 360 read = listReader.read(); 361 } 362 } 363 catch (IOException e) 364 { 365 getLogger().warn("An error occurred parsing file {}", skillsCSVFilePath, e); 366 } 367 368 getLogger().info("Find {} skills into file {}", skills.size(), skillsCSVFilePath); 369 370 return skills; 371 } 372 373 /** 374 * Create a skill table ref content from the skill object 375 * @param skill the skill object 376 * @throws AmetysRepositoryException if a repository error occurred 377 * @throws WorkflowException if a workflow error occurred 378 */ 379 protected void _createSkillTableRef(Skill skill) throws AmetysRepositoryException, WorkflowException 380 { 381 String uri = skill.getConceptUri(); 382 String titleFR = skill.getLabel(); 383 String[] otherNames = skill.getOtherNames(); 384 385 ContentTypeExpression cTypeExpr = new ContentTypeExpression(Operator.EQ, SKILL_CONTENT_TYPE); 386 StringExpression codeExpr = new StringExpression(OdfReferenceTableEntry.CODE, Operator.EQ, uri); 387 388 String xpathQuery = ContentQueryHelper.getContentXPathQuery(new AndExpression(cTypeExpr, codeExpr)); 389 AmetysObjectIterable<ModifiableContent> contents = _resolver.query(xpathQuery); 390 AmetysObjectIterator<ModifiableContent> it = contents.iterator(); 391 392 if (!it.hasNext()) 393 { 394 Map<String, String> titleVariants = new HashMap<>(); 395 titleVariants.put("fr", titleFR); 396 397 Map<String, Object> result = _contentWorkflowHelper.createContent("reference-table", 1, titleFR, titleVariants, new String[] {SKILL_CONTENT_TYPE}, new String[0]); 398 ModifiableContent content = (ModifiableContent) result.get(AbstractContentWorkflowComponent.CONTENT_KEY); 399 400 content.setValue(OdfReferenceTableEntry.CODE, uri); 401 402 if (otherNames.length > 0) 403 { 404 content.setValue(SKILL_OTHER_NAMES_ATTRIBUTE_NAME, otherNames); 405 } 406 407 content.saveChanges(); 408 _contentWorkflowHelper.doAction((WorkflowAwareContent) content, 22); 409 410 getLogger().info("Skill's content \"{}\" ({}) was successfully created", titleFR, content.getId()); 411 } 412 } 413 414 private static class Skill 415 { 416 private String _label; 417 private String[] _otherNames; 418 private String _conceptUri; 419 420 public Skill(String label, String[] otherNames, String conceptUri) 421 { 422 _label = label; 423 _otherNames = otherNames; 424 _conceptUri = conceptUri; 425 } 426 427 public String getLabel() 428 { 429 return _label; 430 } 431 432 public String[] getOtherNames() 433 { 434 return _otherNames; 435 } 436 437 public String getConceptUri() 438 { 439 return _conceptUri; 440 } 441 } 442}