001/* 002 * Copyright 2019 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.odf.skill; 017 018import java.io.BufferedReader; 019import java.io.IOException; 020import java.nio.charset.StandardCharsets; 021import java.nio.file.Files; 022import java.nio.file.Paths; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.HashSet; 026import java.util.LinkedHashMap; 027import java.util.List; 028import java.util.Map; 029import java.util.Objects; 030import java.util.Optional; 031import java.util.Set; 032import java.util.function.Predicate; 033import java.util.stream.Collectors; 034 035import org.apache.avalon.framework.component.Component; 036import org.apache.avalon.framework.service.ServiceException; 037import org.apache.avalon.framework.service.ServiceManager; 038import org.apache.avalon.framework.service.Serviceable; 039import org.apache.commons.lang.ArrayUtils; 040import org.apache.commons.lang3.StringUtils; 041import org.supercsv.io.CsvListReader; 042import org.supercsv.io.ICsvListReader; 043import org.supercsv.prefs.CsvPreference; 044 045import org.ametys.cms.ObservationConstants; 046import org.ametys.cms.data.ContentValue; 047import org.ametys.cms.indexing.solr.SolrIndexHelper; 048import org.ametys.cms.repository.Content; 049import org.ametys.cms.repository.ContentQueryHelper; 050import org.ametys.cms.repository.ContentTypeExpression; 051import org.ametys.cms.repository.ModifiableContent; 052import org.ametys.cms.repository.WorkflowAwareContent; 053import org.ametys.cms.workflow.AbstractContentWorkflowComponent; 054import org.ametys.cms.workflow.ContentWorkflowHelper; 055import org.ametys.core.observation.Event; 056import org.ametys.core.observation.ObservationManager; 057import org.ametys.core.ui.Callable; 058import org.ametys.core.user.CurrentUserProvider; 059import org.ametys.odf.ODFHelper; 060import org.ametys.odf.ProgramItem; 061import org.ametys.odf.course.Course; 062import org.ametys.odf.enumeration.OdfReferenceTableEntry; 063import org.ametys.odf.enumeration.OdfReferenceTableHelper; 064import org.ametys.odf.program.AbstractProgram; 065import org.ametys.odf.program.Container; 066import org.ametys.plugins.repository.AmetysObjectIterable; 067import org.ametys.plugins.repository.AmetysObjectIterator; 068import org.ametys.plugins.repository.AmetysObjectResolver; 069import org.ametys.plugins.repository.AmetysRepositoryException; 070import org.ametys.plugins.repository.ModifiableAmetysObject; 071import org.ametys.plugins.repository.data.holder.group.ModifiableModelAwareRepeater; 072import org.ametys.plugins.repository.data.holder.group.ModifiableModelAwareRepeaterEntry; 073import org.ametys.plugins.repository.query.expression.AndExpression; 074import org.ametys.plugins.repository.query.expression.Expression.Operator; 075import org.ametys.plugins.repository.query.expression.StringExpression; 076import org.ametys.runtime.plugin.component.AbstractLogEnabled; 077 078import com.opensymphony.workflow.WorkflowException; 079 080/** 081 * ODF skills helper 082 */ 083public class ODFSkillsHelper extends AbstractLogEnabled implements Serviceable, Component 084{ 085 /** The avalon role. */ 086 public static final String ROLE = ODFSkillsHelper.class.getName(); 087 088 /** The skills other names attribute name */ 089 public static final String SKILL_OTHER_NAMES_ATTRIBUTE_NAME = "otherNames"; 090 091 /** The internal attribute name to excluded from skills */ 092 public static final String SKILLS_EXCLUDED_INTERNAL_ATTRIBUTE_NAME = "excluded"; 093 094 /** The content workflow helper */ 095 protected ContentWorkflowHelper _contentWorkflowHelper; 096 097 /** The ametys object resolver */ 098 protected AmetysObjectResolver _resolver; 099 100 /** The ODF helper */ 101 protected ODFHelper _odfHelper; 102 103 /** The observation manager */ 104 protected ObservationManager _observationManager; 105 106 /** The Solr index helper */ 107 protected SolrIndexHelper _solrIndexHelper; 108 109 /** The current user provider */ 110 protected CurrentUserProvider _currentUserProvider; 111 112 public void service(ServiceManager manager) throws ServiceException 113 { 114 _contentWorkflowHelper = (ContentWorkflowHelper) manager.lookup(ContentWorkflowHelper.ROLE); 115 _resolver = (AmetysObjectResolver) manager.lookup(AmetysObjectResolver.ROLE); 116 _odfHelper = (ODFHelper) manager.lookup(ODFHelper.ROLE); 117 _observationManager = (ObservationManager) manager.lookup(ObservationManager.ROLE); 118 _currentUserProvider = (CurrentUserProvider) manager.lookup(CurrentUserProvider.ROLE); 119 } 120 121 /** 122 * Get the name of the catalog of a skill content 123 * @param contentId The id of content 124 * @return The catalog's name or null if the content does not have a catalog 125 */ 126 @Callable 127 public String getCatalog(String contentId) 128 { 129 Content content = _resolver.resolveById(contentId); 130 131 if (content.hasValue("catalog")) 132 { 133 return content.getValue("catalog"); 134 } 135 136 return null; 137 } 138 139 /** 140 * Exclude or include the program items from skills display 141 * @param programItemIds the list of program item ids 142 * @param excluded <code>true</code> if the program items need to be excluded. 143 * @return the map of changed program items properties 144 */ 145 @Callable 146 public Map<String, Object> setProgramItemsExclusion(List<String> programItemIds, boolean excluded) 147 { 148 Map<String, Object> results = new HashMap<>(); 149 results.put("allright-program-items", new ArrayList<>()); 150 151 for (String programItemId : programItemIds) 152 { 153 ProgramItem programItem = _resolver.resolveById(programItemId); 154 if (programItem instanceof AbstractProgram || programItem instanceof Container) 155 { 156 ((Content) programItem).getInternalDataHolder().setValue(SKILLS_EXCLUDED_INTERNAL_ATTRIBUTE_NAME, excluded); 157 ((ModifiableAmetysObject) programItem).saveChanges(); 158 159 Map<String, Object> programItem2Json = new HashMap<>(); 160 programItem2Json.put("id", programItem.getId()); 161 programItem2Json.put("title", ((Content) programItem).getTitle()); 162 163 @SuppressWarnings("unchecked") 164 List<Map<String, Object>> allRightProgramItems = (List<Map<String, Object>>) results.get("allright-program-items"); 165 allRightProgramItems.add(programItem2Json); 166 167 Map<String, Object> eventParams = new HashMap<>(); 168 eventParams.put(ObservationConstants.ARGS_CONTENT, programItem); 169 eventParams.put(ObservationConstants.ARGS_CONTENT_ID, programItem.getId()); 170 eventParams.put(org.ametys.odf.observation.OdfObservationConstants.ODF_CONTENT_SKILLS_EXCLUSION_ARG, excluded); 171 _observationManager.notify(new Event(org.ametys.odf.observation.OdfObservationConstants.ODF_CONTENT_SKILLS_EXCLUSION_CHANGED, _currentUserProvider.getUser(), eventParams)); 172 } 173 } 174 175 return results; 176 177 } 178 179 /** 180 * <code>true</code> if the program item is excluded from skills display 181 * @param programItem the program item 182 * @return <code>true</code> if the program item is excluded from skills display 183 */ 184 public boolean isExcluded(ProgramItem programItem) 185 { 186 if (programItem instanceof AbstractProgram || programItem instanceof Container) 187 { 188 return ((Content) programItem).getInternalDataHolder().getValue(SKILLS_EXCLUDED_INTERNAL_ATTRIBUTE_NAME, false); 189 } 190 191 return false; 192 } 193 194 /** 195 * Get the computed skill values for a given {@link AbstractProgram} 196 * @param abstractProgram the abstract program 197 * @param maxDepth the max depth of courses. For example, set to 1 to compute skills over UE only, set 2 to compute skills over UE and EC, ... 198 * @return the skill values computed from the attached courses 199 */ 200 public Set<ContentValue> getComputedSkills(AbstractProgram abstractProgram, int maxDepth) 201 { 202 return _computeSkills(abstractProgram, 1, maxDepth); 203 } 204 205 private Set<ContentValue> _computeSkills(ProgramItem programItem, int depth, int maxDepth) 206 { 207 Set<ContentValue> skills = new HashSet<>(); 208 209 if (programItem instanceof Course) 210 { 211 // Get skills of current course 212 ModifiableModelAwareRepeater repeaterSkills = ((Course) programItem).getRepeater(Course.ACQUIRED_SKILLS); 213 if (repeaterSkills != null) 214 { 215 List< ? extends ModifiableModelAwareRepeaterEntry> entries = repeaterSkills.getEntries(); 216 for (ModifiableModelAwareRepeaterEntry entry : entries) 217 { 218 ModifiableModelAwareRepeater repeater = entry.getRepeater(Course.ACQUIRED_SKILLS_SKILLS); 219 if (repeater != null) 220 { 221 skills.addAll(repeater.getEntries().stream() 222 .map(e -> (ContentValue) e.getValue(Course.ACQUIRED_SKILLS_SKILLS_SKILL, false, null)) 223 .filter(Objects::nonNull) 224 .collect(Collectors.toSet())); 225 } 226 } 227 } 228 229 if (depth < maxDepth) 230 { 231 ((Course) programItem).getCourseLists() 232 .stream() 233 .forEach(cl -> 234 { 235 cl.getCourses() 236 .stream().forEach(c -> 237 { 238 skills.addAll(_computeSkills(c, depth + 1, maxDepth)); 239 }); 240 }); 241 } 242 } 243 else 244 { 245 List<ProgramItem> childProgramItems = _odfHelper.getChildProgramItems(programItem); 246 for (ProgramItem childProgramItem : childProgramItems) 247 { 248 skills.addAll(_computeSkills(childProgramItem, depth, maxDepth)); 249 } 250 } 251 252 return skills; 253 } 254 255 /** 256 * Get the skills distribution by courses over a {@link ProgramItem} 257 * Distribution is computed over the course of first level only 258 * @param programItem the program item 259 * @return the skills distribution 260 */ 261 public Map<Content, Map<Content, Map<Content, Content>>> getSkillsDistribution(ProgramItem programItem) 262 { 263 return getSkillsDistribution(programItem, 1); 264 } 265 266 /** 267 * Get the skills distribution by courses over a {@link ProgramItem} 268 * @param programItem the program item 269 * @param maxDepth the max depth of courses. For example, set to 1 to compute distribution over UE only, set 2 to compute distribution over UE and EC, ... 270 * @return the skills distribution as Map<SkillSet, Map<Skill, Map<Course, AcquisitionLevel>>> 271 */ 272 public Map<Content, Map<Content, Map<Content, Content>>> getSkillsDistribution(ProgramItem programItem, int maxDepth) 273 { 274 // Map<SkillSet, Map<Skill, Map<Course, AcquisitionLevel>>> 275 Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution = new LinkedHashMap<>(); 276 277 if (!isExcluded(programItem)) 278 { 279 _buildSkillsDistribution(programItem, skillsDistribution, maxDepth); 280 } 281 282 return skillsDistribution; 283 } 284 285 286 287 private void _buildSkillsDistribution(ProgramItem programItem, Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution, int maxDepth) 288 { 289 if (programItem instanceof Course) 290 { 291 _buildSkillsDistribution((Course) programItem, (Course) programItem, skillsDistribution, 1, maxDepth); 292 } 293 else 294 { 295 List<ProgramItem> children = _odfHelper.getChildProgramItems(programItem) 296 .stream() 297 .filter(Predicate.not(this::isExcluded)) 298 .collect(Collectors.toList()); 299 for (ProgramItem childProgramItem : children) 300 { 301 _buildSkillsDistribution(childProgramItem, skillsDistribution, maxDepth); 302 } 303 } 304 } 305 306 private void _buildSkillsDistribution(Course course, Course parentCourse, Map<Content, Map<Content, Map<Content, Content>>> skillsDistribution, int depth, int maxDepth) 307 { 308 List<? extends ModifiableModelAwareRepeaterEntry> acquiredSkillEntries = Optional.of(course) 309 .map(e -> e.getRepeater(Course.ACQUIRED_SKILLS)) 310 .map(ModifiableModelAwareRepeater::getEntries) 311 .orElse(List.of()); 312 313 for (ModifiableModelAwareRepeaterEntry acquiredSkillEntry : acquiredSkillEntries) 314 { 315 Optional.of(acquiredSkillEntry) 316 .map(e -> e.<ContentValue>getValue(Course.ACQUIRED_SKILLS_SKILLSET)) 317 .flatMap(ContentValue::getContentIfExists) 318 .ifPresent( 319 skillSet -> 320 { 321 Map<Content, Map<Content, Content>> skills = skillsDistribution.computeIfAbsent(skillSet, __ -> new LinkedHashMap<>()); 322 323 List<? extends ModifiableModelAwareRepeaterEntry> skillEntries = Optional.of(acquiredSkillEntry) 324 .map(e -> e.getRepeater(Course.ACQUIRED_SKILLS_SKILLS)) 325 .map(ModifiableModelAwareRepeater::getEntries) 326 .orElse(List.of()); 327 328 for (ModifiableModelAwareRepeaterEntry skillEntry : skillEntries) 329 { 330 Content skill = Optional.of(skillEntry) 331 .map(entry -> entry.<ContentValue>getValue(Course.ACQUIRED_SKILLS_SKILLS_SKILL)) 332 .flatMap(ContentValue::getContentIfExists) 333 .orElse(null); 334 335 if (skill != null) 336 { 337 Content acquisitionLevel = 338 Optional.of(skillEntry) 339 .map(entry -> entry.<ContentValue>getValue(Course.ACQUIRED_SKILLS_SKILLS_ACQUISITION_LEVEL)) 340 .flatMap(ContentValue::getContentIfExists) 341 .orElse(null); 342 343 Map<Content, Content> courses = skills.computeIfAbsent(skill, s -> new LinkedHashMap<>()); 344 courses.put(parentCourse, _getMaxAcquisitionLevel(acquisitionLevel, courses.get(parentCourse))); 345 } 346 } 347 } 348 ); 349 } 350 351 if (depth < maxDepth) 352 { 353 // Get skills distribution over child courses 354 course.getCourseLists() 355 .stream() 356 .forEach(cl -> 357 { 358 cl.getCourses() 359 .stream().forEach(c -> 360 { 361 _buildSkillsDistribution(c, parentCourse, skillsDistribution, depth + 1, maxDepth); 362 }); 363 }); 364 } 365 } 366 367 private Content _getMaxAcquisitionLevel(Content level1, Content level2) 368 { 369 if (level1 == null) 370 { 371 return level2; 372 } 373 374 if (level2 == null) 375 { 376 return level1; 377 } 378 379 long order1 = level1.getValue(OdfReferenceTableEntry.ORDER, false, -1L); 380 long order2 = level2.getValue(OdfReferenceTableEntry.ORDER, false, -1L); 381 382 if (order1 >= order2) 383 { 384 return level1; 385 } 386 else 387 { 388 return level2; 389 } 390 } 391 392 /** 393 * Create all skills from ESCO file 394 * @param skillsCSVFilePath the skills CSV file path 395 */ 396 public void createSkillsFromESCOFileCSV(String skillsCSVFilePath) 397 { 398 String[] handledEvents = new String[] { 399 ObservationConstants.EVENT_CONTENT_ADDED, 400 ObservationConstants.EVENT_CONTENT_MODIFIED, 401 ObservationConstants.EVENT_CONTENT_WORKFLOW_CHANGED, 402 ObservationConstants.EVENT_CONTENT_TAGGED, 403 ObservationConstants.EVENT_CONTENT_DELETED, 404 }; 405 406 try 407 { 408 _solrIndexHelper.pauseSolrCommitForEvents(handledEvents); 409 for (Skill skill : _getSkillsFromCSVFile(skillsCSVFilePath)) 410 { 411 try 412 { 413 _createSkillTableRef(skill); 414 } 415 catch (AmetysRepositoryException | WorkflowException e) 416 { 417 getLogger().warn("An error occurred creating skill with label {}", skill.getLabel(), e); 418 } 419 } 420 } 421 finally 422 { 423 _solrIndexHelper.restartSolrCommitForEvents(handledEvents); 424 } 425 } 426 427 /** 428 * Get the list of skills from the csv file 429 * @param skillsCSVFilePath the skills CSV file path 430 * @return the list of skills 431 */ 432 protected List<Skill> _getSkillsFromCSVFile(String skillsCSVFilePath) 433 { 434 List<Skill> skills = new ArrayList<>(); 435 try (BufferedReader reader = Files.newBufferedReader(Paths.get(skillsCSVFilePath), StandardCharsets.UTF_8); 436 ICsvListReader listReader = new CsvListReader(reader, CsvPreference.STANDARD_PREFERENCE)) 437 { 438 listReader.getHeader(true); //Skip header 439 440 List<String> read = listReader.read(); 441 while (read != null) 442 { 443 String conceptUri = read.get(1); // Uri 444 String label = read.get(4); // Get label 445 if (StringUtils.isNotBlank(label)) 446 { 447 String otherNamesAsString = read.get(5); // Get other names 448 String[] otherNames = StringUtils.isNotBlank(otherNamesAsString) ? StringUtils.split(otherNamesAsString, "\n") : ArrayUtils.EMPTY_STRING_ARRAY; 449 skills.add(new Skill(label, otherNames, conceptUri)); 450 } 451 read = listReader.read(); 452 } 453 } 454 catch (IOException e) 455 { 456 getLogger().warn("An error occurred parsing file {}", skillsCSVFilePath, e); 457 } 458 459 getLogger().info("Find {} skills into file {}", skills.size(), skillsCSVFilePath); 460 461 return skills; 462 } 463 464 /** 465 * Create a skill table ref content from the skill object 466 * @param skill the skill object 467 * @throws AmetysRepositoryException if a repository error occurred 468 * @throws WorkflowException if a workflow error occurred 469 */ 470 protected void _createSkillTableRef(Skill skill) throws AmetysRepositoryException, WorkflowException 471 { 472 String uri = skill.getConceptUri(); 473 String titleFR = skill.getLabel(); 474 String[] otherNames = skill.getOtherNames(); 475 476 ContentTypeExpression cTypeExpr = new ContentTypeExpression(Operator.EQ, OdfReferenceTableHelper.SKILL); 477 StringExpression codeExpr = new StringExpression(OdfReferenceTableEntry.CODE, Operator.EQ, uri); 478 479 String xpathQuery = ContentQueryHelper.getContentXPathQuery(new AndExpression(cTypeExpr, codeExpr)); 480 AmetysObjectIterable<ModifiableContent> contents = _resolver.query(xpathQuery); 481 AmetysObjectIterator<ModifiableContent> it = contents.iterator(); 482 483 if (!it.hasNext()) 484 { 485 Map<String, String> titleVariants = new HashMap<>(); 486 titleVariants.put("fr", titleFR); 487 488 Map<String, Object> result = _contentWorkflowHelper.createContent("reference-table", 1, titleFR, titleVariants, new String[] {OdfReferenceTableHelper.SKILL}, new String[0]); 489 ModifiableContent content = (ModifiableContent) result.get(AbstractContentWorkflowComponent.CONTENT_KEY); 490 491 content.setValue(OdfReferenceTableEntry.CODE, uri); 492 493 if (otherNames.length > 0) 494 { 495 content.setValue(SKILL_OTHER_NAMES_ATTRIBUTE_NAME, otherNames); 496 } 497 498 content.saveChanges(); 499 _contentWorkflowHelper.doAction((WorkflowAwareContent) content, 22); 500 501 getLogger().info("Skill's content \"{}\" ({}) was successfully created", titleFR, content.getId()); 502 } 503 } 504 505 private static class Skill 506 { 507 private String _label; 508 private String[] _otherNames; 509 private String _conceptUri; 510 511 public Skill(String label, String[] otherNames, String conceptUri) 512 { 513 _label = label; 514 _otherNames = otherNames; 515 _conceptUri = conceptUri; 516 } 517 518 public String getLabel() 519 { 520 return _label; 521 } 522 523 public String[] getOtherNames() 524 { 525 return _otherNames; 526 } 527 528 public String getConceptUri() 529 { 530 return _conceptUri; 531 } 532 } 533}