001/* 002 * Copyright 2020 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.plugins.contentio.csv; 017 018import java.io.BufferedReader; 019import java.io.ByteArrayInputStream; 020import java.io.FileInputStream; 021import java.io.IOException; 022import java.io.InputStream; 023import java.io.InputStreamReader; 024import java.nio.charset.Charset; 025import java.nio.file.Files; 026import java.nio.file.Paths; 027import java.util.ArrayList; 028import java.util.Arrays; 029import java.util.HashMap; 030import java.util.List; 031import java.util.Map; 032import java.util.Objects; 033import java.util.Optional; 034import java.util.function.Predicate; 035import java.util.stream.Collectors; 036 037import org.apache.avalon.framework.activity.Initializable; 038import org.apache.avalon.framework.component.Component; 039import org.apache.avalon.framework.service.ServiceException; 040import org.apache.avalon.framework.service.ServiceManager; 041import org.apache.avalon.framework.service.Serviceable; 042import org.apache.commons.io.IOUtils; 043import org.apache.commons.lang3.StringUtils; 044import org.apache.tika.detect.DefaultEncodingDetector; 045import org.apache.tika.metadata.Metadata; 046import org.supercsv.io.CsvListReader; 047import org.supercsv.io.CsvMapReader; 048import org.supercsv.io.ICsvListReader; 049import org.supercsv.io.ICsvMapReader; 050import org.supercsv.prefs.CsvPreference; 051 052import org.ametys.cms.contenttype.ContentAttributeDefinition; 053import org.ametys.cms.contenttype.ContentType; 054import org.ametys.cms.contenttype.ContentTypeExtensionPoint; 055import org.ametys.cms.contenttype.ContentTypesHelper; 056import org.ametys.cms.repository.Content; 057import org.ametys.core.ui.Callable; 058import org.ametys.core.user.CurrentUserProvider; 059import org.ametys.core.user.User; 060import org.ametys.core.user.UserIdentity; 061import org.ametys.core.user.UserManager; 062import org.ametys.core.util.I18nUtils; 063import org.ametys.core.util.mail.SendMailHelper; 064import org.ametys.core.util.mail.SendMailHelper.MailBuilder; 065import org.ametys.plugins.contentio.csv.SynchronizeModeEnumerator.ImportMode; 066import org.ametys.plugins.repository.AmetysObjectResolver; 067import org.ametys.plugins.repository.model.RepeaterDefinition; 068import org.ametys.runtime.config.Config; 069import org.ametys.runtime.i18n.I18nizableText; 070import org.ametys.runtime.i18n.I18nizableTextParameter; 071import org.ametys.runtime.model.ElementDefinition; 072import org.ametys.runtime.model.ModelHelper; 073import org.ametys.runtime.model.ModelItem; 074import org.ametys.runtime.model.ModelItemGroup; 075import org.ametys.runtime.model.ModelViewItemGroup; 076import org.ametys.runtime.model.View; 077import org.ametys.runtime.model.ViewElement; 078import org.ametys.runtime.model.ViewElementAccessor; 079import org.ametys.runtime.model.ViewItem; 080import org.ametys.runtime.model.exception.BadItemTypeException; 081import org.ametys.runtime.plugin.component.AbstractLogEnabled; 082 083import jakarta.mail.MessagingException; 084 085/** 086 * Import contents from an uploaded CSV file. 087 */ 088public class ImportCSVFileHelper extends AbstractLogEnabled implements Component, Serviceable, Initializable 089{ 090 /** Avalon Role */ 091 public static final String ROLE = ImportCSVFileHelper.class.getName(); 092 093 /** Result key containing updated content's ids */ 094 public static final String RESULT_IMPORTED_COUNT = "importedCount"; 095 /** Result key containing number of errors */ 096 public static final String RESULT_NB_ERRORS = "nbErrors"; 097 /** Result key containing number of warnings */ 098 public static final String RESULT_NB_WARNINGS = "nbWarnings"; 099 /** Result key containing a short an explanation for a general error */ 100 public static final String RESULT_ERROR = "error"; 101 /** Result key containing the filename */ 102 public static final String RESULT_FILENAME = "fileName"; 103 /** Column name of attribute path in CSV mapping */ 104 public static final String MAPPING_COLUMN_ATTRIBUTE_PATH = "attributePath"; 105 /** Column name of header in CSV mapping */ 106 public static final String MAPPING_COLUMN_HEADER = "header"; 107 /** Column name to identify if column is an ID in CSV mapping */ 108 public static final String MAPPING_COLUMN_IS_ID = "isId"; 109 /** Key in nested mapping to define the ID columns */ 110 public static final String NESTED_MAPPING_ID = "id"; 111 /** Key in nested mapping to define the values columns */ 112 public static final String NESTED_MAPPING_VALUES = "values"; 113 114 private CurrentUserProvider _currentUserProvider; 115 private UserManager _userManager; 116 117 private ContentTypeExtensionPoint _contentTypeEP; 118 private ContentTypesHelper _contentTypesHelper; 119 private AmetysObjectResolver _resolver; 120 121 private I18nUtils _i18nUtils; 122 private String _sysadminMail; 123 124 private CSVImporter _csvImporter; 125 126 @Override 127 public void service(ServiceManager serviceManager) throws ServiceException 128 { 129 _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE); 130 _userManager = (UserManager) serviceManager.lookup(UserManager.ROLE); 131 132 _contentTypeEP = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE); 133 _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE); 134 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 135 136 _i18nUtils = (I18nUtils) serviceManager.lookup(I18nUtils.ROLE); 137 138 _csvImporter = (CSVImporter) serviceManager.lookup(CSVImporter.ROLE); 139 } 140 141 public void initialize() throws Exception 142 { 143 _sysadminMail = Config.getInstance().getValue("smtp.mail.sysadminto"); 144 } 145 146 /** 147 * Retrieves the values for import CSV parameters 148 * @param contentTypeId the configured content type identifier 149 * @return the values for import CSV parameters 150 * @throws Exception if an error occurs. 151 */ 152 @Callable 153 public Map<String, Object> getImportCSVParametersValues(String contentTypeId) throws Exception 154 { 155 Map<String, Object> values = new HashMap<>(); 156 157 // Content types 158 List<String> contentTypeIds = StringUtils.isEmpty(contentTypeId) ? List.of() : List.of(contentTypeId); 159 Map<String, Object> filteredContentTypes = _contentTypesHelper.getContentTypesList(contentTypeIds, true, true, true, false, false); 160 values.put("availableContentTypes", filteredContentTypes.get("contentTypes")); 161 162 // Recipient - get current user email 163 String currentUserEmail = null; 164 UserIdentity currentUser = _currentUserProvider.getUser(); 165 166 String login = currentUser.getLogin(); 167 if (StringUtils.isNotBlank(login)) 168 { 169 String userPopulationId = currentUser.getPopulationId(); 170 User user = _userManager.getUser(userPopulationId, login); 171 currentUserEmail = user.getEmail(); 172 } 173 values.put("defaultRecipient", currentUserEmail); 174 175 return values; 176 } 177 178 /** 179 * Gets the configuration for creating/editing a collection of synchronizable contents. 180 * @param contentTypeId Content type id 181 * @param formValues map of form values 182 * @param mappingValues list of header and content attribute mapping 183 * @return A map containing information about what is needed to create/edit a collection of synchronizable contents 184 * @throws IOException IOException while reading CSV 185 */ 186 @SuppressWarnings("unchecked") 187 @Callable 188 public Map<String, Object> validateConfiguration(String contentTypeId, Map formValues, List<Map<String, Object>> mappingValues) throws IOException 189 { 190 Map<String, Object> result = new HashMap<>(); 191 192 List<Map<String, Object>> filteredMappingValues = _filterMapping(mappingValues); 193 194 List<String> contentAttributes = filteredMappingValues.stream() 195 .map(column -> column.get(MAPPING_COLUMN_ATTRIBUTE_PATH)) 196 .map(String.class::cast) 197 .collect(Collectors.toList()); 198 199 Map<String, Object> mapping = new HashMap<>(); 200 filteredMappingValues.forEach(column -> generateNestedMapping(mapping, column)); 201 202 ContentType contentType = _contentTypeEP.getExtension(contentTypeId); 203 result.put("contentTypeName", contentType.getLabel()); 204 View view = null; 205 for (String itemPath : contentAttributes) 206 { 207 if (!ModelHelper.hasModelItem(itemPath, List.of(contentType))) 208 { 209 result.put("error", "bad-mapping"); 210 result.put("details", List.of(itemPath, contentTypeId)); 211 return result; 212 } 213 } 214 215 try 216 { 217 view = View.of(contentType, contentAttributes.toArray(new String[contentAttributes.size()])); 218 } 219 catch (IllegalArgumentException | BadItemTypeException e) 220 { 221 getLogger().error("Error while creating view", e); 222 result.put("error", "cant-create-view"); 223 result.put("details", contentTypeId); 224 return result; 225 } 226 227 if (!mapping.containsKey(NESTED_MAPPING_ID)) 228 { 229 result.put("error", "missing-id"); 230 return result; 231 } 232 233 for (ViewItem viewItem : view.getViewItems()) 234 { 235 if (!_checkViewItemContainer(viewItem, (Map<String, Object>) mapping.get(NESTED_MAPPING_VALUES), result)) 236 { 237 return result; 238 } 239 } 240 result.put("success", true); 241 return result; 242 } 243 /** 244 * Gets the configuration for creating/editing a collection of synchronizable contents. 245 * @param config get all CSV related parameters: path, separating/escaping char, charset and contentType 246 * @param formValues map of form values 247 * @param mappingValues list of header and content attribute mapping 248 * @return A map containing information about what is needed to create/edit a collection of synchronizable contents 249 * @throws IOException IOException while reading CSV 250 */ 251 @Callable 252 public Map<String, Object> importContents(Map<String, Object> config, Map<String, Object> formValues, List<Map<String, Object>> mappingValues) throws IOException 253 { 254 Map<String, Object> result = new HashMap<>(); 255 result.put(RESULT_FILENAME, config.get("fileName")); 256 257 String path = (String) config.get("path"); 258 String contentTypeId = (String) config.get("contentType"); 259 ContentType contentType = _contentTypeEP.getExtension(contentTypeId); 260 261 // CSV configuration 262 CsvPreference csvPreference = new CsvPreference.Builder( 263 config.getOrDefault("escaping-char", '"').toString().charAt(0), 264 config.getOrDefault("separating-char", ';').toString().charAt(0), 265 "\r\n" 266 ) 267 .build(); 268 269 String encoding = (String) config.get("charset"); 270 Charset charset = Charset.forName(encoding); 271 272 String language = (String) formValues.get("language"); 273 int createAction = Integer.valueOf((String) formValues.getOrDefault("createAction", "1")); 274 int editAction = Integer.valueOf((String) formValues.getOrDefault("editAction", "2")); 275 String workflowName = (String) formValues.get("workflow"); 276 277 ImportMode importMode = Optional.ofNullable((String) formValues.get("importMode")).filter(StringUtils::isNotEmpty).map(ImportMode::valueOf).orElse(ImportMode.CREATE_AND_UPDATE); 278 Optional<String> recipient = Optional.ofNullable(config.get("recipient")) 279 .filter(String.class::isInstance) 280 .map(String.class::cast) 281 .filter(StringUtils::isNotBlank); 282 283 Optional< ? extends Content> parentContent = Optional.empty(); 284 String parentId = (String) config.get("parentId"); 285 if (parentId != null) 286 { 287 Content parent = _resolver.resolveById(parentId); 288 if (parent != null) 289 { 290 parentContent = Optional.of(parent); 291 } 292 } 293 294 @SuppressWarnings("unchecked") 295 Map<String, Object> additionalTransientVars = (Map<String, Object>) config.getOrDefault("additionalTransientVars", new HashMap<>()); 296 297 List<Map<String, Object>> filteredMappingValues = _filterMapping(mappingValues); 298 299 try ( 300 InputStream fileInputStream = new FileInputStream(path); 301 InputStream inputStream = IOUtils.buffer(fileInputStream); 302 ) 303 { 304 result.putAll(_importContents(inputStream, csvPreference, charset, contentType, workflowName, language, createAction, editAction, filteredMappingValues, importMode, parentContent, additionalTransientVars)); 305 306 // If an exception occured and the recipient is empty, transfer the error mail to the configured sysadmin 307 if ("error".equals(result.get(RESULT_ERROR)) && recipient.isEmpty()) 308 { 309 recipient = Optional.ofNullable(_sysadminMail) 310 .filter(StringUtils::isNotBlank); 311 } 312 313 if (recipient.isPresent()) 314 { 315 _sendMail(result, recipient.get()); 316 } 317 } 318 finally 319 { 320 deleteFile(path); 321 } 322 323 return result; 324 } 325 326 /** 327 * Import contents from path, content type and language. 328 * CSV Excel north Europe preferences are used, charset is detected on the file, workflow is detected on content type, default creation (1) and 329 * edition (2) actions are used, values are automatically mapped from the header. Headers should contains a star (*) to detect identifier columns. 330 * @param inputStream The (buffered) input stream to the CSV resource 331 * @param contentType The content type 332 * @param language The language 333 * @param importMode The import mode 334 * @return the result of the CSV import 335 * @throws IOException if an error occurs 336 */ 337 public Map<String, Object> importContents(InputStream inputStream, ContentType contentType, String language, ImportMode importMode) throws IOException 338 { 339 CsvPreference csvPreference = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE; 340 341 // Detect the charset of the current file 342 Charset charset = detectCharset(inputStream); 343 344 // Copy the first 8192 bytes of the initial input stream to another input stream (to get headers) 345 inputStream.mark(8192); 346 try (InputStream headerStream = new ByteArrayInputStream(inputStream.readNBytes(8192))) 347 { 348 inputStream.reset(); 349 350 // Get headers 351 String[] headers = extractHeaders(headerStream, csvPreference, charset); 352 353 // Get the mapping of the values 354 List<Map<String, Object>> mappingValues = getMapping(contentType, headers); 355 356 // Get the default workflow associated to the content type 357 String workflowName = contentType.getDefaultWorkflowName().orElseThrow(() -> new IllegalArgumentException("The workflow can't be defined.")); 358 359 // Import contents (last use of the input stream) 360 return _importContents(inputStream, csvPreference, charset, contentType, workflowName, language, 1, 2, mappingValues, importMode); 361 } 362 } 363 364 private Map<String, Object> _importContents(InputStream inputStream, CsvPreference csvPreference, Charset charset, ContentType contentType, String workflowName, String language, int createAction, int editAction, List<Map<String, Object>> mappingValues, ImportMode importMode) 365 { 366 return _importContents(inputStream, csvPreference, charset, contentType, workflowName, language, createAction, editAction, mappingValues, importMode, Optional.empty(), Map.of()); 367 } 368 369 private Map<String, Object> _importContents(InputStream inputStream, CsvPreference csvPreference, Charset charset, ContentType contentType, String workflowName, String language, int createAction, int editAction, List<Map<String, Object>> mappingValues, ImportMode importMode, Optional< ? extends Content> parentContent, Map<String, Object> additionalTransientVars) 370 { 371 Map<String, Object> result = new HashMap<>(); 372 373 List<String> contentAttributes = mappingValues.stream() 374 .map(column -> column.get(MAPPING_COLUMN_ATTRIBUTE_PATH)) 375 .map(String.class::cast) 376 .collect(Collectors.toList()); 377 378 Map<String, Object> mapping = new HashMap<>(); 379 mappingValues.forEach(column -> generateNestedMapping(mapping, column)); 380 381 View view = View.of(contentType, contentAttributes.toArray(new String[contentAttributes.size()])); 382 383 try ( 384 InputStreamReader inputStreamReader = new InputStreamReader(inputStream, charset); 385 BufferedReader reader = new BufferedReader(inputStreamReader); 386 ICsvListReader csvMapReadernew = new CsvListReader(reader, csvPreference) 387 ) 388 { 389 Map<String, Object> importResult = _csvImporter.importContentsFromCSV(mapping, view, contentType, csvMapReadernew, createAction, editAction, workflowName, language, importMode, parentContent, additionalTransientVars); 390 391 @SuppressWarnings("unchecked") 392 List<String> contentIds = (List<String>) importResult.get(CSVImporter.RESULT_CONTENT_IDS); 393 394 // fill RESULT_ERROR only if the content list is empty and if there are errors, as an empty file or no import in CREATE_ONLY and UPDATE_ONLY mode should not be considered as errors 395 if (contentIds.size() == 0 && (int) importResult.get(CSVImporter.RESULT_NB_ERRORS) > 0) 396 { 397 result.put(RESULT_ERROR, "no-import"); 398 } 399 else 400 { 401 result.put(RESULT_IMPORTED_COUNT, contentIds.size()); 402 result.put(RESULT_NB_ERRORS, importResult.get(CSVImporter.RESULT_NB_ERRORS)); 403 result.put(RESULT_NB_WARNINGS, importResult.get(CSVImporter.RESULT_NB_WARNINGS)); 404 } 405 } 406 catch (Exception e) 407 { 408 getLogger().error("Error while importing contents", e); 409 result.put(RESULT_ERROR, "error"); 410 } 411 412 return result; 413 } 414 415 private boolean _checkViewItemContainer(ViewItem viewItem, Map<String, Object> mapping, Map<String, Object> result) 416 { 417 if (viewItem instanceof ViewElement) 418 { 419 ViewElement viewElement = (ViewElement) viewItem; 420 ElementDefinition elementDefinition = viewElement.getDefinition(); 421 if (elementDefinition instanceof ContentAttributeDefinition) 422 { 423 ViewElementAccessor viewElementAccessor = (ViewElementAccessor) viewElement; 424 ContentAttributeDefinition contentAttributeDefinition = (ContentAttributeDefinition) elementDefinition; 425 return _checkViewItemContentContainer(viewElementAccessor, mapping, result, contentAttributeDefinition); 426 } 427 else 428 { 429 return true; 430 } 431 } 432 else if (viewItem instanceof ModelViewItemGroup) 433 { 434 return _checkViewItemGroupContainer(viewItem, mapping, result); 435 } 436 else 437 { 438 result.put("error", "unknown-type"); 439 result.put("details", viewItem.getName()); 440 return false; 441 } 442 } 443 444 @SuppressWarnings("unchecked") 445 private boolean _checkViewItemContentContainer(ViewElementAccessor viewElementAccessor, Map<String, Object> mapping, Map<String, Object> result, ContentAttributeDefinition contentAttributeDefinition) 446 { 447 448 boolean success = true; 449 String contentTypeId = contentAttributeDefinition.getContentTypeId(); 450 ContentType contentType = _contentTypeEP.getExtension(contentTypeId); 451 if (mapping.get(viewElementAccessor.getName()) instanceof String) 452 { 453 // Path matches a content instead of a contents attribute 454 result.put("error", "string-as-container"); 455 result.put("details", mapping.get(viewElementAccessor.getName())); 456 return false; 457 } 458 Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewElementAccessor.getName()); 459 if (!nestedMap.containsKey(NESTED_MAPPING_ID) || ((List<String>) nestedMap.get(NESTED_MAPPING_ID)).isEmpty()) 460 { 461 // No identifier for the content 462 result.put("error", "missing-id-content"); 463 result.put("details", List.of(contentType.getLabel(), contentAttributeDefinition.getName())); 464 return false; 465 } 466 if (viewElementAccessor.getDefinition().isMultiple()) 467 { 468 Optional<RepeaterDefinition> repeater = _getRepeaterAncestorIfExists(viewElementAccessor.getDefinition()); 469 if (repeater.isPresent()) 470 { 471 // Content is multiple inside a repeater, we do not handle this as we can't discriminate content id from repeater id 472 result.put("error", "multiple-content-inside-repeater"); 473 result.put("details", List.of(contentType.getLabel(), repeater.get().getLabel())); 474 return false; 475 } 476 } 477 Map<String, Object> nestedMapValues = (Map<String, Object>) (nestedMap.get(NESTED_MAPPING_VALUES)); 478 for (ViewItem nestedViewItem : viewElementAccessor.getViewItems()) 479 { 480 success = success && _checkViewItemContainer(nestedViewItem, nestedMapValues, result); 481 } 482 return success; 483 } 484 485 /** 486 * Get the repeater ancestor of the given model item if exists, searching parent's parent recursively if needed. 487 * @param modelItem the model item 488 * @return the repeater ancestor if exists 489 */ 490 private Optional<RepeaterDefinition> _getRepeaterAncestorIfExists(ModelItem modelItem) 491 { 492 ModelItemGroup parent = modelItem.getParent(); 493 // There is no parent anymore: we stop there 494 if (parent == null) 495 { 496 return Optional.empty(); 497 } 498 // Parent is a repeater: we stop there and return the repeater 499 if (parent instanceof RepeaterDefinition repeaterParent) 500 { 501 return Optional.of(repeaterParent); 502 } 503 // The parent is not a repeater, but his parent may be one. Check if the parent have a repeater as parent. 504 else 505 { 506 return _getRepeaterAncestorIfExists(parent); 507 } 508 } 509 510 @SuppressWarnings("unchecked") 511 private boolean _checkViewItemGroupContainer(ViewItem viewItem, Map<String, Object> mapping, Map<String, Object> result) 512 { 513 boolean success = true; 514 ModelViewItemGroup modelViewItemGroup = (ModelViewItemGroup) viewItem; 515 List<ViewItem> elementDefinition = modelViewItemGroup.getViewItems(); 516 Map<String, Object> nestedMap = (Map<String, Object>) mapping.get(viewItem.getName()); 517 518 Map<String, Object> nestedMapValues = (Map<String, Object>) (nestedMap.get(NESTED_MAPPING_VALUES)); 519 520 for (ViewItem nestedViewItem : elementDefinition) 521 { 522 success = success && _checkViewItemContainer(nestedViewItem, nestedMapValues, result); 523 } 524 return success; 525 } 526 527 private static Map<String, Object> generateNestedMapping(Map<String, Object> mapping, Map<String, Object> column) 528 { 529 return generateNestedMapping(mapping, (String) column.get(MAPPING_COLUMN_ATTRIBUTE_PATH), (String) column.get(ImportCSVFileHelper.MAPPING_COLUMN_HEADER), (boolean) column.get(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID)); 530 } 531 532 @SuppressWarnings("unchecked") 533 private static Map<String, Object> generateNestedMapping(Map<String, Object> mapping, String attributePath, String column, boolean isId) 534 { 535 Map<String, Object> values = (Map<String, Object>) mapping.computeIfAbsent(NESTED_MAPPING_VALUES, __ -> new HashMap<>()); 536 537 int separatorIndex = attributePath.indexOf('/'); 538 if (separatorIndex == -1) 539 { 540 if (isId) 541 { 542 List<String> ids = (List<String>) mapping.computeIfAbsent(NESTED_MAPPING_ID, __ -> new ArrayList<>()); 543 ids.add(attributePath); 544 } 545 values.put(attributePath, column); 546 } 547 else 548 { 549 Map<String, Object> subValuesMapping = (Map<String, Object>) values.computeIfAbsent(attributePath.substring(0, separatorIndex), __ -> new HashMap<>()); 550 generateNestedMapping(subValuesMapping, attributePath.substring(separatorIndex + 1), column, isId); 551 } 552 553 mapping.put(NESTED_MAPPING_VALUES, values); 554 return mapping; 555 } 556 557 private void _sendMail(Map<String, Object> importResult, String recipient) 558 { 559 I18nizableText subject = _getMailSubject(importResult); 560 I18nizableText body = _getMailBody(importResult); 561 562 try 563 { 564 MailBuilder mailBuilder = SendMailHelper.newMail() 565 .withSubject(_i18nUtils.translate(subject)) 566 .withRecipient(recipient) 567 .withTextBody(_i18nUtils.translate(body)); 568 569 mailBuilder.sendMail(); 570 } 571 catch (MessagingException | IOException e) 572 { 573 if (getLogger().isWarnEnabled()) 574 { 575 getLogger().warn("Unable to send the e-mail '" + subject + "' to '" + recipient + "'", e); 576 } 577 } 578 catch (Exception e) 579 { 580 getLogger().error("An unknown error has occured while sending the mail.", e); 581 } 582 } 583 584 private I18nizableText _getMailSubject(Map<String, Object> importResult) 585 { 586 Map<String, I18nizableTextParameter> i18nParams = Map.of("fileName", new I18nizableText((String) importResult.get(RESULT_FILENAME))); 587 if (importResult.containsKey(ImportCSVFileHelper.RESULT_ERROR)) 588 { 589 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_ERROR_SUBJECT", i18nParams); 590 } 591 else 592 { 593 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_SUCCESS_SUBJECT", i18nParams); 594 } 595 } 596 597 private I18nizableText _getMailBody(Map<String, Object> importResult) 598 { 599 Map<String, I18nizableTextParameter> i18nParams = new HashMap<>(); 600 i18nParams.put("fileName", new I18nizableText((String) importResult.get(RESULT_FILENAME))); 601 602 if (importResult.containsKey(ImportCSVFileHelper.RESULT_ERROR)) 603 { 604 if ("no-import".equals(importResult.get(ImportCSVFileHelper.RESULT_ERROR))) 605 { 606 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_ERROR_BODY_NO_IMPORT", i18nParams); 607 } 608 else 609 { 610 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_ERROR_BODY_TEXT", i18nParams); 611 } 612 } 613 else 614 { 615 int nbErrors = (int) importResult.get(RESULT_NB_ERRORS); 616 int nbWarnings = (int) importResult.get(RESULT_NB_WARNINGS); 617 i18nParams.put("importedCount", new I18nizableText(String.valueOf(importResult.get(RESULT_IMPORTED_COUNT)))); 618 i18nParams.put("nbErrors", new I18nizableText(String.valueOf(nbErrors))); 619 i18nParams.put("nbWarnings", new I18nizableText(String.valueOf(nbWarnings))); 620 621 if (nbErrors == 0 && nbWarnings == 0) 622 { 623 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_SUCCESS_BODY", i18nParams); 624 } 625 else if (nbWarnings == 0) 626 { 627 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_PARTIAL_SUCCESS_BODY_ERRORS", i18nParams); 628 } 629 else if (nbErrors == 0) 630 { 631 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_PARTIAL_SUCCESS_BODY_WARNINGS", i18nParams); 632 } 633 else 634 { 635 return new I18nizableText("plugin.contentio", "PLUGINS_CONTENTIO_CONTENT_IMPORT_MAIL_PARTIAL_SUCCESS_BODY_ERRORS_AND_WARNINGS", i18nParams); 636 } 637 } 638 } 639 640 /** 641 * Delete the file related to the given path 642 * @param path path of the file 643 * @throws IOException if an error occurs while deleting the file 644 */ 645 @Callable 646 public void deleteFile(String path) throws IOException 647 { 648 Files.delete(Paths.get(path)); 649 } 650 651 /** 652 * Detect the charset of a given resource. 653 * @param inputStream The input stream to the resource, need to support {@link InputStream#mark(int)} and {@link InputStream#reset()}. 654 * @return the charset 655 * @throws IOException if an error occurs 656 */ 657 public Charset detectCharset(InputStream inputStream) throws IOException 658 { 659 assert inputStream.markSupported(); 660 DefaultEncodingDetector defaultEncodingDetector = new DefaultEncodingDetector(); 661 return defaultEncodingDetector.detect(inputStream, new Metadata()); 662 } 663 664 /** 665 * Extract headers from the CSV resource. 666 * @param inputStream The input stream to the resource 667 * @param csvPreference The CSV preference (separators) 668 * @param charset The charset of the resource 669 * @return the list of headers in file 670 * @throws IOException if an error occurs 671 */ 672 public String[] extractHeaders(InputStream inputStream, CsvPreference csvPreference, Charset charset) throws IOException 673 { 674 try ( 675 InputStreamReader inputStreamReader = new InputStreamReader(inputStream, charset); 676 BufferedReader reader = new BufferedReader(inputStreamReader); 677 ICsvMapReader mapReader = new CsvMapReader(reader, csvPreference); 678 ) 679 { 680 String[] headers = mapReader.getHeader(true); 681 return headers; 682 } 683 } 684 685 /** 686 * Get the mapping from the headers and corresponding to the content type. 687 * @param contentType The content type 688 * @param headers The CSV headers 689 * @return the list of columns descriptions. An item of the list is one column, each column is defined by a header "header" (text defined into the CSV header), an 690 * attribute path "attributePath" (real path extracted from the header, empty if the model item does not exist) and an identifier flag "isId" (<code>true</code> 691 * if the header ends with a star "*"). 692 */ 693 public List<Map<String, Object>> getMapping(ContentType contentType, String[] headers) 694 { 695 return Arrays.asList(headers) 696 .stream() 697 .filter(StringUtils::isNotEmpty) 698 .map(header -> _transformHeaderToMap(contentType, header)) 699 .collect(Collectors.toList()); 700 } 701 702 private Map<String, Object> _transformHeaderToMap(ContentType contentType, String header) 703 { 704 Map<String, Object> headerMap = new HashMap<>(); 705 headerMap.put(ImportCSVFileHelper.MAPPING_COLUMN_HEADER, header); 706 headerMap.put( 707 MAPPING_COLUMN_ATTRIBUTE_PATH, 708 Optional.of(header) 709 .map(s -> s.replace("*", "")) 710 .map(s -> s.replace(".", "/")) 711 .map(String::trim) 712 .filter(contentType::hasModelItem) 713 .orElse(StringUtils.EMPTY) 714 ); 715 headerMap.put(ImportCSVFileHelper.MAPPING_COLUMN_IS_ID, header.endsWith("*")); 716 return headerMap; 717 } 718 719 private List<Map<String, Object>> _filterMapping(List<Map<String, Object>> mappingValues) 720 { 721 // Suppress empty attributePath 722 // Transform attributePath with . to attributePath with / (Autocompletion in IHM has . as separator, but the API need /) 723 return mappingValues.stream() 724 .map(column -> 725 { 726 Optional<String> attributePath = Optional.of(MAPPING_COLUMN_ATTRIBUTE_PATH) 727 .map(column::get) 728 .map(String.class::cast) 729 .filter(StringUtils::isNotEmpty) 730 .map(s -> s.replace(".", "/")); 731 732 if (attributePath.isEmpty()) 733 { 734 return null; 735 } 736 737 // Replace original column 738 Map<String, Object> columnCopy = new HashMap<>(column); 739 columnCopy.put(MAPPING_COLUMN_ATTRIBUTE_PATH, attributePath.get()); 740 return columnCopy; 741 } 742 ) 743 .filter(Predicate.not(Objects::isNull)) 744 .collect(Collectors.toList()); 745 } 746}