001/* 002 * Copyright 2017 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.plugins.contentio.synchronize; 017 018import java.util.ArrayList; 019import java.util.Collections; 020import java.util.HashMap; 021import java.util.LinkedHashMap; 022import java.util.List; 023import java.util.Map; 024import java.util.stream.Collectors; 025 026import org.apache.avalon.framework.configuration.Configuration; 027import org.apache.avalon.framework.configuration.ConfigurationException; 028import org.apache.avalon.framework.service.ServiceException; 029import org.apache.avalon.framework.service.ServiceManager; 030import org.apache.commons.lang3.StringUtils; 031import org.slf4j.Logger; 032 033import org.ametys.cms.ObservationConstants; 034import org.ametys.cms.content.ContentHelper; 035import org.ametys.cms.contenttype.ContentType; 036import org.ametys.cms.repository.Content; 037import org.ametys.cms.repository.DefaultContent; 038import org.ametys.cms.repository.ModifiableDefaultContent; 039import org.ametys.cms.repository.WorkflowAwareContent; 040import org.ametys.core.observation.Event; 041import org.ametys.plugins.repository.AmetysObjectIterable; 042import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata; 043 044import com.google.common.collect.ImmutableMap; 045 046/** 047 * Abstract implementation of {@link SynchronizableContentsCollection}. 048 */ 049public abstract class AbstractSimpleSynchronizableContentsCollection extends AbstractSynchronizableContentsCollection 050{ 051 /** The extension point for Synchronizing Content Operators */ 052 protected SynchronizingContentOperatorExtensionPoint _synchronizingContentOperatorEP; 053 054 /** SCC helper */ 055 protected SynchronizableContentsCollectionHelper _sccHelper; 056 /** The content helper */ 057 protected ContentHelper _contentHelper; 058 059 private List<String> _handledContents; 060 061 @Override 062 public void service(ServiceManager manager) throws ServiceException 063 { 064 super.service(manager); 065 _contentHelper = (ContentHelper) manager.lookup(ContentHelper.ROLE); 066 _synchronizingContentOperatorEP = (SynchronizingContentOperatorExtensionPoint) manager.lookup(SynchronizingContentOperatorExtensionPoint.ROLE); 067 _sccHelper = (SynchronizableContentsCollectionHelper) manager.lookup(SynchronizableContentsCollectionHelper.ROLE); 068 } 069 070 @Override 071 public void configure(Configuration configuration) throws ConfigurationException 072 { 073 super.configure(configuration); 074 _handledContents = new ArrayList<>(); 075 } 076 077 @Override 078 public List<ModifiableDefaultContent> populate(Logger logger) 079 { 080 _handledContents.clear(); 081 List<ModifiableDefaultContent> populatedContents = super.populate(logger); 082 _handledContents.clear(); 083 return populatedContents; 084 } 085 086 @Override 087 protected List<ModifiableDefaultContent> _internalPopulate(Logger logger) 088 { 089 return _importOrSynchronizeContents(new HashMap<>(), false, logger); 090 } 091 092 /** 093 * Adds the given content as handled (i.e. will not be removed if _removalSync is true) 094 * @param id The id of the content 095 */ 096 protected void _handleContent(String id) 097 { 098 _handledContents.add(id); 099 } 100 101 /** 102 * Returns true if the given content is handled 103 * @param id The content to test 104 * @return true if the given content is handled 105 */ 106 protected boolean _isHandled(String id) 107 { 108 return _handledContents.contains(id); 109 } 110 111 /** 112 * Imports or synchronizes a content for each available language 113 * @param idValue The unique identifier of the content 114 * @param remoteValues The remote values 115 * @param forceImport To force import and ignoring the synchronize existing contents only option 116 * @param logger The logger 117 * @return The list of synchronized or imported contents 118 */ 119 protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger) 120 { 121 List<ModifiableDefaultContent> contents = new ArrayList<>(); 122 123 for (String lang : getLanguages()) 124 { 125 contents.addAll(_importOrSynchronizeContent(idValue, lang, remoteValues, forceImport, logger)); 126 } 127 128 return contents; 129 } 130 131 /** 132 * Imports or synchronizes a content for a given language 133 * @param idValue The unique identifier of the content 134 * @param lang The language of content to import or synchronize 135 * @param remoteValues The remote values 136 * @param forceImport To force import and ignoring the synchronize existing contents only option 137 * @param logger The logger 138 * @return The list of imported and synchronized contents 139 */ 140 protected List<ModifiableDefaultContent> _importOrSynchronizeContent(String idValue, String lang, Map<String, List<Object>> remoteValues, boolean forceImport, Logger logger) 141 { 142 List<ModifiableDefaultContent> contents = new ArrayList<>(); 143 144 try 145 { 146 ModifiableDefaultContent content = getContent(lang, idValue); 147 if (content != null) 148 { 149 contents.add(_synchronizeContent(content, remoteValues, logger)); 150 } 151 else if (forceImport || !synchronizeExistingContentsOnly()) 152 { 153 contents.add(_importContent(idValue, null, lang, remoteValues, logger)); 154 } 155 } 156 catch (Exception e) 157 { 158 _nbError++; 159 logger.error("An error occurred while importing or synchronizing content", e); 160 } 161 162 return contents; 163 } 164 165 @Override 166 public void synchronizeContent(ModifiableDefaultContent content, Logger logger) throws Exception 167 { 168 String idValue = content.getMetadataHolder().getString(getIdField()); 169 170 Map<String, Object> parameters = putIdParameter(idValue); 171 Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger); 172 if (!results.isEmpty()) 173 { 174 try 175 { 176 _synchronizeContent(content, results.get(idValue), logger); 177 } 178 catch (Exception e) 179 { 180 _nbError++; 181 logger.error("An error occurred while importing or synchronizing content", e); 182 throw e; 183 } 184 } 185 } 186 187 /** 188 * Synchronize a content with remove values. 189 * @param content The content to synchronize 190 * @param remoteValues Values to synchronize 191 * @param logger The logger 192 * @return The synchronized content 193 * @throws Exception if an error occurs. 194 */ 195 protected ModifiableDefaultContent _synchronizeContent(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) throws Exception 196 { 197 long startTime = System.currentTimeMillis(); 198 199 String contentTitle = content.getTitle(); 200 String lang = content.getLanguage(); 201 202 // Update content 203 logger.info("Start synchronizing content '{}' for language {}", contentTitle, lang); 204 205 _ensureTitleIsPresent(content, remoteValues, logger); 206 207 boolean hasChanges = _fillContent(remoteValues, content, false, logger); 208 hasChanges = additionalSynchronizeOperations(content, remoteValues, logger) || hasChanges; 209 210 if (hasChanges) 211 { 212 boolean success = applyChanges(content, logger); 213 if (success) 214 { 215 _nbSynchronizedContents++; 216 logger.info("Some changes were detected for content '{}' and language {}", contentTitle, lang); 217 } 218 } 219 else 220 { 221 _nbNotChangedContents++; 222 logger.info("No changes detected for content '{}' and language {}", contentTitle, lang); 223 } 224 225 // Do additional operation on the content 226 SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator()); 227 if (synchronizingContentOperator != null) 228 { 229 synchronizingContentOperator.additionalOperation(content, remoteValues, logger); 230 } 231 else 232 { 233 logger.warn("Cannot find synchronizing content operator with id '{}'. No additional operation has been done.", getSynchronizingContentOperator()); 234 } 235 236 long endTime = System.currentTimeMillis(); 237 logger.info("End synchronization of content '{}' for language {} in {} ms", contentTitle, lang, endTime - startTime); 238 239 return content; 240 } 241 242 @Override 243 public List<ModifiableDefaultContent> importContent(String idValue, Map<String, Object> importParams, Logger logger) throws Exception 244 { 245 List<ModifiableDefaultContent> createdContents = new ArrayList<>(); 246 247 Map<String, Object> parameters = putIdParameter(idValue); 248 Map<String, Map<String, List<Object>>> results = getTransformedRemoteValues(parameters, logger); 249 if (!results.isEmpty()) 250 { 251 for (String lang : getLanguages()) 252 { 253 try 254 { 255 createdContents.add(_importContent(idValue, importParams, lang, results.get(idValue), logger)); 256 } 257 catch (Exception e) 258 { 259 _nbError++; 260 logger.error("An error occurred while importing or synchronizing content", e); 261 } 262 } 263 } 264 265 return createdContents; 266 } 267 268 /** 269 * Set search parameters for the ID value. 270 * @param idValue Value to search 271 * @return Map with the search parameters 272 */ 273 protected abstract Map<String, Object> putIdParameter(String idValue); 274 275 /** 276 * Import a content from remote values. 277 * @param idValue Id (for import/synchronization) of the content to import 278 * @param importParams Specific parameters for import 279 * @param lang Lang of the content 280 * @param remoteValues Values of the content 281 * @param logger The logger 282 * @return The content created by import, or null 283 * @throws Exception if an error occurs. 284 */ 285 protected ModifiableDefaultContent _importContent(String idValue, Map<String, Object> importParams, String lang, Map<String, List<Object>> remoteValues, Logger logger) throws Exception 286 { 287 ModifiableDefaultContent content = getContent(lang, idValue); 288 if (content != null) 289 { 290 logger.warn("The content '{}' for language {} already exists and cannot be imported ", idValue, lang); 291 } 292 else 293 { 294 long startTime = System.currentTimeMillis(); 295 296 // Calculate contentTitle 297 String contentTitle = idValue; 298 if (remoteValues.containsKey("title")) 299 { 300 List<Object> remoteTitles = remoteValues.get("title"); 301 contentTitle = (String) remoteTitles.stream().filter(obj -> obj instanceof String && StringUtils.isNotEmpty((String) obj)).findFirst().orElse(idValue); 302 } 303 304 // Create new content 305 logger.info("Start importing content '{}' for language {}", contentTitle, lang); 306 307 content = createContentAction(lang, contentTitle, logger); 308 if (content != null) 309 { 310 // Synchronize content metadata 311 _fillContent(remoteValues, content, true, logger); 312 updateSCCProperty(content); 313 314 additionalImportOperations(content, remoteValues, importParams, logger); 315 316 content.saveChanges(); 317 content.checkpoint(); 318 319 // Validate content if allowed 320 validateContent(content, logger); 321 322 _nbCreatedContents++; 323 324 // Do additional operation on the content 325 SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator()); 326 synchronizingContentOperator.additionalOperation(content, remoteValues, logger); 327 328 // Notify a content was imported 329 Map<String, Object> eventParams = new HashMap<>(); 330 eventParams.put(ObservationConstants.ARGS_CONTENT, content); 331 eventParams.put(ObservationConstants.ARGS_CONTENT_ID, content.getId()); 332 _observationManager.notify(new Event(org.ametys.plugins.contentio.synchronize.observation.ObservationConstants.EVENT_CONTENT_SYNCHRONIZED, _currentUserProvider.getUser(), eventParams)); 333 334 long endTime = System.currentTimeMillis(); 335 logger.info("End import of content '{}' for language {} in {} ms", content.getId(), lang, endTime - startTime); 336 } 337 } 338 339 return content; 340 } 341 342 /** 343 * Add specific fields to the content. 344 * @param content Content to update 345 * @param remoteValues Values of the content 346 * @param importParams Import parameters 347 * @param logger The logger 348 * @return <code>true</code> if there are changes 349 */ 350 protected boolean additionalImportOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger) 351 { 352 return additionalCommonOperations(content, remoteValues, importParams, logger); 353 } 354 355 /** 356 * Add specific fields to the content. 357 * @param content Content to update 358 * @param remoteValues Values of the content 359 * @param logger The logger 360 * @return <code>true</code> if there are changes 361 */ 362 protected boolean additionalSynchronizeOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Logger logger) 363 { 364 return additionalCommonOperations(content, remoteValues, null, logger); 365 } 366 367 /** 368 * Add specific fields to the content during import or synchronization. 369 * @param content Content to update 370 * @param remoteValues Values of the content 371 * @param importParams the import params 372 * @param logger The logger 373 * @return <code>true</code> if there are changes 374 */ 375 protected boolean additionalCommonOperations(ModifiableDefaultContent content, Map<String, List<Object>> remoteValues, Map<String, Object> importParams, Logger logger) 376 { 377 // Do nothing by default 378 return false; 379 } 380 381 private void _ensureTitleIsPresent(Content content, Map<String, List<Object>> remoteValues, Logger logger) 382 { 383 if (remoteValues.containsKey("title")) 384 { 385 List<Object> titleValues = remoteValues.get("title"); 386 boolean atLeastOneTitle = titleValues.stream() 387 .filter(String.class::isInstance) 388 .map(String.class::cast) 389 .anyMatch(StringUtils::isNotBlank); 390 if (atLeastOneTitle) 391 { 392 return; 393 } 394 } 395 396 // Force to current title 397 logger.warn("The remote value of 'title' is empty for the content {}. The 'title' metadata is mandatory, the current title will remain.", content); 398 remoteValues.put("title", Collections.singletonList(content.getTitle())); 399 } 400 401 @Override 402 public ModifiableDefaultContent getContent(String lang, String idValue) 403 { 404 String query = _getContentPathQuery(lang, idValue, getContentType()); 405 AmetysObjectIterable<ModifiableDefaultContent> contents = _resolver.query(query); 406 407 if (contents.getSize() > 0) 408 { 409 return contents.iterator().next(); 410 } 411 return null; 412 } 413 414 /** 415 * Creates content action with result from request 416 * @param lang The language 417 * @param contentTitle The content title 418 * @param logger The logger 419 * @return The content id, or null of a workflow error occured 420 */ 421 protected ModifiableDefaultContent createContentAction(String lang, String contentTitle, Logger logger) 422 { 423 return createContentAction(getContentType(), getWorkflowName(), getInitialActionId(), lang, contentTitle, logger); 424 } 425 426 /** 427 * Fill the content with remote values. 428 * @param remoteValues The remote values 429 * @param content The content to synchronize 430 * @param create <code>true</code> if content is creating, false if it is updated 431 * @param logger The logger 432 * @return <code>true</code> if changes were made 433 */ 434 protected boolean _fillContent(Map<String, List<Object>> remoteValues, ModifiableDefaultContent content, boolean create, Logger logger) 435 { 436 boolean hasChanges = false; 437 438 if (content.isLocked()) 439 { 440 logger.warn("The content '{}' ({}) is currently locked by user {}: it cannot be synchronized", content.getTitle(), content.getId(), content.getLockOwner()); 441 } 442 else 443 { 444 Map<String, Object> params = ImmutableMap.of("contentType", getContentType()); 445 ContentType contentType = _contentTypeEP.getExtension(getContentType()); 446 for (String metadataPath : remoteValues.keySet()) 447 { 448 hasChanges = _synchronizeMetadata(content, contentType, metadataPath, metadataPath, remoteValues.get(metadataPath), getLocalAndExternalFields(params).contains(metadataPath), create, logger) || hasChanges; 449 } 450 } 451 452 return hasChanges; 453 } 454 455 /** 456 * Validates a content after import 457 * @param content The content to validate 458 * @param logger The logger 459 */ 460 protected void validateContent(WorkflowAwareContent content, Logger logger) 461 { 462 if (validateAfterImport()) 463 { 464 validateContent(content, getValidateActionId(), logger); 465 } 466 } 467 468 /** 469 * Get the value of metadata holding the unique identifier of the synchronized content 470 * @param content The content 471 * @return The value 472 */ 473 protected String _getIdFieldValue(DefaultContent content) 474 { 475 ModifiableCompositeMetadata metadataHolder = _getMetadataHolder(content.getMetadataHolder(), getIdField()); 476 477 String[] pathSegments = getIdField().split("/"); 478 String metadataName = pathSegments[pathSegments.length - 1]; 479 480 return metadataHolder.getString(metadataName, null); 481 } 482 483 @Override 484 public Map<String, Map<String, Object>> search(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger) 485 { 486 // Search 487 Map<String, Map<String, Object>> results = internalSearch(_removeEmptyParameters(parameters), offset, limit, sort, logger); 488 489 return results; 490 } 491 492 /** 493 * Search values and return the result without any treatment. 494 * @param parameters Search parameters to restrict the search 495 * @param offset Begin of the search 496 * @param limit Number of results 497 * @param sort Sort of results (ignored for LDAP results) 498 * @param logger The logger 499 * @return Map of results without any treatment. 500 */ 501 protected abstract Map<String, Map<String, Object>> internalSearch(Map<String, Object> parameters, int offset, int limit, List<Object> sort, Logger logger); 502 503 /** 504 * Search values and return the result organized by metadata and transformed by the {@link SynchronizingContentOperator} if exists. 505 * @param parameters Search parameters to restrict the search 506 * @param logger The logger 507 * @return Map of results organized by metadata. 508 */ 509 protected Map<String, Map<String, List<Object>>> getTransformedRemoteValues(Map<String, Object> parameters, Logger logger) 510 { 511 Map<String, Map<String, List<Object>>> remoteValues = getRemoteValues(parameters, logger); 512 513 SynchronizingContentOperator synchronizingContentOperator = _synchronizingContentOperatorEP.getExtension(getSynchronizingContentOperator()); 514 if (synchronizingContentOperator != null) 515 { 516 Map<String, Map<String, List<Object>>> transformedRemoteValues = new LinkedHashMap<>(); 517 for (String key : remoteValues.keySet()) 518 { 519 transformedRemoteValues.put(key, synchronizingContentOperator.transform(remoteValues.get(key), logger)); 520 } 521 522 return transformedRemoteValues; 523 } 524 else 525 { 526 logger.warn("Cannot find synchronizing content operator with id '{}'. No transformation has applied on remote values", getSynchronizingContentOperator()); 527 return remoteValues; // no transformation 528 } 529 } 530 531 /** 532 * Search values and return the result organized by metadata 533 * @param parameters Search parameters to restrict the search 534 * @param logger The logger 535 * @return Map of results organized by metadata. 536 */ 537 protected abstract Map<String, Map<String, List<Object>>> getRemoteValues(Map<String, Object> parameters, Logger logger); 538 539 @Override 540 public void updateSyncInformations(ModifiableDefaultContent content, String syncCode, Logger logger) throws Exception 541 { 542 updateSCCProperty(content); 543 content.getMetadataHolder().setMetadata(getIdField(), syncCode); 544 content.saveChanges(); 545 content.checkpoint(); 546 } 547 548 @Override 549 public int getTotalCount(Map<String, Object> parameters, Logger logger) 550 { 551 return search(parameters, 0, Integer.MAX_VALUE, null, logger).size(); 552 } 553 554 /** 555 * Import or synchronize several contents from search params. 556 * @param searchParams Search parameters 557 * @param forceImport To force import and ignoring the synchronize existing contents only option 558 * @param logger The logger 559 * @return The {@link List} of imported or synchronized {@link ModifiableDefaultContent} 560 */ 561 protected List<ModifiableDefaultContent> _importOrSynchronizeContents(Map<String, Object> searchParams, boolean forceImport, Logger logger) 562 { 563 List<ModifiableDefaultContent> contents = new ArrayList<>(); 564 565 Map<String, Map<String, List<Object>>> remoteValuesByContent = getTransformedRemoteValues(searchParams, logger); 566 for (String idValue : remoteValuesByContent.keySet()) 567 { 568 Map<String, List<Object>> remoteValues = remoteValuesByContent.get(idValue); 569 _handleContent(idValue); 570 contents.addAll(_importOrSynchronizeContent(idValue, remoteValues, forceImport, logger)); 571 } 572 573 return contents; 574 } 575 576 @Override 577 protected List<Content> _getContentsToRemove(AmetysObjectIterable<ModifiableDefaultContent> contents) 578 { 579 return contents.stream() 580 .filter(content -> !_isHandled(_getIdFieldValue(content))) 581 .collect(Collectors.toList()); 582 } 583}