001/* 002 * Copyright 2020 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.core.migration; 017 018import java.util.ArrayList; 019import java.util.Collections; 020import java.util.Comparator; 021import java.util.HashMap; 022import java.util.List; 023import java.util.Map; 024import java.util.Map.Entry; 025import java.util.Set; 026import java.util.stream.Collectors; 027 028import org.apache.avalon.framework.component.Component; 029import org.apache.avalon.framework.configuration.Configuration; 030import org.apache.avalon.framework.configuration.ConfigurationException; 031import org.apache.avalon.framework.service.ServiceException; 032import org.apache.avalon.framework.service.ServiceManager; 033import org.apache.avalon.framework.service.Serviceable; 034import org.apache.avalon.framework.thread.ThreadSafe; 035import org.apache.commons.lang3.StringUtils; 036import org.apache.commons.lang3.tuple.Pair; 037 038import org.ametys.core.ObservationConstants; 039import org.ametys.core.migration.action.Action; 040import org.ametys.core.migration.action.ActionExtensionPoint; 041import org.ametys.core.migration.action.data.ActionData; 042import org.ametys.core.migration.configuration.VersionConfiguration; 043import org.ametys.core.migration.handler.VersionHandler; 044import org.ametys.core.migration.handler.VersionHandlerExtensionPoint; 045import org.ametys.core.migration.version.Version; 046import org.ametys.core.observation.Event; 047import org.ametys.core.observation.ObservationManager; 048import org.ametys.runtime.plugin.ExtensionPoint; 049import org.ametys.runtime.plugin.component.AbstractLogEnabled; 050 051/** 052 * Migration Extension Point that will list all migration needed by the current state of the application 053 */ 054public class MigrationExtensionPoint extends AbstractLogEnabled implements ExtensionPoint<MigrationConfiguration>, ThreadSafe, Component, Serviceable 055{ 056 /** Avalon Role */ 057 public static final String ROLE = MigrationExtensionPoint.class.getName(); 058 059 private Map<String, MigrationConfiguration> _configurations = new HashMap<>(); 060 061 private Map<String, String> _expectedVersionsForExtensions = new HashMap<>(); 062 063 private VersionHandlerExtensionPoint _versionHandlerEP; 064 065 private ActionExtensionPoint _upgradeEP; 066 private ActionExtensionPoint _initializationEP; 067 068 private ObservationManager _observationManager; 069 070 public void service(ServiceManager smanager) throws ServiceException 071 { 072 _versionHandlerEP = (VersionHandlerExtensionPoint) smanager.lookup(VersionHandlerExtensionPoint.ROLE); 073 _upgradeEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_UPGRADE); 074 _initializationEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_INITIALIZATION); 075 076 if (smanager.hasService(ObservationManager.ROLE)) 077 { 078 _observationManager = (ObservationManager) smanager.lookup(ObservationManager.ROLE); 079 } 080 } 081 082 public void addExtension(String id, String pluginName, String featureName, Configuration configuration) throws ConfigurationException 083 { 084 _configurations.put(id, new MigrationConfiguration(id, pluginName, featureName, configuration)); 085 } 086 087 public void initializeExtensions() throws Exception 088 { 089 // Nothing here 090 } 091 092 /** 093 * All migrations are parsed and executed one per one in the right order 094 * @return true if the migration requires a server restart ; false otherwise 095 */ 096 public boolean doMigration() 097 { 098 try 099 { 100 getLogger().info("Automatic migration start."); 101 102 List<ActionData> allUpgradeActions = new ArrayList<>(); 103 Map<Version, List<ActionData>> allInitActions = new HashMap<>(); 104 boolean versionDeterminationFailed = false; 105 Set<String> keySet = _configurations.keySet(); 106 for (String id : keySet) 107 { 108 MigrationConfiguration migrationConfiguration = _configurations.get(id); 109 110 try 111 { 112 String expectedVersion = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration); 113 _expectedVersionsForExtensions.put(id, expectedVersion); 114 } 115 catch (ConfigurationException e) 116 { 117 throw new MigrationException("Impossible to read the upgrade configuration for component '" + id + "'", e); 118 } 119 120 try 121 { 122 Configuration configuration = migrationConfiguration.getConfiguration(); 123 124 List<Version> versions = new ArrayList<>(); 125 for (Configuration versionConfiguration : configuration.getChild("versions").getChildren("version")) 126 { 127 try 128 { 129 VersionHandler versionHandler = _getVersionHandler(versionConfiguration); 130 VersionConfiguration versionConfigurationObject = versionHandler.getConfiguration(id, versionConfiguration); 131 versions.addAll(versionHandler.getCurrentVersions(id, versionConfigurationObject)); 132 } 133 catch (NotMigrableInSafeModeException e) 134 { 135 getLogger().warn("The migration '{}' cannot be done in safe mode", id, e); 136 } 137 catch (MigrationException e) 138 { 139 versionDeterminationFailed = true; 140 getLogger().error("Error during version determination for component {}", id, e); 141 } 142 } 143 144 Map<Version, List<ActionData>> initActions = _getInitializationActions(versions, migrationConfiguration); 145 allInitActions.putAll(initActions); 146 147 List<ActionData> upgradeActions = _getUpgradeActions(versions, migrationConfiguration); 148 allUpgradeActions.addAll(upgradeActions); 149 } 150 catch (ConfigurationException | MigrationException e) 151 { 152 throw new MigrationException("Exception occured in migration " + id, e); 153 } 154 } 155 156 if (_applyInitializationActions(allInitActions) 157 || _applyUpgradeActions(allUpgradeActions)) 158 { 159 getLogger().info("Automatic migration is restarting the server to continue migration"); 160 return true; 161 } 162 163 if (!versionDeterminationFailed) 164 { 165 _notifyEndOfMigration(); 166 167 // To free some memory space, not done on failing 168 _configurations = null; 169 } 170 171 getLogger().info("Automatic migration finished."); 172 } 173 catch (MigrationException e) 174 { 175 getLogger().error("Error during the automatic migration", e); 176 } 177 178 return false; 179 } 180 181 private VersionHandler _getVersionHandler(Configuration versionConfiguration) throws ConfigurationException 182 { 183 String versionHandlerType = versionConfiguration.getAttribute("type"); 184 185 VersionHandler extension = _versionHandlerEP.getExtension(versionHandlerType); 186 if (extension == null) 187 { 188 throw new ConfigurationException("A migration requests a versionHandler with id '" + versionHandlerType + "', which does not exists, this migration can not be done.", versionConfiguration); 189 } 190 return extension; 191 } 192 193 /** 194 * Notify with an {@link Event} the end of the migration. 195 */ 196 protected void _notifyEndOfMigration() 197 { 198 if (_observationManager != null) 199 { 200 _observationManager.notify(new Event(ObservationConstants.EVENT_MIGRATION_ENDED, null, Map.of())); 201 } 202 } 203 204 /** 205 * Execute all needed upgrades 206 * @param allInitializationActions list of needed upgrades 207 * @return true if an action requires a restart 208 * @throws MigrationException Something went wrong 209 */ 210 protected boolean _applyInitializationActions(Map<Version, List<ActionData>> allInitializationActions) throws MigrationException 211 { 212 if (allInitializationActions.isEmpty()) 213 { 214 getLogger().debug("No initialization to do"); 215 } 216 else 217 { 218 for (Entry<Version, List<ActionData>> entry : allInitializationActions.entrySet()) 219 { 220 Version version = entry.getKey(); 221 List<ActionData> actions = entry.getValue(); 222 223 for (ActionData initializationAction : actions) 224 { 225 Action actionExtension = _initializationEP.getExtension(initializationAction.getType()); 226 getLogger().info("Run initialization : " + initializationAction.toString()); 227 actionExtension.doAction(initializationAction); 228 229 if (initializationAction.requiresRestart()) 230 { 231 return true; 232 } 233 } 234 235 // The ID of the version had already be setted to the most recent action id 236 _upgradeVersion(version); 237 } 238 } 239 return false; 240 } 241 /** 242 * Execute all needed upgrades 243 * @param allActions list of needed upgrades 244 * @return true if an action requires a restart 245 * @throws MigrationException Something went wrong 246 */ 247 protected boolean _applyUpgradeActions(List<ActionData> allActions) throws MigrationException 248 { 249 allActions.sort(Comparator.comparing(ActionData::getVersionNumber)); 250 if (allActions.isEmpty()) 251 { 252 getLogger().debug("No upgrade to do"); 253 } 254 else 255 { 256 for (ActionData action : allActions) 257 { 258 Action upgradeExtension = _upgradeEP.getExtension(action.getType()); 259 getLogger().info("Run upgrade : " + action.toString()); 260 upgradeExtension.doAction(action); 261 _upgradeVersion(action); 262 263 if (action.requiresRestart()) 264 { 265 return true; 266 } 267 } 268 } 269 return false; 270 } 271 272 /** 273 * When an upgrade (or an initialization) have been done, we write the new version 274 * @param action upgrade/initialization executed 275 * @throws MigrationException Something went wrong 276 */ 277 protected void _upgradeVersion(ActionData action) throws MigrationException 278 { 279 Version newVersion = action.getVersion().copyFromActionData(action); 280 _upgradeVersion(newVersion); 281 } 282 283 /** 284 * When an upgrade (or an initialization) have been done, we write the new version 285 * @param newVersion upgrade/initialization executed 286 * @throws MigrationException Something went wrong 287 */ 288 protected void _upgradeVersion(Version newVersion) throws MigrationException 289 { 290 getLogger().debug("Update version info vor version : " + newVersion.toString()); 291 String versionHandlerId = newVersion.getVersionHandlerId(); 292 VersionHandler versionHandler = _versionHandlerEP.getExtension(versionHandlerId); 293 if (versionHandler == null) 294 { 295 throw new MigrationException("Version '" + newVersion.toString() + "' requires a versionHandler that does not exists: '" + versionHandlerId + "'."); 296 } 297 else 298 { 299 versionHandler.addVersion(newVersion); 300 } 301 } 302 303 /** 304 * Analyze the configuration to get a list of upgrades needed based on the current Versions 305 * @param currentVersions list of current versions 306 * @param migrationConfiguration configuration for this extension 307 * @return a list of actions to work on, or null if an error occured 308 * @throws MigrationException impossible to parse the configuration 309 */ 310 protected List<ActionData> _getUpgradeActions(List<Version> currentVersions, MigrationConfiguration migrationConfiguration) throws MigrationException 311 { 312 String componentId = migrationConfiguration.getId(); 313 try 314 { 315 Configuration[] upgradeList = migrationConfiguration.getConfiguration().getChild("upgrades").getChildren("upgrade"); 316 317 List<ActionData> result = new ArrayList<>(); 318 319 for (Version version : currentVersions) 320 { 321 if (version == null) 322 { 323 throw new MigrationException("The migration for '" + componentId + "' got a null Version."); 324 } 325 else if (version.getVersionNumber() != null) 326 { 327 getLogger().debug("Check the upgrades needed for version: {}", version.toString()); 328 String versionNumber = version.getVersionNumber(); 329 330 String higherCurrentUpgradeVersionNumber = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration); 331 if (higherCurrentUpgradeVersionNumber.compareTo(versionNumber) < 0) 332 { 333 getLogger().warn("There is a version stored more recent that any version available in conf for version: {}", version.toString()); 334 } 335 336 List<ActionData> thisVersionUpgrades = new ArrayList<>(); 337 for (Configuration upgradeConf : upgradeList) 338 { 339 String upgradeVersionNumber = upgradeConf.getAttribute("versionNumber"); 340 String type = upgradeConf.getAttribute("type"); 341 String from = upgradeConf.getAttribute("from", null); 342 String comment = upgradeConf.getAttribute("comment", null); 343 boolean restartRequired = upgradeConf.getAttributeAsBoolean("restart", false); 344 345 if (comment == null) 346 { 347 comment = "Automatic Upgrade."; 348 } 349 else 350 { 351 comment = "Automatic Upgrade: " + comment; 352 } 353 354 Action actionExtension = _upgradeEP.getExtension(type); 355 356 if (actionExtension == null) 357 { 358 throw new ConfigurationException("The type '" + type + "' does not exist.", upgradeConf); 359 } 360 361 if (versionNumber.compareTo(upgradeVersionNumber) >= 0 // Obsolete migrations (migrating to older or current version) 362 || from != null && versionNumber.compareTo(from) > 0) // Future migration migrating from an older version 363 { 364 continue; 365 } 366 else 367 { 368 ActionData upgradeData = actionExtension.generateActionData(upgradeVersionNumber, version, comment, from, type, migrationConfiguration.getPluginName(), upgradeConf, restartRequired); 369 thisVersionUpgrades.add(upgradeData); 370 } 371 } 372 373 result.addAll(_removeDuplicatedActions(thisVersionUpgrades)); 374 } 375 } 376 377 return result; 378 } 379 catch (ConfigurationException e) 380 { 381 throw new MigrationException("Error while parsing configuration for component '" + componentId + "'", e); 382 } 383 } 384 385 /** 386 * Analyze the configuration to get a list of initializations needed based on the current Versions 387 * @param currentVersions list of current versions 388 * @param migrationConfiguration configuration for this extension 389 * @return a list of actions to work on, or null if an error occured 390 * @throws MigrationException impossible to parse the configuration 391 */ 392 protected Map<Version, List<ActionData>> _getInitializationActions(List<Version> currentVersions, MigrationConfiguration migrationConfiguration) throws MigrationException 393 { 394 String componentId = migrationConfiguration.getId(); 395 Map<Version, List<ActionData>> result = new HashMap<>(); 396 try 397 { 398 Configuration[] initializations = migrationConfiguration.getConfiguration().getChild("initializations").getChildren("initialization"); 399 400 String versionComment = migrationConfiguration.getConfiguration().getChild("initializations").getAttribute("comment", null); 401 402 if (versionComment == null) 403 { 404 versionComment = "Automatic Initialization."; 405 } 406 else 407 { 408 versionComment = "Automatic Initialization: " + versionComment; 409 } 410 411 for (Version version : currentVersions) 412 { 413 if (version == null) 414 { 415 throw new MigrationException("The migration for '" + componentId + "' got a null Version."); 416 } 417 else if (version.getVersionNumber() == null) 418 { 419 List<ActionData> thisVersionActions = new ArrayList<>(); 420 getLogger().debug("No version number, this is an initialization: {}", version.toString()); 421 422 String higherCurrentUpgradeVersionNumber = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration); 423 version.setVersionNumber(higherCurrentUpgradeVersionNumber); 424 version.setComment(versionComment); 425 426 for (Configuration initialization : initializations) 427 { 428 String type = initialization.getAttribute("type"); 429 boolean restartRequired = initialization.getAttributeAsBoolean("restart", false); 430 String actionComment = null; // No comment will be used here, only the version will have a comment in an initialization 431 432 Action actionExtension = _initializationEP.getExtension(type); 433 434 if (actionExtension == null) 435 { 436 throw new ConfigurationException("The type '" + type + "' does not exist.", initialization); 437 } 438 ActionData initializationData = actionExtension.generateActionData(higherCurrentUpgradeVersionNumber, version, actionComment, null, type, migrationConfiguration.getPluginName(), initialization, restartRequired); 439 thisVersionActions.add(initializationData); 440 } 441 442 result.put(version, thisVersionActions); 443 } 444 445 } 446 447 return result; 448 } 449 catch (ConfigurationException e) 450 { 451 throw new MigrationException("Error while parsing configuration for component '" + componentId + "'", e); 452 } 453 } 454 455 456 /** 457 * Returns the most recent version to apply (after an initialization) 458 * @param migrationConfiguration Configuration of the extension 459 * @return the id of the most current available version, or "0" if none available 460 * @throws ConfigurationException Sompthing wrong while reading the configuration 461 */ 462 protected String _getHigherCurrentUpgradeVersionNumber(MigrationConfiguration migrationConfiguration) throws ConfigurationException 463 { 464 Configuration[] upgradeList = migrationConfiguration.getConfiguration().getChild("upgrades").getChildren("upgrade"); 465 List<String> upgradeIds = new ArrayList<>(); 466 for (Configuration upgradeConf : upgradeList) 467 { 468 String id = upgradeConf.getAttribute("versionNumber"); 469 upgradeIds.add(id); 470 } 471 472 if (!upgradeIds.isEmpty()) 473 { 474 upgradeIds.sort(String.CASE_INSENSITIVE_ORDER); 475 return upgradeIds.get(upgradeIds.size() - 1); 476 } 477 else 478 { 479 return "0"; 480 } 481 } 482 483 /** 484 * Parse the list of upgrade to remove the group-migration 485 * @param actions list of upgrades to clean 486 * @return a list with only needed upgrades. 487 * @throws MigrationException list of actions incoherent 488 */ 489 protected List<ActionData> _removeDuplicatedActions(List<ActionData> actions) throws MigrationException 490 { 491 // First, sort the upgrades by if to have the right order 492 actions.sort( 493 Comparator.comparing(ActionData::getVersionNumber) 494 .thenComparing( 495 ActionData::getFrom, 496 Comparator.nullsFirst(Comparator.naturalOrder()) 497 ) 498 ); 499 500 // Create a list of pairs for each upgrade containing a "for" attribute to konw from which version to which version we should remove the lines 501 List<Pair<String, String>> fromTo = actions.stream() 502 .filter(u -> StringUtils.isNotBlank(u.getFrom())) 503 .map(u -> Pair.of(u.getFrom(), u.getVersionNumber())) 504 .collect(Collectors.toList()); 505 506 _checkFromUpgrades(actions, fromTo); 507 508 List<ActionData> result = actions; 509 510 // Invert the order so we will start to apply the most recent one, it may override other lines with a From inside 511 Collections.reverse(fromTo); 512 // For each pair, remove the right versions 513 for (Pair<String, String> pair : fromTo) 514 { 515 String from = pair.getLeft(); 516 String to = pair.getRight(); 517 518 result = result.stream() 519 .filter( 520 a -> a.getVersionNumber().compareTo(from) <= 0 521 || to.equals(a.getVersionNumber()) && from.equals(a.getFrom()) 522 || a.getVersionNumber().compareTo(to) > 0 523 ) 524 .collect(Collectors.toList()); 525 } 526 527 return result; 528 } 529 530 /** 531 * Tests that in the list of fromTo, that each "fromTo" is linked to an action with the same id and without "from" 532 * @param actions list of actions 533 * @param fromTo list of actions overriding multiple actions 534 * @throws MigrationException there is an overriding version without a "simple" version with the same id 535 */ 536 protected void _checkFromUpgrades(List<ActionData> actions, List<Pair<String, String>> fromTo) throws MigrationException 537 { 538 for (Pair<String, String> pair : fromTo) 539 { 540 String from = pair.getLeft(); 541 String to = pair.getRight(); 542 boolean anyMatch = actions.stream().anyMatch(a -> a.getFrom() == null && to.equals(a.getVersionNumber())); 543 if (!anyMatch) 544 { 545 throw new MigrationException("The action from '" + from + "' to '" + to + "' does not contain a normal upgrade from '" + from + "'"); 546 } 547 } 548 } 549 550 private Map<String, MigrationConfiguration> _getConfigurationsForReading() 551 { 552 if (_configurations == null) 553 { 554 throw new UnsupportedOperationException("Configurations have been emptied from the extension point, this method can't be used for now."); 555 } 556 return _configurations; 557 } 558 559 public boolean hasExtension(String id) 560 { 561 return _getConfigurationsForReading().containsKey(id); 562 } 563 564 public MigrationConfiguration getExtension(String id) 565 { 566 return _getConfigurationsForReading().get(id); 567 } 568 569 public Set<String> getExtensionsIds() 570 { 571 return _getConfigurationsForReading().keySet(); 572 } 573 574 /** 575 * Get the expected version for a component. 576 * This is the latest upgrade number declared 577 * @param id the id of the component 578 * @return the expected version number for each version in this component 579 * @throws MigrationException Component not found in the calculated map of expected versions 580 */ 581 public String getExpectedVersionForComponent(String id) throws MigrationException 582 { 583 if (!_expectedVersionsForExtensions.containsKey(id)) 584 { 585 throw new MigrationException("Component '" + id + "' is not found in the calculated expected versions"); 586 } 587 return _expectedVersionsForExtensions.get(id); 588 } 589}