Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.core.migration;
017
018import java.util.ArrayList;
019import java.util.Collections;
020import java.util.Comparator;
021import java.util.HashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.Map.Entry;
025import java.util.Set;
026import java.util.stream.Collectors;
027
028import org.apache.avalon.framework.component.Component;
029import org.apache.avalon.framework.configuration.Configuration;
030import org.apache.avalon.framework.configuration.ConfigurationException;
031import org.apache.avalon.framework.context.Context;
032import org.apache.avalon.framework.context.ContextException;
033import org.apache.avalon.framework.context.Contextualizable;
034import org.apache.avalon.framework.logger.Logger;
035import org.apache.avalon.framework.service.ServiceException;
036import org.apache.avalon.framework.service.ServiceManager;
037import org.apache.avalon.framework.service.Serviceable;
038import org.apache.avalon.framework.thread.ThreadSafe;
039import org.apache.cocoon.Constants;
040import org.apache.cocoon.util.log.SLF4JLoggerAdapter;
041import org.apache.commons.lang3.StringUtils;
042import org.apache.commons.lang3.tuple.Pair;
043
044import org.ametys.core.ObservationConstants;
045import org.ametys.core.engine.BackgroundEngineHelper;
046import org.ametys.core.migration.action.Action;
047import org.ametys.core.migration.action.ActionExtensionPoint;
048import org.ametys.core.migration.action.data.ActionData;
049import org.ametys.core.migration.configuration.VersionConfiguration;
050import org.ametys.core.migration.handler.VersionHandler;
051import org.ametys.core.migration.handler.VersionHandlerExtensionPoint;
052import org.ametys.core.migration.version.Version;
053import org.ametys.core.observation.Event;
054import org.ametys.core.observation.ObservationManager;
055import org.ametys.runtime.plugin.ExtensionPoint;
056import org.ametys.runtime.plugin.component.AbstractLogEnabled;
057import org.ametys.runtime.servlet.RuntimeServlet;
058import org.ametys.runtime.servlet.RuntimeServlet.MaintenanceStatus;
059
060/**
061 * Migration Extension Point that will list all migration needed by the current state of the application
062 */
063public class MigrationExtensionPoint extends AbstractLogEnabled implements ExtensionPoint<MigrationConfiguration>, ThreadSafe, Component, Serviceable, Contextualizable
064{
065    /** Avalon Role */
066    public static final String ROLE = MigrationExtensionPoint.class.getName();
067    
068    private Map<String, MigrationConfiguration> _configurations = new HashMap<>();
069    
070    private Map<String, String> _expectedVersionsForExtensions = new HashMap<>();
071
072    private VersionHandlerExtensionPoint _versionHandlerEP;
073
074    private ActionExtensionPoint _upgradeEP;
075    private ActionExtensionPoint _initializationEP;
076
077    private ObservationManager _observationManager;
078
079    private ServiceManager _manager;
080
081    private org.apache.cocoon.environment.Context _context;
082    
083    public void service(ServiceManager smanager) throws ServiceException
084    {
085        _manager = smanager;
086        
087        _versionHandlerEP = (VersionHandlerExtensionPoint) smanager.lookup(VersionHandlerExtensionPoint.ROLE);
088        _upgradeEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_UPGRADE);
089        _initializationEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_INITIALIZATION);
090        
091        if (smanager.hasService(ObservationManager.ROLE))
092        {
093            _observationManager = (ObservationManager) smanager.lookup(ObservationManager.ROLE);
094        }
095    }
096    
097    public void contextualize(Context context) throws ContextException
098    {
099        _context = (org.apache.cocoon.environment.Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
100    }
101    
102    public void addExtension(String id, String pluginName, String featureName, Configuration configuration) throws ConfigurationException
103    {
104        _configurations.put(id, new MigrationConfiguration(id, pluginName, featureName, configuration));
105    }
106
107    public void initializeExtensions() throws Exception
108    {
109        // Nothing here
110    }
111    
112    /**
113     * All migrations are parsed and executed one per one in the right order
114     * @return true if the migration requires a server restart ; false otherwise
115     */
116    public boolean doMigration()
117    {
118        try
119        {
120            getLogger().info("Automatic migration start.");
121
122            // For env
123            Logger logger = new SLF4JLoggerAdapter(getLogger());
124            
125            List<ActionData> allUpgradeActions = new ArrayList<>();
126            Map<Version, List<ActionData>> allInitActions = new HashMap<>();
127            boolean versionDeterminationFailed = false;
128            Set<String> keySet = _configurations.keySet();
129            for (String id : keySet)
130            {
131                MigrationConfiguration migrationConfiguration = _configurations.get(id);
132
133                try
134                {
135                    String expectedVersion = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration);
136                    _expectedVersionsForExtensions.put(id, expectedVersion);
137                }
138                catch (ConfigurationException e)
139                {
140                    throw new MigrationException("Impossible to read the upgrade configuration for component '" + id + "'", e);
141                }
142                
143                try
144                {
145                    Configuration configuration = migrationConfiguration.getConfiguration();
146                    
147                    List<Version> versions = new ArrayList<>();
148                    for (Configuration versionConfiguration : configuration.getChild("versions").getChildren("version"))
149                    {
150                        Map<String, Object> environmentInformation = null;
151                        
152                        try
153                        {
154                            // Create the environment.
155                            environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, logger);
156                            
157                            VersionHandler versionHandler = _getVersionHandler(versionConfiguration);
158                            VersionConfiguration versionConfigurationObject = versionHandler.getConfiguration(id, versionConfiguration);
159                            versions.addAll(versionHandler.getCurrentVersions(id, versionConfigurationObject));
160                        }
161                        catch (NotMigrableInSafeModeException e)
162                        {
163                            getLogger().warn("The migration '{}' cannot be done in safe mode", id, e);
164                        }
165                        catch (MigrationException e)
166                        {
167                            versionDeterminationFailed = true;
168                            getLogger().error("Error during version determination for component {}", id,  e);
169                        }
170                        finally
171                        {
172                            BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
173                        }                            
174                    }
175                    
176                    Map<Version, List<ActionData>> initActions = _getInitializationActions(versions, migrationConfiguration);
177                    allInitActions.putAll(initActions);
178
179                    List<ActionData> upgradeActions = _getUpgradeActions(versions, migrationConfiguration);
180                    allUpgradeActions.addAll(upgradeActions);
181                }
182                catch (ConfigurationException | MigrationException e)
183                {
184                    throw new MigrationException("Exception occured in migration " + id, e);
185                }
186            }
187
188            if (_applyInitializationActions(allInitActions, logger)
189                || _applyUpgradeActions(allUpgradeActions, logger))
190            {
191                getLogger().info("Automatic migration is restarting the server to continue migration");
192                return true;
193            }
194            
195            if (!versionDeterminationFailed)
196            {
197                
198                _notifyEndOfMigration();
199                
200                // To free some memory space, not done on failing
201                _configurations = null;
202            }
203            else
204            {
205                RuntimeServlet.setMaintenanceStatus(MaintenanceStatus.AUTOMATIC, null);
206            }
207            
208            getLogger().info("Automatic migration finished.");
209        }
210        catch (MigrationException e)
211        {
212            RuntimeServlet.setMaintenanceStatus(MaintenanceStatus.AUTOMATIC, null);
213            
214            getLogger().error("Error during the automatic migration", e);
215        }
216        
217        return false;
218    }
219    
220    private VersionHandler _getVersionHandler(Configuration versionConfiguration) throws ConfigurationException
221    {
222        String versionHandlerType = versionConfiguration.getAttribute("type");
223        
224        VersionHandler extension = _versionHandlerEP.getExtension(versionHandlerType);
225        if (extension == null)
226        {
227            throw new ConfigurationException("A migration requests a versionHandler with id '" + versionHandlerType + "', which does not exists, this migration can not be done.", versionConfiguration);
228        }
229        return extension;
230    }
231
232    /**
233     * Notify with an {@link Event} the end of the migration.
234     */
235    protected void _notifyEndOfMigration()
236    {
237        if (_observationManager != null)
238        {
239            _observationManager.notify(new Event(ObservationConstants.EVENT_MIGRATION_ENDED, null, Map.of()));
240        }
241    }
242    
243    /**
244     * Execute all needed upgrades
245     * @param allInitializationActions list of needed upgrades
246     * @param logger The avalon wrapped logger
247     * @return true if an action requires a restart
248     * @throws MigrationException Something went wrong
249     */
250    protected boolean _applyInitializationActions(Map<Version, List<ActionData>> allInitializationActions, Logger logger) throws MigrationException
251    {
252        if (allInitializationActions.isEmpty())
253        {
254            getLogger().debug("No initialization to do");
255        }
256        else
257        {
258            for (Entry<Version, List<ActionData>> entry : allInitializationActions.entrySet())
259            {
260                Version version = entry.getKey();
261                List<ActionData> actions = entry.getValue();
262                
263                for (ActionData initializationAction : actions)
264                {
265                    Map<String, Object> environmentInformation = null;
266
267                    try
268                    {
269                        // Create the environment.
270                        environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, logger);
271                        
272                        Action actionExtension = _initializationEP.getExtension(initializationAction.getType());
273                        getLogger().info("Run initialization : " + initializationAction.toString());
274                        actionExtension.doAction(initializationAction);
275                        
276                        if (initializationAction.requiresRestart())
277                        {
278                            return true;
279                        }
280                    }
281                    finally
282                    {
283                        BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
284                    }
285                }
286                
287                // The ID of the version had already be set to the most recent action id
288                _upgradeVersion(version);
289            }
290        }
291        return false;
292    }
293    /**
294     * Execute all needed upgrades
295     * @param allActions list of needed upgrades
296     * @param logger The avalon wrapped logger
297     * @return true if an action requires a restart
298     * @throws MigrationException Something went wrong
299     */
300    protected boolean _applyUpgradeActions(List<ActionData> allActions, Logger logger) throws MigrationException
301    {
302        allActions.sort(Comparator.comparing(ActionData::getVersionNumber));
303        if (allActions.isEmpty())
304        {
305            getLogger().debug("No upgrade to do");
306        }
307        else
308        {
309            _logUpgradeActionsBeforeExecution(allActions);
310            for (ActionData action : allActions)
311            {
312                Map<String, Object> environmentInformation = null;
313
314                try
315                {
316                    // Create the environment.
317                    environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, logger);
318                    
319                    Action upgradeExtension = _upgradeEP.getExtension(action.getType());
320                    getLogger().info("Run upgrade : " + action.toString());
321                    upgradeExtension.doAction(action);
322                    _upgradeVersion(action);
323                    
324                    if (action.requiresRestart())
325                    {
326                        return true;
327                    }
328                }
329                finally
330                {
331                    BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
332                }
333            }
334        }
335        return false;
336    }
337
338    private void _logUpgradeActionsBeforeExecution(List<ActionData> allActions)
339    {
340        if (getLogger().isInfoEnabled())
341        {
342            int count = allActions.size();
343            StringBuilder sb = new StringBuilder().append(count).append(count > 1 ? " pending upgrade(s):" : " pending upgrade:");
344            for (ActionData action : allActions)
345            {
346                sb.append(System.lineSeparator());
347                sb.append("* ").append(action.toString());
348            }
349            getLogger().info(sb.toString());
350        }
351    }
352
353    /**
354     * When an upgrade (or an initialization) have been done, we write the new version
355     * @param action upgrade/initialization executed
356     * @throws MigrationException Something went wrong
357     */
358    protected void _upgradeVersion(ActionData action) throws MigrationException
359    {
360        Version newVersion = action.getVersion().copyFromActionData(action);
361        _upgradeVersion(newVersion);
362    }
363    
364    /**
365     * When an upgrade (or an initialization) have been done, we write the new version
366     * @param newVersion upgrade/initialization executed
367     * @throws MigrationException Something went wrong
368     */
369    protected void _upgradeVersion(Version newVersion) throws MigrationException
370    {
371        getLogger().debug("Update version info vor version : " + newVersion.toString());
372        String versionHandlerId = newVersion.getVersionHandlerId();
373        VersionHandler versionHandler = _versionHandlerEP.getExtension(versionHandlerId);
374        if (versionHandler == null)
375        {
376            throw new MigrationException("Version '" + newVersion.toString() + "' requires a versionHandler that does not exists: '" + versionHandlerId + "'.");
377        }
378        else
379        {
380            versionHandler.addVersion(newVersion);
381        }
382    }
383    
384    /**
385     * Analyze the configuration to get a list of upgrades needed based on the current Versions
386     * @param currentVersions list of current versions
387     * @param migrationConfiguration configuration for this extension
388     * @return a list of actions to work on, or null if an error occured
389     * @throws MigrationException impossible to parse the configuration
390     */
391    protected List<ActionData> _getUpgradeActions(List<Version> currentVersions, MigrationConfiguration migrationConfiguration) throws MigrationException
392    {
393        String componentId = migrationConfiguration.getId();
394        try
395        {
396            Configuration[] upgradeList = migrationConfiguration.getConfiguration().getChild("upgrades").getChildren("upgrade");
397            
398            List<ActionData> result = new ArrayList<>();
399            
400            for (Version version : currentVersions)
401            {
402                if (version == null)
403                {
404                    throw new MigrationException("The migration for '" + componentId + "' got a null Version.");
405                }
406                else if (version.getVersionNumber() != null)
407                {
408                    getLogger().debug("Check the upgrades needed for version: {}", version.toString());
409                    String versionNumber = version.getVersionNumber();
410
411                    String higherCurrentUpgradeVersionNumber = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration);
412                    if (higherCurrentUpgradeVersionNumber.compareTo(versionNumber) < 0)
413                    {
414                        getLogger().warn("There is a version stored more recent that any version available in conf for version: {}", version.toString());
415                    }
416                    
417                    List<ActionData> thisVersionUpgrades = new ArrayList<>();
418                    for (Configuration upgradeConf : upgradeList)
419                    {
420                        String upgradeVersionNumber = upgradeConf.getAttribute("versionNumber");
421                        String type = upgradeConf.getAttribute("type");
422                        String from = upgradeConf.getAttribute("from", null);
423                        String comment = upgradeConf.getAttribute("comment", null);
424                        boolean restartRequired = upgradeConf.getAttributeAsBoolean("restart", false);
425
426                        if (comment == null)
427                        {
428                            comment = "Automatic Upgrade.";
429                        }
430                        else
431                        {
432                            comment = "Automatic Upgrade: " + comment;
433                        }
434                        
435                        Action actionExtension = _upgradeEP.getExtension(type);
436                        
437                        if (actionExtension == null)
438                        {
439                            throw new ConfigurationException("The type '" + type + "' does not exist.", upgradeConf);
440                        }
441                        
442                        if (versionNumber.compareTo(upgradeVersionNumber) >= 0 // Obsolete migrations (migrating to older or current version) 
443                            || from != null && versionNumber.compareTo(from) > 0) // Future migration migrating from an older version
444                        {
445                            continue;
446                        }
447                        else
448                        {
449                            ActionData upgradeData = actionExtension.generateActionData(upgradeVersionNumber, version, comment, from, type, migrationConfiguration.getPluginName(), upgradeConf, restartRequired);
450                            thisVersionUpgrades.add(upgradeData);
451                        }
452                    }
453                    
454                    result.addAll(_removeDuplicatedActions(thisVersionUpgrades));
455                }
456            }
457            
458            return result;
459        }
460        catch (ConfigurationException e)
461        {
462            throw new MigrationException("Error while parsing configuration for component '" + componentId + "'", e);
463        }
464    }
465    
466    /**
467     * Analyze the configuration to get a list of initializations needed based on the current Versions
468     * @param currentVersions list of current versions
469     * @param migrationConfiguration configuration for this extension
470     * @return a list of actions to work on, or null if an error occured
471     * @throws MigrationException impossible to parse the configuration
472     */
473    protected Map<Version, List<ActionData>> _getInitializationActions(List<Version> currentVersions, MigrationConfiguration migrationConfiguration) throws MigrationException
474    {
475        String componentId = migrationConfiguration.getId();
476        Map<Version, List<ActionData>> result = new HashMap<>();
477        try
478        {
479            Configuration[] initializations = migrationConfiguration.getConfiguration().getChild("initializations").getChildren("initialization");
480            
481            String versionComment = migrationConfiguration.getConfiguration().getChild("initializations").getAttribute("comment", null);
482            
483            if (versionComment == null)
484            {
485                versionComment = "Automatic Initialization.";
486            }
487            else
488            {
489                versionComment = "Automatic Initialization: " + versionComment;
490            }
491            
492            for (Version version : currentVersions)
493            {
494                if (version == null)
495                {
496                    throw new MigrationException("The migration for '" + componentId + "' got a null Version.");
497                }
498                else if (version.getVersionNumber() == null)
499                {
500                    List<ActionData> thisVersionActions = new ArrayList<>();
501                    getLogger().debug("No version number, this is an initialization: {}", version.toString());
502                    
503                    String higherCurrentUpgradeVersionNumber = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration);
504                    version.setVersionNumber(higherCurrentUpgradeVersionNumber);
505                    version.setComment(versionComment);
506                    
507                    for (Configuration initialization : initializations)
508                    {
509                        String type = initialization.getAttribute("type");
510                        boolean restartRequired = initialization.getAttributeAsBoolean("restart", false);
511                        String actionComment = null; // No comment will be used here, only the version will have a comment in an initialization
512                        
513                        Action actionExtension = _initializationEP.getExtension(type);
514                        
515                        if (actionExtension == null)
516                        {
517                            throw new ConfigurationException("The type '" + type + "' does not exist.", initialization);
518                        }
519                        ActionData initializationData = actionExtension.generateActionData(higherCurrentUpgradeVersionNumber, version, actionComment, null, type, migrationConfiguration.getPluginName(), initialization, restartRequired);
520                        thisVersionActions.add(initializationData);
521                    }
522
523                    result.put(version, thisVersionActions);
524                }
525                
526            }
527            
528            return result;
529        }
530        catch (ConfigurationException e)
531        {
532            throw new MigrationException("Error while parsing configuration for component '" + componentId + "'", e);
533        }
534    }
535    
536    
537    /**
538     * Returns the most recent version to apply (after an initialization)
539     * @param migrationConfiguration Configuration of the extension
540     * @return the id of the most current available version, or "0" if none available
541     * @throws ConfigurationException Sompthing wrong while reading the configuration
542     */
543    protected String _getHigherCurrentUpgradeVersionNumber(MigrationConfiguration migrationConfiguration) throws ConfigurationException
544    {
545        Configuration[] upgradeList = migrationConfiguration.getConfiguration().getChild("upgrades").getChildren("upgrade");
546        List<String> upgradeIds = new ArrayList<>();
547        for (Configuration upgradeConf : upgradeList)
548        {
549            String id = upgradeConf.getAttribute("versionNumber");
550            upgradeIds.add(id);
551        }
552        
553        if (!upgradeIds.isEmpty())
554        {
555            upgradeIds.sort(String.CASE_INSENSITIVE_ORDER);
556            return upgradeIds.get(upgradeIds.size() - 1);
557        }
558        else
559        {
560            return "0";
561        }
562    }
563    
564    /**
565     * Parse the list of upgrade to remove the group-migration
566     * @param actions list of upgrades to clean
567     * @return a list with only needed upgrades.
568     * @throws MigrationException list of actions incoherent
569     */
570    protected List<ActionData> _removeDuplicatedActions(List<ActionData> actions) throws MigrationException
571    {
572        // First, sort the upgrades by if to have the right order
573        actions.sort(
574            Comparator.comparing(ActionData::getVersionNumber)
575                .thenComparing(
576                    ActionData::getFrom,
577                    Comparator.nullsFirst(Comparator.naturalOrder())
578                )
579        );
580        
581        // Create a list of pairs for each upgrade containing a "for" attribute to konw from which version to which version we should remove the lines
582        List<Pair<String, String>> fromTo = actions.stream()
583                                                    .filter(u -> StringUtils.isNotBlank(u.getFrom()))
584                                                    .map(u -> Pair.of(u.getFrom(), u.getVersionNumber()))
585                                                    .collect(Collectors.toList());
586
587        _checkFromUpgrades(actions, fromTo);
588        
589        List<ActionData> result = actions;
590        
591        // Invert the order so we will start to apply the most recent one, it may override other lines with a From inside
592        Collections.reverse(fromTo);
593        // For each pair, remove the right versions
594        for (Pair<String, String> pair : fromTo)
595        {
596            String from = pair.getLeft();
597            String to = pair.getRight();
598            
599            result = result.stream()
600                        .filter(
601                            a -> a.getVersionNumber().compareTo(from) <= 0 
602                            || to.equals(a.getVersionNumber()) && from.equals(a.getFrom())
603                            || a.getVersionNumber().compareTo(to) > 0
604                        )
605                        .collect(Collectors.toList());
606        }
607        
608        return result;
609    }
610    
611    /**
612     * Tests that in the list of fromTo, that each "fromTo" is linked to an action with the same id and without "from" 
613     * @param actions list of actions
614     * @param fromTo list of actions overriding multiple actions
615     * @throws MigrationException there is an overriding version without a "simple" version with the same id
616     */
617    protected void _checkFromUpgrades(List<ActionData> actions, List<Pair<String, String>> fromTo) throws MigrationException
618    {
619        for (Pair<String, String> pair : fromTo)
620        {
621            String from = pair.getLeft();
622            String to = pair.getRight();
623            boolean anyMatch = actions.stream().anyMatch(a -> a.getFrom() == null && to.equals(a.getVersionNumber()));
624            if (!anyMatch)
625            {
626                throw new MigrationException("The action from '" + from + "' to '" + to + "' does not contain a normal upgrade from '" + from + "'");
627            }
628        }
629    }
630    
631    private Map<String, MigrationConfiguration> _getConfigurationsForReading()
632    {
633        if (_configurations == null)
634        {
635            throw new UnsupportedOperationException("Configurations have been emptied from the extension point, this method can't be used for now.");
636        }
637        return _configurations;
638    }
639    
640    public boolean hasExtension(String id)
641    {
642        return _getConfigurationsForReading().containsKey(id);
643    }
644
645    public MigrationConfiguration getExtension(String id)
646    {
647        return _getConfigurationsForReading().get(id);
648    }
649
650    public Set<String> getExtensionsIds()
651    {
652        return _getConfigurationsForReading().keySet();
653    }
654    
655    /**
656     * Get the expected version for a component.
657     * This is the latest upgrade number declared
658     * @param id the id of the component
659     * @return the expected version number for each version in this component
660     * @throws MigrationException Component not found in the calculated map of expected versions
661     */
662    public String getExpectedVersionForComponent(String id) throws MigrationException
663    {
664        if (!_expectedVersionsForExtensions.containsKey(id))
665        {
666            throw new MigrationException("Component '" + id + "' is not found in the calculated expected versions");
667        }
668        return _expectedVersionsForExtensions.get(id);
669    }
670}