001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.core.migration;
017
018import java.util.ArrayList;
019import java.util.Collections;
020import java.util.Comparator;
021import java.util.HashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.Map.Entry;
025import java.util.Set;
026import java.util.stream.Collectors;
027
028import org.apache.avalon.framework.component.Component;
029import org.apache.avalon.framework.configuration.Configuration;
030import org.apache.avalon.framework.configuration.ConfigurationException;
031import org.apache.avalon.framework.context.Context;
032import org.apache.avalon.framework.context.ContextException;
033import org.apache.avalon.framework.context.Contextualizable;
034import org.apache.avalon.framework.logger.Logger;
035import org.apache.avalon.framework.service.ServiceException;
036import org.apache.avalon.framework.service.ServiceManager;
037import org.apache.avalon.framework.service.Serviceable;
038import org.apache.avalon.framework.thread.ThreadSafe;
039import org.apache.cocoon.Constants;
040import org.apache.cocoon.util.log.SLF4JLoggerAdapter;
041import org.apache.commons.lang3.StringUtils;
042import org.apache.commons.lang3.tuple.Pair;
043
044import org.ametys.core.ObservationConstants;
045import org.ametys.core.engine.BackgroundEngineHelper;
046import org.ametys.core.migration.action.Action;
047import org.ametys.core.migration.action.ActionExtensionPoint;
048import org.ametys.core.migration.action.data.ActionData;
049import org.ametys.core.migration.configuration.VersionConfiguration;
050import org.ametys.core.migration.handler.VersionHandler;
051import org.ametys.core.migration.handler.VersionHandlerExtensionPoint;
052import org.ametys.core.migration.version.Version;
053import org.ametys.core.observation.Event;
054import org.ametys.core.observation.ObservationManager;
055import org.ametys.runtime.plugin.ExtensionPoint;
056import org.ametys.runtime.plugin.component.AbstractLogEnabled;
057
058/**
059 * Migration Extension Point that will list all migration needed by the current state of the application
060 */
061public class MigrationExtensionPoint extends AbstractLogEnabled implements ExtensionPoint<MigrationConfiguration>, ThreadSafe, Component, Serviceable, Contextualizable
062{
063    /** Avalon Role */
064    public static final String ROLE = MigrationExtensionPoint.class.getName();
065    
066    private Map<String, MigrationConfiguration> _configurations = new HashMap<>();
067    
068    private Map<String, String> _expectedVersionsForExtensions = new HashMap<>();
069
070    private VersionHandlerExtensionPoint _versionHandlerEP;
071
072    private ActionExtensionPoint _upgradeEP;
073    private ActionExtensionPoint _initializationEP;
074
075    private ObservationManager _observationManager;
076
077    private ServiceManager _manager;
078
079    private org.apache.cocoon.environment.Context _context;
080    
081    public void service(ServiceManager smanager) throws ServiceException
082    {
083        _manager = smanager;
084        
085        _versionHandlerEP = (VersionHandlerExtensionPoint) smanager.lookup(VersionHandlerExtensionPoint.ROLE);
086        _upgradeEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_UPGRADE);
087        _initializationEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_INITIALIZATION);
088        
089        if (smanager.hasService(ObservationManager.ROLE))
090        {
091            _observationManager = (ObservationManager) smanager.lookup(ObservationManager.ROLE);
092        }
093    }
094    
095    public void contextualize(Context context) throws ContextException
096    {
097        _context = (org.apache.cocoon.environment.Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
098    }
099    
100    public void addExtension(String id, String pluginName, String featureName, Configuration configuration) throws ConfigurationException
101    {
102        _configurations.put(id, new MigrationConfiguration(id, pluginName, featureName, configuration));
103    }
104
105    public void initializeExtensions() throws Exception
106    {
107        // Nothing here
108    }
109    
110    /**
111     * All migrations are parsed and executed one per one in the right order
112     * @return true if the migration requires a server restart ; false otherwise
113     */
114    public boolean doMigration()
115    {
116        try
117        {
118            getLogger().info("Automatic migration start.");
119
120            // For env
121            Logger logger = new SLF4JLoggerAdapter(getLogger());
122            
123            
124            List<ActionData> allUpgradeActions = new ArrayList<>();
125            Map<Version, List<ActionData>> allInitActions = new HashMap<>();
126            boolean versionDeterminationFailed = false;
127            Set<String> keySet = _configurations.keySet();
128            for (String id : keySet)
129            {
130                MigrationConfiguration migrationConfiguration = _configurations.get(id);
131
132                try
133                {
134                    String expectedVersion = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration);
135                    _expectedVersionsForExtensions.put(id, expectedVersion);
136                }
137                catch (ConfigurationException e)
138                {
139                    throw new MigrationException("Impossible to read the upgrade configuration for component '" + id + "'", e);
140                }
141                
142                try
143                {
144                    Configuration configuration = migrationConfiguration.getConfiguration();
145                    
146                    List<Version> versions = new ArrayList<>();
147                    for (Configuration versionConfiguration : configuration.getChild("versions").getChildren("version"))
148                    {
149                        Map<String, Object> environmentInformation = null;
150                        
151                        try
152                        {
153                            // Create the environment.
154                            environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, logger);
155                            
156                            VersionHandler versionHandler = _getVersionHandler(versionConfiguration);
157                            VersionConfiguration versionConfigurationObject = versionHandler.getConfiguration(id, versionConfiguration);
158                            versions.addAll(versionHandler.getCurrentVersions(id, versionConfigurationObject));
159                        }
160                        catch (NotMigrableInSafeModeException e)
161                        {
162                            getLogger().warn("The migration '{}' cannot be done in safe mode", id, e);
163                        }
164                        catch (MigrationException e)
165                        {
166                            versionDeterminationFailed = true;
167                            getLogger().error("Error during version determination for component {}", id,  e);
168                        }
169                        finally
170                        {
171                            BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
172                        }                            
173                    }
174                    
175                    Map<Version, List<ActionData>> initActions = _getInitializationActions(versions, migrationConfiguration);
176                    allInitActions.putAll(initActions);
177
178                    List<ActionData> upgradeActions = _getUpgradeActions(versions, migrationConfiguration);
179                    allUpgradeActions.addAll(upgradeActions);
180                }
181                catch (ConfigurationException | MigrationException e)
182                {
183                    throw new MigrationException("Exception occured in migration " + id, e);
184                }
185            }
186
187            if (_applyInitializationActions(allInitActions, logger)
188                || _applyUpgradeActions(allUpgradeActions, logger))
189            {
190                getLogger().info("Automatic migration is restarting the server to continue migration");
191                return true;
192            }
193            
194            if (!versionDeterminationFailed)
195            {
196                _notifyEndOfMigration();
197                
198                // To free some memory space, not done on failing
199                _configurations = null;
200            }
201            
202            getLogger().info("Automatic migration finished.");
203        }
204        catch (MigrationException e)
205        {
206            getLogger().error("Error during the automatic migration", e);
207        }
208        
209        return false;
210    }
211    
212    private VersionHandler _getVersionHandler(Configuration versionConfiguration) throws ConfigurationException
213    {
214        String versionHandlerType = versionConfiguration.getAttribute("type");
215        
216        VersionHandler extension = _versionHandlerEP.getExtension(versionHandlerType);
217        if (extension == null)
218        {
219            throw new ConfigurationException("A migration requests a versionHandler with id '" + versionHandlerType + "', which does not exists, this migration can not be done.", versionConfiguration);
220        }
221        return extension;
222    }
223
224    /**
225     * Notify with an {@link Event} the end of the migration.
226     */
227    protected void _notifyEndOfMigration()
228    {
229        if (_observationManager != null)
230        {
231            _observationManager.notify(new Event(ObservationConstants.EVENT_MIGRATION_ENDED, null, Map.of()));
232        }
233    }
234    
235    /**
236     * Execute all needed upgrades
237     * @param allInitializationActions list of needed upgrades
238     * @param logger The avalon wrapped logger
239     * @return true if an action requires a restart
240     * @throws MigrationException Something went wrong
241     */
242    protected boolean _applyInitializationActions(Map<Version, List<ActionData>> allInitializationActions, Logger logger) throws MigrationException
243    {
244        if (allInitializationActions.isEmpty())
245        {
246            getLogger().debug("No initialization to do");
247        }
248        else
249        {
250            for (Entry<Version, List<ActionData>> entry : allInitializationActions.entrySet())
251            {
252                Version version = entry.getKey();
253                List<ActionData> actions = entry.getValue();
254                
255                for (ActionData initializationAction : actions)
256                {
257                    Map<String, Object> environmentInformation = null;
258
259                    try
260                    {
261                        // Create the environment.
262                        environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, logger);
263                        
264                        Action actionExtension = _initializationEP.getExtension(initializationAction.getType());
265                        getLogger().info("Run initialization : " + initializationAction.toString());
266                        actionExtension.doAction(initializationAction);
267                        
268                        if (initializationAction.requiresRestart())
269                        {
270                            return true;
271                        }
272                    }
273                    finally
274                    {
275                        BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
276                    }
277                }
278                
279                // The ID of the version had already be set to the most recent action id
280                _upgradeVersion(version);
281            }
282        }
283        return false;
284    }
285    /**
286     * Execute all needed upgrades
287     * @param allActions list of needed upgrades
288     * @param logger The avalon wrapped logger
289     * @return true if an action requires a restart
290     * @throws MigrationException Something went wrong
291     */
292    protected boolean _applyUpgradeActions(List<ActionData> allActions, Logger logger) throws MigrationException
293    {
294        allActions.sort(Comparator.comparing(ActionData::getVersionNumber));
295        if (allActions.isEmpty())
296        {
297            getLogger().debug("No upgrade to do");
298        }
299        else
300        {
301            _logUpgradeActionsBeforeExecution(allActions);
302            for (ActionData action : allActions)
303            {
304                Map<String, Object> environmentInformation = null;
305
306                try
307                {
308                    // Create the environment.
309                    environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, logger);
310                    
311                    Action upgradeExtension = _upgradeEP.getExtension(action.getType());
312                    getLogger().info("Run upgrade : " + action.toString());
313                    upgradeExtension.doAction(action);
314                    _upgradeVersion(action);
315                    
316                    if (action.requiresRestart())
317                    {
318                        return true;
319                    }
320                }
321                finally
322                {
323                    BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
324                }
325            }
326        }
327        return false;
328    }
329
330    private void _logUpgradeActionsBeforeExecution(List<ActionData> allActions)
331    {
332        if (getLogger().isInfoEnabled())
333        {
334            int count = allActions.size();
335            StringBuilder sb = new StringBuilder().append(count).append(count > 1 ? " pending upgrade(s):" : " pending upgrade:");
336            for (ActionData action : allActions)
337            {
338                sb.append(System.lineSeparator());
339                sb.append("* ").append(action.toString());
340            }
341            getLogger().info(sb.toString());
342        }
343    }
344
345    /**
346     * When an upgrade (or an initialization) have been done, we write the new version
347     * @param action upgrade/initialization executed
348     * @throws MigrationException Something went wrong
349     */
350    protected void _upgradeVersion(ActionData action) throws MigrationException
351    {
352        Version newVersion = action.getVersion().copyFromActionData(action);
353        _upgradeVersion(newVersion);
354    }
355    
356    /**
357     * When an upgrade (or an initialization) have been done, we write the new version
358     * @param newVersion upgrade/initialization executed
359     * @throws MigrationException Something went wrong
360     */
361    protected void _upgradeVersion(Version newVersion) throws MigrationException
362    {
363        getLogger().debug("Update version info vor version : " + newVersion.toString());
364        String versionHandlerId = newVersion.getVersionHandlerId();
365        VersionHandler versionHandler = _versionHandlerEP.getExtension(versionHandlerId);
366        if (versionHandler == null)
367        {
368            throw new MigrationException("Version '" + newVersion.toString() + "' requires a versionHandler that does not exists: '" + versionHandlerId + "'.");
369        }
370        else
371        {
372            versionHandler.addVersion(newVersion);
373        }
374    }
375    
376    /**
377     * Analyze the configuration to get a list of upgrades needed based on the current Versions
378     * @param currentVersions list of current versions
379     * @param migrationConfiguration configuration for this extension
380     * @return a list of actions to work on, or null if an error occured
381     * @throws MigrationException impossible to parse the configuration
382     */
383    protected List<ActionData> _getUpgradeActions(List<Version> currentVersions, MigrationConfiguration migrationConfiguration) throws MigrationException
384    {
385        String componentId = migrationConfiguration.getId();
386        try
387        {
388            Configuration[] upgradeList = migrationConfiguration.getConfiguration().getChild("upgrades").getChildren("upgrade");
389            
390            List<ActionData> result = new ArrayList<>();
391            
392            for (Version version : currentVersions)
393            {
394                if (version == null)
395                {
396                    throw new MigrationException("The migration for '" + componentId + "' got a null Version.");
397                }
398                else if (version.getVersionNumber() != null)
399                {
400                    getLogger().debug("Check the upgrades needed for version: {}", version.toString());
401                    String versionNumber = version.getVersionNumber();
402
403                    String higherCurrentUpgradeVersionNumber = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration);
404                    if (higherCurrentUpgradeVersionNumber.compareTo(versionNumber) < 0)
405                    {
406                        getLogger().warn("There is a version stored more recent that any version available in conf for version: {}", version.toString());
407                    }
408                    
409                    List<ActionData> thisVersionUpgrades = new ArrayList<>();
410                    for (Configuration upgradeConf : upgradeList)
411                    {
412                        String upgradeVersionNumber = upgradeConf.getAttribute("versionNumber");
413                        String type = upgradeConf.getAttribute("type");
414                        String from = upgradeConf.getAttribute("from", null);
415                        String comment = upgradeConf.getAttribute("comment", null);
416                        boolean restartRequired = upgradeConf.getAttributeAsBoolean("restart", false);
417
418                        if (comment == null)
419                        {
420                            comment = "Automatic Upgrade.";
421                        }
422                        else
423                        {
424                            comment = "Automatic Upgrade: " + comment;
425                        }
426                        
427                        Action actionExtension = _upgradeEP.getExtension(type);
428                        
429                        if (actionExtension == null)
430                        {
431                            throw new ConfigurationException("The type '" + type + "' does not exist.", upgradeConf);
432                        }
433                        
434                        if (versionNumber.compareTo(upgradeVersionNumber) >= 0 // Obsolete migrations (migrating to older or current version) 
435                            || from != null && versionNumber.compareTo(from) > 0) // Future migration migrating from an older version
436                        {
437                            continue;
438                        }
439                        else
440                        {
441                            ActionData upgradeData = actionExtension.generateActionData(upgradeVersionNumber, version, comment, from, type, migrationConfiguration.getPluginName(), upgradeConf, restartRequired);
442                            thisVersionUpgrades.add(upgradeData);
443                        }
444                    }
445                    
446                    result.addAll(_removeDuplicatedActions(thisVersionUpgrades));
447                }
448            }
449            
450            return result;
451        }
452        catch (ConfigurationException e)
453        {
454            throw new MigrationException("Error while parsing configuration for component '" + componentId + "'", e);
455        }
456    }
457    
458    /**
459     * Analyze the configuration to get a list of initializations needed based on the current Versions
460     * @param currentVersions list of current versions
461     * @param migrationConfiguration configuration for this extension
462     * @return a list of actions to work on, or null if an error occured
463     * @throws MigrationException impossible to parse the configuration
464     */
465    protected Map<Version, List<ActionData>> _getInitializationActions(List<Version> currentVersions, MigrationConfiguration migrationConfiguration) throws MigrationException
466    {
467        String componentId = migrationConfiguration.getId();
468        Map<Version, List<ActionData>> result = new HashMap<>();
469        try
470        {
471            Configuration[] initializations = migrationConfiguration.getConfiguration().getChild("initializations").getChildren("initialization");
472            
473            String versionComment = migrationConfiguration.getConfiguration().getChild("initializations").getAttribute("comment", null);
474            
475            if (versionComment == null)
476            {
477                versionComment = "Automatic Initialization.";
478            }
479            else
480            {
481                versionComment = "Automatic Initialization: " + versionComment;
482            }
483            
484            for (Version version : currentVersions)
485            {
486                if (version == null)
487                {
488                    throw new MigrationException("The migration for '" + componentId + "' got a null Version.");
489                }
490                else if (version.getVersionNumber() == null)
491                {
492                    List<ActionData> thisVersionActions = new ArrayList<>();
493                    getLogger().debug("No version number, this is an initialization: {}", version.toString());
494                    
495                    String higherCurrentUpgradeVersionNumber = _getHigherCurrentUpgradeVersionNumber(migrationConfiguration);
496                    version.setVersionNumber(higherCurrentUpgradeVersionNumber);
497                    version.setComment(versionComment);
498                    
499                    for (Configuration initialization : initializations)
500                    {
501                        String type = initialization.getAttribute("type");
502                        boolean restartRequired = initialization.getAttributeAsBoolean("restart", false);
503                        String actionComment = null; // No comment will be used here, only the version will have a comment in an initialization
504                        
505                        Action actionExtension = _initializationEP.getExtension(type);
506                        
507                        if (actionExtension == null)
508                        {
509                            throw new ConfigurationException("The type '" + type + "' does not exist.", initialization);
510                        }
511                        ActionData initializationData = actionExtension.generateActionData(higherCurrentUpgradeVersionNumber, version, actionComment, null, type, migrationConfiguration.getPluginName(), initialization, restartRequired);
512                        thisVersionActions.add(initializationData);
513                    }
514
515                    result.put(version, thisVersionActions);
516                }
517                
518            }
519            
520            return result;
521        }
522        catch (ConfigurationException e)
523        {
524            throw new MigrationException("Error while parsing configuration for component '" + componentId + "'", e);
525        }
526    }
527    
528    
529    /**
530     * Returns the most recent version to apply (after an initialization)
531     * @param migrationConfiguration Configuration of the extension
532     * @return the id of the most current available version, or "0" if none available
533     * @throws ConfigurationException Sompthing wrong while reading the configuration
534     */
535    protected String _getHigherCurrentUpgradeVersionNumber(MigrationConfiguration migrationConfiguration) throws ConfigurationException
536    {
537        Configuration[] upgradeList = migrationConfiguration.getConfiguration().getChild("upgrades").getChildren("upgrade");
538        List<String> upgradeIds = new ArrayList<>();
539        for (Configuration upgradeConf : upgradeList)
540        {
541            String id = upgradeConf.getAttribute("versionNumber");
542            upgradeIds.add(id);
543        }
544        
545        if (!upgradeIds.isEmpty())
546        {
547            upgradeIds.sort(String.CASE_INSENSITIVE_ORDER);
548            return upgradeIds.get(upgradeIds.size() - 1);
549        }
550        else
551        {
552            return "0";
553        }
554    }
555    
556    /**
557     * Parse the list of upgrade to remove the group-migration
558     * @param actions list of upgrades to clean
559     * @return a list with only needed upgrades.
560     * @throws MigrationException list of actions incoherent
561     */
562    protected List<ActionData> _removeDuplicatedActions(List<ActionData> actions) throws MigrationException
563    {
564        // First, sort the upgrades by if to have the right order
565        actions.sort(
566            Comparator.comparing(ActionData::getVersionNumber)
567                .thenComparing(
568                    ActionData::getFrom,
569                    Comparator.nullsFirst(Comparator.naturalOrder())
570                )
571        );
572        
573        // Create a list of pairs for each upgrade containing a "for" attribute to konw from which version to which version we should remove the lines
574        List<Pair<String, String>> fromTo = actions.stream()
575                                                    .filter(u -> StringUtils.isNotBlank(u.getFrom()))
576                                                    .map(u -> Pair.of(u.getFrom(), u.getVersionNumber()))
577                                                    .collect(Collectors.toList());
578
579        _checkFromUpgrades(actions, fromTo);
580        
581        List<ActionData> result = actions;
582        
583        // Invert the order so we will start to apply the most recent one, it may override other lines with a From inside
584        Collections.reverse(fromTo);
585        // For each pair, remove the right versions
586        for (Pair<String, String> pair : fromTo)
587        {
588            String from = pair.getLeft();
589            String to = pair.getRight();
590            
591            result = result.stream()
592                        .filter(
593                            a -> a.getVersionNumber().compareTo(from) <= 0 
594                            || to.equals(a.getVersionNumber()) && from.equals(a.getFrom())
595                            || a.getVersionNumber().compareTo(to) > 0
596                        )
597                        .collect(Collectors.toList());
598        }
599        
600        return result;
601    }
602    
603    /**
604     * Tests that in the list of fromTo, that each "fromTo" is linked to an action with the same id and without "from" 
605     * @param actions list of actions
606     * @param fromTo list of actions overriding multiple actions
607     * @throws MigrationException there is an overriding version without a "simple" version with the same id
608     */
609    protected void _checkFromUpgrades(List<ActionData> actions, List<Pair<String, String>> fromTo) throws MigrationException
610    {
611        for (Pair<String, String> pair : fromTo)
612        {
613            String from = pair.getLeft();
614            String to = pair.getRight();
615            boolean anyMatch = actions.stream().anyMatch(a -> a.getFrom() == null && to.equals(a.getVersionNumber()));
616            if (!anyMatch)
617            {
618                throw new MigrationException("The action from '" + from + "' to '" + to + "' does not contain a normal upgrade from '" + from + "'");
619            }
620        }
621    }
622    
623    private Map<String, MigrationConfiguration> _getConfigurationsForReading()
624    {
625        if (_configurations == null)
626        {
627            throw new UnsupportedOperationException("Configurations have been emptied from the extension point, this method can't be used for now.");
628        }
629        return _configurations;
630    }
631    
632    public boolean hasExtension(String id)
633    {
634        return _getConfigurationsForReading().containsKey(id);
635    }
636
637    public MigrationConfiguration getExtension(String id)
638    {
639        return _getConfigurationsForReading().get(id);
640    }
641
642    public Set<String> getExtensionsIds()
643    {
644        return _getConfigurationsForReading().keySet();
645    }
646    
647    /**
648     * Get the expected version for a component.
649     * This is the latest upgrade number declared
650     * @param id the id of the component
651     * @return the expected version number for each version in this component
652     * @throws MigrationException Component not found in the calculated map of expected versions
653     */
654    public String getExpectedVersionForComponent(String id) throws MigrationException
655    {
656        if (!_expectedVersionsForExtensions.containsKey(id))
657        {
658            throw new MigrationException("Component '" + id + "' is not found in the calculated expected versions");
659        }
660        return _expectedVersionsForExtensions.get(id);
661    }
662}