001/*
002 *  Copyright 2023 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.core.migration;
017
018import java.time.Instant;
019import java.util.ArrayList;
020import java.util.Collections;
021import java.util.Comparator;
022import java.util.HashMap;
023import java.util.List;
024import java.util.Map;
025import java.util.stream.Collectors;
026
027import org.apache.avalon.framework.component.Component;
028import org.apache.avalon.framework.context.Context;
029import org.apache.avalon.framework.context.ContextException;
030import org.apache.avalon.framework.context.Contextualizable;
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.avalon.framework.service.Serviceable;
034import org.apache.cocoon.Constants;
035import org.apache.cocoon.util.log.SLF4JLoggerAdapter;
036import org.apache.commons.lang3.StringUtils;
037import org.apache.commons.lang3.tuple.Pair;
038
039import org.ametys.core.ObservationConstants;
040import org.ametys.core.engine.BackgroundEngineHelper;
041import org.ametys.core.migration.action.Action;
042import org.ametys.core.migration.action.ActionConfiguration;
043import org.ametys.core.migration.action.ActionExtensionPoint;
044import org.ametys.core.migration.version.Version;
045import org.ametys.core.migration.version.VersionConfiguration;
046import org.ametys.core.migration.version.handler.VersionHandler;
047import org.ametys.core.migration.version.storage.VersionStorage;
048import org.ametys.core.observation.Event;
049import org.ametys.core.observation.ObservationManager;
050import org.ametys.runtime.i18n.I18nizableText;
051import org.ametys.runtime.plugin.component.AbstractLogEnabled;
052import org.ametys.runtime.servlet.RuntimeServlet;
053import org.ametys.runtime.servlet.RuntimeServlet.MaintenanceStatus;
054
055/**
056 * Main entry point for all automatic migration related tasks.
057 */
058public class MigrationEngine extends AbstractLogEnabled implements Contextualizable, Serviceable, Component
059{
060    /** Avalon Role */
061    public static final String ROLE = MigrationEngine.class.getName();
062    
063    private MigrationExtensionPoint _migrationExtensionPoint;
064    private MigrationExtensionPoint _migrationDataExtensionPoint;
065    
066    private ActionExtensionPoint _upgradeEP;
067    private ActionExtensionPoint _initializationEP;
068
069    private ObservationManager _observationManager;
070
071    private ServiceManager _manager;
072
073    private org.apache.cocoon.environment.Context _context;
074    
075    private ActionData _failedAction;
076    private MigrationException _failedException;
077    
078    public void contextualize(Context context) throws ContextException
079    {
080        _context = (org.apache.cocoon.environment.Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT);
081    }
082    
083    public void service(ServiceManager smanager) throws ServiceException
084    {
085        _manager = smanager;
086        
087        _migrationExtensionPoint = (MigrationExtensionPoint) smanager.lookup(MigrationExtensionPoint.ROLE);
088        _migrationDataExtensionPoint = (MigrationExtensionPoint) smanager.lookup(MigrationExtensionPoint.ROLE + "/internal");
089        
090        _upgradeEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_UPGRADE);
091        _initializationEP = (ActionExtensionPoint) smanager.lookup(ActionExtensionPoint.ROLE_INITIALIZATION);
092        
093        if (smanager.hasService(ObservationManager.ROLE))
094        {
095            _observationManager = (ObservationManager) smanager.lookup(ObservationManager.ROLE);
096        }
097    }
098    
099    /**
100     * All migrations are parsed and executed one per one in the right order
101     * @return true if the migration requires a server restart ; false otherwise
102     */
103    public boolean migrate()
104    {
105        getLogger().info("Automatic migration start");
106        try
107        {
108            boolean hasToRestart = _doMigration(_migrationDataExtensionPoint);
109            
110            if (hasToRestart)
111            {
112                return true;
113            }
114            
115            hasToRestart = _doMigration(_migrationExtensionPoint);
116            
117            if (hasToRestart)
118            {
119                return true;
120            }
121            
122            _notifyEndOfMigration();
123          
124            getLogger().info("Automatic migration finished.");
125        }
126        catch (MigrationException e)
127        {
128            RuntimeServlet.setMaintenanceStatus(MaintenanceStatus.AUTOMATIC, null);
129            getLogger().error("Error during the automatic migration", e);
130        }
131        
132        return false;
133    }
134
135    private boolean _doMigration(MigrationExtensionPoint extPoint) throws MigrationException
136    {
137        boolean versionDeterminationFailed = false;
138        List<ActionData> allInitActions = new ArrayList<>(); 
139        List<ActionData> allUpgradeActions = new ArrayList<>();
140        
141        // First gather all versions to init and upgrade from all migration components
142        for (String componentId : extPoint.getExtensionsIds())
143        {
144            MigrationComponent component = extPoint.getExtension(componentId);
145            
146            try
147            {
148                Versions versions = getVersions(component);
149                
150                List<VersionList> versionLists = _getVersionLists(versions);
151                
152                for (VersionList list : versionLists)
153                {
154                    Version versionToUpgrade = null;
155                    
156                    if (list.versions().isEmpty())
157                    {
158                        // no versions, the associated component has to be initialized
159                        Version newVersion = component.versionStorage().createVersion(list.componentId(), component, list.storageConfiguration(), list.additionalValues());
160
161                        if (newVersion.getVersionNumber() == null)
162                        {
163                            allInitActions.add(new ActionData(newVersion, getHighestUpgradeVersionNumber(component), component.initialization(), list.id()));
164                        }
165                        else
166                        {
167                            // non null versionNumer, meaning we are using it as our current version
168                            // let's store it and add it to the upgrade flow 
169                            _createAndStoreVersion(new ActionData(newVersion, newVersion.getVersionNumber(), null, list.id()), "Initial version");
170                            versionToUpgrade = newVersion;
171                        }
172                    }
173                    else
174                    {
175                        versionToUpgrade = getLatestVersion(list.versions());
176                    }
177                    
178                    if (versionToUpgrade != null)
179                    {
180                        for (ActionConfiguration configuration : getUpgrades(versionToUpgrade, component.upgrades()))
181                        {
182                            allUpgradeActions.add(new ActionData(versionToUpgrade, null, configuration, list.id()));
183                        }
184                    }
185                }
186            }
187            catch (MigrationException e)
188            {
189                // we're trying to execute as much actions as possible, so we store the issue and throw later
190                versionDeterminationFailed = true;
191                _failedException = e;
192                getLogger().error("Error during version determination for component {}", componentId,  e);
193            }
194        }
195        
196        // then execute all needed actions
197        if (_executeInitializationActions(allInitActions) || _executeUpgradeActions(allUpgradeActions))
198        {
199            getLogger().info("Automatic migration is restarting the server to continue migration");
200            return true;
201        }
202        
203        if (versionDeterminationFailed)
204        {
205            // at least one version determination has failed
206            throw new MigrationException("Version determination failed. See previous error messages for more details.");
207        }
208        
209        return false;
210    }
211    
212    /**
213     * Return all upgrades to execute for the current version
214     * @param currentVersion the current version
215     * @param availableUpgrades all available upgrades
216     * @return all upgrade to execute
217     * @throws MigrationException if the upgrades are not compatible
218     */
219    public List<ActionConfiguration> getUpgrades(Version currentVersion, List<ActionConfiguration> availableUpgrades) throws MigrationException
220    {
221        List<ActionConfiguration> actualUpgrades = new ArrayList<>();
222        
223        String versionNumber = currentVersion.getVersionNumber();
224
225        String higherCurrentUpgradeVersionNumber = getHighestUpgradeVersionNumber(currentVersion.getComponent());
226        if (higherCurrentUpgradeVersionNumber.compareTo(versionNumber) < 0)
227        {
228            getLogger().warn("There is a version stored more recent that any version available in conf for version: {}", currentVersion.toString());
229        }
230        
231        for (ActionConfiguration upgrade : availableUpgrades)
232        {
233            String from = upgrade.getFrom();
234            String upgradeVersionNumber = upgrade.getVersionNumber();
235            
236            if (versionNumber.compareTo(upgradeVersionNumber) < 0 // Accepts all upgrades to newer versions
237                    && (from == null || versionNumber.compareTo(from) <= 0)) // and older than the from attribute, if any (allowing to bypass some upgrades)
238            {
239                actualUpgrades.add(upgrade);
240            }
241        }
242
243        return _removeDuplicatedActions(actualUpgrades);
244    }
245    
246    /**
247     * Retrives all stored versions for the given component
248     * @param component the component to check
249     * @return all stored versions.
250     * @throws MigrationException if something wrong occured
251     */
252    public Versions getVersions(MigrationComponent component) throws MigrationException
253    {
254        Map<String, Object> environmentInformation = null;
255        try
256        {
257            environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, new SLF4JLoggerAdapter(getLogger()));
258
259            return component.versionHandler.getVersions(component);
260        }
261        finally
262        {
263            BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
264        }                            
265    }
266    
267    private List<VersionList> _getVersionLists(Versions versions)
268    {
269        if (versions instanceof VersionList versionList)
270        {
271            return List.of(versionList);
272        }
273        else
274        {
275            List<VersionList> versionLists = new ArrayList<>();
276
277            for (Versions subVersions : ((VersionsContainer) versions).values())
278            {
279                versionLists.addAll(_getVersionLists(subVersions));
280            }
281            
282            return versionLists;
283        }
284    }
285    
286    /**
287     * Search a given version list among all thos returned by a {@link VersionHandler}
288     * @param versions the version tree
289     * @param id the wanted id
290     * @return the corresponding {@link VersionList}
291     */
292    public VersionList getVersionList(Versions versions, String id)
293    {
294        List<VersionList> lists = _getVersionLists(versions);
295        return lists.stream()
296                    .filter(list -> id.equals(list.id()))
297                    .findFirst()
298                    .orElse(null);
299    }
300    
301    /**
302     * Returns the latest version (according to version numbers) among the given list of versions
303     * @param versions a list of {@link Version}
304     * @return the latest version, or null if the list is empty
305     */
306    public Version getLatestVersion(List<Version> versions)
307    {
308        return versions.stream()
309                       .sorted(Comparator.comparing(Version::getVersionNumber).reversed())
310                       .findFirst()
311                       .orElse(null);
312    }
313
314    private void _notifyEndOfMigration()
315    {
316        if (_observationManager != null)
317        {
318            _observationManager.notify(new Event(ObservationConstants.EVENT_MIGRATION_ENDED, null, Map.of()));
319        }
320    }
321    
322    private boolean _executeInitializationActions(List<ActionData> allInitializationActions) throws MigrationException
323    {
324        for (ActionData action : allInitializationActions)
325        {
326            try
327            {
328                if (initializeVersion(action))
329                {
330                    // The migration require a restart. Stop the iteration here.
331                    return true;
332                }
333            }
334            catch (MigrationException e)
335            {
336                // store the failing action and rethrow
337                _failedAction = e.getFailedAction();
338                _failedException = e;
339                throw e;
340            }
341        }
342        
343        return false;
344    }
345    
346    private boolean _executeUpgradeActions(List<ActionData> allUpgradeActions) throws MigrationException
347    {
348        allUpgradeActions.sort(Comparator.comparing(action -> action.configuration().getVersionNumber()));
349        
350        if (allUpgradeActions.isEmpty())
351        {
352            getLogger().debug("No upgrade to do");
353        }
354        else
355        {
356            _logUpgradeActionsBeforeExecution(allUpgradeActions);
357            for (ActionData action : allUpgradeActions)
358            {
359                try
360                {
361                    if (upgradeVersion(action))
362                    {
363                        // The migration require a restart. Stop the iteration here.
364                        return true;
365                    }
366                }
367                catch (MigrationException e)
368                {
369                    // store the failing action and rethrow
370                    _failedAction = action;
371                    _failedException = e;
372                    throw e;
373                }
374            }
375        }
376        return false;
377    }
378
379    /**
380     * Execute all needed initializations
381     * @param action the version and action configuration to initialize
382     * @return true if an action requires a restart
383     * @throws MigrationException Something went wrong
384     */
385    public boolean initializeVersion(ActionData action) throws MigrationException
386    {
387        ActionConfiguration configuration = action.configuration();
388        String versionComment = configuration != null ? configuration.getComment() : null;
389        if (StringUtils.isBlank(versionComment))
390        {
391            versionComment = "Automatic Initialization";
392        }
393        else
394        {
395            versionComment = "Automatic Initialization: " + versionComment;
396        }
397        
398        boolean restartRequired = false;
399        if (configuration != null)
400        {
401            restartRequired = doAction(action, _initializationEP);
402        }
403        
404        _createAndStoreVersion(action, versionComment);
405        
406        return restartRequired;
407    }
408   
409    /**
410     * Execute an upgrade
411     * @param action the version and action configuration to upgrade
412     * @return true if an action requires a restart
413     * @throws MigrationException Something went wrong
414     */
415    public boolean upgradeVersion(ActionData action) throws MigrationException
416    {
417        String versionComment = action.configuration().getComment();
418        if (StringUtils.isBlank(versionComment))
419        {
420            versionComment = "Automatic Upgrade";
421        }
422        else
423        {
424            versionComment = "Automatic Upgrade: " + versionComment;
425        }
426        
427        boolean restartRequired = doAction(action, _upgradeEP);
428        _createAndStoreVersion(action, versionComment);
429        return restartRequired;
430    }
431   
432    /**
433     * Executes a migration action.
434     * @param actionData  the version and action configuration to execute
435     * @param extensionPoint the related {@link ActionExtensionPoint}
436     * @return true if the action needs a restart after its execution
437     * @throws MigrationException Something went wrong
438     */
439    public boolean doAction(ActionData actionData, ActionExtensionPoint extensionPoint) throws MigrationException
440    {
441        Map<String, Object> environmentInformation = null;
442        
443        try
444        {
445            // Create the environment.
446            environmentInformation = BackgroundEngineHelper.createAndEnterEngineEnvironment(_manager, _context, new SLF4JLoggerAdapter(getLogger()));
447            
448            Action action = extensionPoint.getExtension(actionData.configuration().getType());
449            getLogger().info("Run action : {}", actionData.configuration());
450            
451            action.act(actionData);
452            
453            if (actionData.configuration().requiresRestart())
454            {
455                return true;
456            }
457        }
458        catch (MigrationException e)
459        {
460            // rethrow with attached context
461            actionData.currentVersion().setExecutionInstant(Instant.now());
462            throw new MigrationException(e.getMessage(), e.getFailureMessage(), e.getCause(), actionData);
463        }
464        finally
465        {
466            BackgroundEngineHelper.leaveEngineEnvironment(environmentInformation);
467        }
468        
469        return false;
470    }
471
472    private void _logUpgradeActionsBeforeExecution(List<ActionData> allActions)
473    {
474        if (getLogger().isInfoEnabled())
475        {
476            int count = allActions.size();
477            StringBuilder sb = new StringBuilder().append(count).append(count > 1 ? " pending upgrade(s):" : " pending upgrade:");
478            for (ActionData action : allActions)
479            {
480                sb.append(System.lineSeparator());
481                sb.append("* ").append(action.toString());
482            }
483            getLogger().info(sb.toString());
484        }
485    }
486
487    private Version _createAndStoreVersion(ActionData action, String comment) throws MigrationException
488    {
489        Version currentVersion = action.currentVersion();
490        Version newVersion = currentVersion.getStorage().createVersion(currentVersion.getComponentId(), currentVersion.getComponent(), currentVersion.getStorageConfiguration(), currentVersion.getAdditionalValues());
491        newVersion.setExecutionInstant(Instant.now());
492        newVersion.setComment(comment);
493        newVersion.setVersionNumber(action.targetVersionNumber() != null ? action.targetVersionNumber() : action.configuration() != null ? action.configuration().getVersionNumber() : "0");
494        
495        getLogger().info("Store new version: {}", newVersion);
496
497        newVersion.getStorage().storeVersion(newVersion);
498        
499        return newVersion;
500    }
501    
502    /**
503     * Returns the highest version number among all components' upgrades
504     * @param component the {@link MigrationComponent} to consider
505     * @return the highest upgrade version number
506     */
507    public String getHighestUpgradeVersionNumber(MigrationComponent component)
508    {
509        return component.upgrades().stream()
510                                   .map(ActionConfiguration::getVersionNumber)
511                                   .sorted(Comparator.reverseOrder())
512                                   .findFirst()
513                                   .orElse("0");
514    }
515    
516    private List<ActionConfiguration> _removeDuplicatedActions(List<ActionConfiguration> actions) throws MigrationException
517    {
518        // First, sort the upgrades by if to have the right order
519        actions.sort(Comparator.comparing(ActionConfiguration::getVersionNumber)
520                               .thenComparing(ActionConfiguration::getFrom, Comparator.nullsFirst(Comparator.naturalOrder())));
521        
522        // Create a list of pairs for each upgrade containing a "from" attribute to know from which version to which version we should remove the lines
523        List<Pair<String, String>> fromTo = actions.stream()
524                                                   .filter(u -> StringUtils.isNotBlank(u.getFrom()))
525                                                   .map(u -> Pair.of(u.getFrom(), u.getVersionNumber()))
526                                                   .collect(Collectors.toList());
527
528        _checkFromUpgrades(actions, fromTo);
529        
530        List<ActionConfiguration> result = actions;
531        
532        // Invert the order so we will start to apply the most recent one, it may override other lines with a From inside
533        Collections.reverse(fromTo);
534        
535        // For each pair, remove the right versions
536        for (Pair<String, String> pair : fromTo)
537        {
538            String from = pair.getLeft();
539            String to = pair.getRight();
540            
541            result = result.stream()
542                           .filter(a -> a.getVersionNumber().compareTo(from) <= 0 
543                               || to.equals(a.getVersionNumber()) && from.equals(a.getFrom())
544                               || a.getVersionNumber().compareTo(to) > 0)
545                           .collect(Collectors.toList());
546        }
547        
548        return result;
549    }
550    
551    /**
552     * Tests that in the list of fromTo, that each "fromTo" is linked to an action with the same id and without "from" 
553     * @param actions list of actions
554     * @param fromTo list of actions overriding multiple actions
555     * @throws MigrationException there is an overriding version without a "simple" version with the same id
556     */
557    protected void _checkFromUpgrades(List<ActionConfiguration> actions, List<Pair<String, String>> fromTo) throws MigrationException
558    {
559        for (Pair<String, String> pair : fromTo)
560        {
561            String from = pair.getLeft();
562            String to = pair.getRight();
563            
564            boolean anyMatch = actions.stream().anyMatch(a -> a.getFrom() == null && to.equals(a.getVersionNumber()));
565            
566            if (!anyMatch)
567            {
568                throw new MigrationException("The action from '" + from + "' to '" + to + "' does not contain a normal upgrade from '" + from + "'");
569            }
570        }
571    }
572    
573    /**
574     * If the upgrade process has failed, returns the corresponding {@link ActionConfiguration}.<br>
575     * Returns null if the process went well.
576     * @return the failed action data 
577     */
578    public ActionData getFailedAction()
579    {
580        return _failedAction;
581    }
582    
583    /**
584     * If the upgrade process has failed, returns the corresponding {@link MigrationException}.<br>
585     * Returns null if the process went well.
586     * @return the exception causing the upgrade failure 
587     */
588    public MigrationException getFailedException()
589    {
590        return _failedException;
591    }
592    
593    /**
594     * A migration component represents a versioned feature set 
595     * @param id the component id
596     * @param internal if this component is used for internal migration data
597     * @param pluginName the component's plugin
598     * @param featureName the component's feature
599     * @param versionHandlerType the {@link VersionHandler} type
600     * @param versionHandler the {@link VersionHandler}
601     * @param versionStorage the {@link VersionStorage}
602     * @param versionConfiguration the specific {@link VersionConfiguration} created by the associated {@link VersionHandler}
603     * @param initialization the initialization action, if any
604     * @param upgrades all actions defined for uprades
605     */
606    public record MigrationComponent(String id, boolean internal, String pluginName, String featureName, String versionHandlerType, VersionHandler versionHandler, VersionStorage versionStorage, VersionConfiguration versionConfiguration, ActionConfiguration initialization, List<ActionConfiguration> upgrades) { /* empty */ }
607    
608    /**
609     * A version tree, grouped recursively by sub-components.
610     */
611    public sealed interface Versions permits VersionsContainer, VersionList 
612    { 
613        // empty interface
614    }
615    
616    /**
617     * Contains a list of versions for a sub-component
618     */
619    public static final class VersionsContainer extends HashMap<I18nizableText, Versions> implements Versions
620    { 
621        // empty class
622    }
623    
624    /**
625     * The leaf of the {@link Versions} tree. Contains the actual stored data and all informations needed to create a new version from storage.
626     * @param id the id of this list, unique for a given {@link MigrationComponent}.
627     * @param versions the version list
628     * @param componentId the component id
629     * @param storageConfiguration the storage configuration
630     * @param additionalValues an opaque object holding necessary information for the {@link VersionStorage} to create new versions if needed.
631     */
632    public record VersionList(String id, List<Version> versions, String componentId, VersionConfiguration storageConfiguration, Map<String, Object> additionalValues) implements Versions { /* empty */ }
633    
634    /**
635     * All data needed to actually execute an upgrade action: the current version and the action configuration.
636     * @param currentVersion the current version
637     * @param targetVersionNumber the target versionNumber. If not present (mainly in case of upgrades), the target version is taken from the configuration.
638     * @param configuration the action configuration
639     * @param versionListId the encapsulating {@link VersionList}'s id, or null if not known
640     */
641    public record ActionData(Version currentVersion, String targetVersionNumber, ActionConfiguration configuration, String versionListId) { /* empty */ }
642}