Source code

001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.extraction.execution;
017
018import java.io.File;
019import java.io.OutputStream;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.Set;
028import java.util.stream.Collectors;
029import java.util.stream.StreamSupport;
030
031import org.apache.avalon.framework.component.Component;
032import org.apache.avalon.framework.service.ServiceException;
033import org.apache.avalon.framework.service.ServiceManager;
034import org.apache.avalon.framework.service.Serviceable;
035import org.apache.cocoon.xml.AttributesImpl;
036import org.apache.cocoon.xml.XMLUtils;
037import org.apache.commons.collections4.ListUtils;
038import org.apache.commons.lang3.StringUtils;
039import org.apache.commons.lang3.tuple.Pair;
040import org.apache.excalibur.source.Source;
041import org.apache.excalibur.source.SourceResolver;
042import org.apache.excalibur.source.impl.FileSource;
043import org.xml.sax.ContentHandler;
044
045import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
046import org.ametys.cms.contenttype.ContentTypesHelper;
047import org.ametys.cms.repository.Content;
048import org.ametys.core.right.RightManager;
049import org.ametys.core.right.RightManager.RightResult;
050import org.ametys.core.user.CurrentUserProvider;
051import org.ametys.core.util.DateUtils;
052import org.ametys.plugins.extraction.ExtractionConstants;
053import org.ametys.plugins.extraction.component.ExtractionComponent;
054import org.ametys.plugins.extraction.component.TwoStepsExecutingExtractionComponent;
055import org.ametys.plugins.extraction.execution.pipeline.Pipeline;
056import org.ametys.plugins.extraction.execution.pipeline.PipelineDescriptor;
057import org.ametys.plugins.extraction.execution.pipeline.Pipelines;
058import org.ametys.plugins.repository.AmetysObjectResolver;
059import org.ametys.runtime.plugin.component.AbstractLogEnabled;
060import org.ametys.runtime.util.AmetysHomeHelper;
061
062import com.google.common.base.Predicates;
063
064/**
065 * Extracts query results form a XML definition file
066 */
067public class ExtractionExecutor extends AbstractLogEnabled implements Component, Serviceable
068{
069    /** The Avalon role. */
070    public static final String ROLE = ExtractionExecutor.class.getName();
071    
072    private RightManager _rightManager;
073    private ExtractionDefinitionReader _reader;
074    private CurrentUserProvider _currentUserProvider;
075    private ContentTypeExtensionPoint _contentTypeExtensionPoint;
076    private AmetysObjectResolver _resolver;
077    private ContentTypesHelper _contentTypesHelper;
078    private SourceResolver _sourceResolver;
079    private PathResolver _resultPathResolver;
080
081    @Override
082    public void service(ServiceManager serviceManager) throws ServiceException
083    {
084        _rightManager = (RightManager) serviceManager.lookup(RightManager.ROLE);
085        _reader = (ExtractionDefinitionReader) serviceManager.lookup(ExtractionDefinitionReader.ROLE);
086        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
087        _contentTypeExtensionPoint = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
088        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
089        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
090        _sourceResolver = (SourceResolver) serviceManager.lookup(SourceResolver.ROLE);
091        _resultPathResolver = (PathResolver) serviceManager.lookup(PathResolver.ROLE);
092    }
093    
094    /**
095     * Executes the given extraction
096     * @param relativeFilePath The path of the extraction file to execute
097     * @param defaultResultFileName The default file name for the result (it can be unused, for instance if resultSubFolder is a file, i.e. contains a '.' in its last element)
098     * @param lang The language
099     * @param parameters The parameters
100     * @param pipeline The execution pipeline
101     * @return The set of the result files path
102     * @throws Exception if an errors occurred
103     */
104    public Set<Path> execute(String relativeFilePath, String defaultResultFileName, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
105    {
106        return _execute(relativeFilePath, defaultResultFileName, null, lang, parameters, pipeline);
107    }
108    
109    /**
110     * Executes the given extraction
111     * @param relativeFilePath The path of the extraction file to execute
112     * @param resultOutputStream The result output stream
113     * @param lang The language
114     * @param parameters The parameters
115     * @param pipeline The execution pipeline
116     * @throws Exception if an errors occurred
117     */
118    public void execute(String relativeFilePath, OutputStream resultOutputStream, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
119    {
120        _execute(relativeFilePath, null, resultOutputStream, lang, parameters, pipeline);
121    }
122    
123    private Set<Path> _execute(
124            String relativeFilePath,
125            String defaultResultFileName, 
126            OutputStream resultOutputStream, 
127            String lang, 
128            Map<String, Object> parameters, 
129            PipelineDescriptor pipeline) throws Exception
130    {
131        _checkRights();
132        
133        Pair<Extraction, String> extractionAndName = _getExtraction(relativeFilePath);
134        Extraction extraction = extractionAndName.getLeft();
135        String definitionFilename = extractionAndName.getRight();
136        
137        AttributesImpl attributes = _getAttrs(definitionFilename);
138
139        ExtractionExecutionContext context = _getContext(extraction, lang, parameters);
140        
141        if (resultOutputStream != null)
142        {
143            _doExecuteForPathWithNoVar(attributes, extraction, context, resultOutputStream, pipeline);
144            return Set.of();
145        }
146        else
147        {
148            return _doExecute(attributes, extraction, relativeFilePath, context, defaultResultFileName, pipeline);
149        }
150    }
151    
152    private void _checkRights()
153    {
154        if (_rightManager.hasRight(_currentUserProvider.getUser(), ExtractionConstants.EXECUTE_EXTRACTION_RIGHT_ID, "/admin") != RightResult.RIGHT_ALLOW)
155        {
156            String errorMessage = "User " + _currentUserProvider.getUser() + " tried to execute extraction with no sufficient rights"; 
157            getLogger().error(errorMessage);
158            throw new IllegalStateException(errorMessage);
159        }
160    }
161    
162    private Pair<Extraction, String> _getExtraction(String relativeFilePath) throws Exception
163    {
164        String absoluteFilePath = ExtractionConstants.DEFINITIONS_DIR + relativeFilePath;
165        Source src = _sourceResolver.resolveURI(absoluteFilePath);
166
167        try
168        {
169            File file = ((FileSource) src).getFile();
170            
171            if (!file.exists())
172            {
173                throw new IllegalArgumentException("The file " + relativeFilePath + " does not exist.");
174            }
175            
176            Extraction extraction = _reader.readExtractionDefinitionFile(file);
177            return Pair.of(extraction, file.getName());
178        }
179        catch (Exception e)
180        {
181            throw new IllegalStateException("An unexpected error occured.", e);
182        }
183        finally
184        {
185            _sourceResolver.release(src);
186        }
187    }
188    
189    private AttributesImpl _getAttrs(String definitionFilename)
190    {
191        AttributesImpl attributes = new AttributesImpl();
192        attributes.addCDATAAttribute("user", _currentUserProvider.getUser().getLogin());
193        attributes.addCDATAAttribute("date", ZonedDateTime.now().format(DateUtils.getISODateTimeFormatter()));
194        attributes.addCDATAAttribute("name", definitionFilename);
195        return attributes;
196    }
197    
198    private ExtractionExecutionContext _getContext(Extraction extraction, String lang, Map<String, Object> parameters)
199    {
200        ExtractionExecutionContext context = new ExtractionExecutionContext();
201        if (StringUtils.isNotEmpty(lang))
202        {
203            context.setDefaultLocale(new Locale(lang));
204        }
205        context.setDisplayOptionalColumns(_getDisplayOptionalColumns(extraction.getDisplayOptionalColumnsNames(), parameters));
206        context.setClauseVariables(_getQueryVariables(extraction.getQueryVariablesNamesAndContentTypes(), parameters));
207        return context;
208    }
209
210    Map<String, Boolean> _getDisplayOptionalColumns(List<String> displayOptionalColumnsNames, Map<String, Object> parameters)
211    {
212        Map<String, Boolean> result = new HashMap<>();
213        for (String name : displayOptionalColumnsNames)
214        {
215            Boolean value = _getDipslayOptionalColumn(name, parameters);
216            if (value == null)
217            {
218                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
219            }
220            else
221            {
222                result.put(name, value);
223            }
224        }
225        return result;
226    }
227    
228    private Boolean _getDipslayOptionalColumn(String optionalColumnName, Map<String, Object> parameters)
229    {
230        Object value = parameters.get(optionalColumnName);
231        
232        if (value instanceof Boolean)
233        {
234            return (Boolean) value;
235        }
236        else if (value instanceof String)
237        {
238            return Boolean.valueOf((String) value);
239        }
240        else
241        {
242            return null;
243        }
244    }
245
246    private Map<String, String> _getQueryVariables(Map<String, String> queryVariablesNamesAndContentTypes, Map<String, Object> parameters)
247    {
248        Map<String, String> result = new HashMap<>();
249        for (Map.Entry<String, String> entry : queryVariablesNamesAndContentTypes.entrySet())
250        {
251            String name = entry.getKey();
252            String contentTypeId = entry.getValue();
253            if (!_contentTypeExtensionPoint.hasExtension(contentTypeId))
254            {
255                throw new IllegalArgumentException("Extraction - content type '" + contentTypeId + "' used in variable '" + name + "' definition does not exist");
256            }
257            
258            String contentId = (String) parameters.get(name);
259            if (null == contentId)
260            {
261                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
262            }
263            
264            Content content = _resolver.resolveById(contentId);
265            if (!_contentTypesHelper.isInstanceOf(content, contentTypeId))
266            {
267                throw new IllegalArgumentException("Extraction - content '" + contentId + "' is not an instance of content type '" + contentTypeId + "', defined by the variable named '" + name + "'");
268            }
269            
270            result.put(name, contentId);
271        }
272        return result;
273    }
274    
275    private Set<Path> _doExecute(AttributesImpl attributes, Extraction extraction, String extractionFilePath, ExtractionExecutionContext context, String defaultResultFileName, PipelineDescriptor pipeline) throws Exception
276    {
277        String unresolvedPath = pipeline.getResultSubfolder();
278        Path basePath = Paths.get(AmetysHomeHelper.getAmetysHomeData().toPath().toString(), ExtractionConstants.RESULT_EXTRACTION_DIR_NAME);
279       
280        if (_resultPathResolver.hasVariable(unresolvedPath))
281        {
282            List<ExtractionComponent> extractionComponents = extraction.getExtractionComponents();
283            boolean allAreNotTwoStepsComponent = extractionComponents.stream()
284                    .filter(Predicates.not(TwoStepsExecutingExtractionComponent.class::isInstance))
285                    .findFirst()
286                    .isPresent();
287            if (allAreNotTwoStepsComponent)
288            {
289                throw new IllegalArgumentException("The extraction " + extractionFilePath + " has an invalid component at first level which does not support a subfolder containing variables.");
290            }
291            
292            List<TwoStepsExecutingExtractionComponent> components = extractionComponents.stream()
293                    .map(TwoStepsExecutingExtractionComponent.class::cast)
294                    .collect(Collectors.toList());
295            return _doExecuteForPathWithVar(attributes, extraction, context, components, basePath, unresolvedPath, pipeline);
296        }
297        else
298        {
299            Path fileOrFolderPath = _resultPathResolver.resolvePath(unresolvedPath, null, extraction, basePath).keySet().iterator().next();
300            Path filePath = _filePathWhenNoVar(fileOrFolderPath, defaultResultFileName);
301            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);)
302            {
303                _doExecuteForPathWithNoVar(attributes, extraction, context, fileOs, pipeline);
304                return Set.of(filePath);
305            }
306        }
307    }
308    
309    private Path _filePathWhenNoVar(Path fileOrFolderPath, String defaultFileName)
310    {
311        Path fileName = fileOrFolderPath.getFileName();
312        if (fileName.toString().contains("."))
313        {
314            // is already a file, do not use default file name
315            return fileOrFolderPath;
316        }
317        return Paths.get(fileOrFolderPath.toString(), defaultFileName);
318    }
319    
320    private void _doExecuteForPathWithNoVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, OutputStream outputStream, PipelineDescriptor pipelineDescriptor) throws Exception
321    {
322        try (Pipeline pipeline = pipelineDescriptor.newPipeline(outputStream);)
323        {
324            ContentHandler contentHandler = pipeline.getHandler();
325            _noVarExecute(contentHandler, attributes, extraction, context);
326            pipeline.serialize();
327        }
328    }
329    
330    private void _noVarExecute(ContentHandler contentHandler, AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context) throws Exception
331    {
332        contentHandler.startDocument();
333        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
334        
335        for (ExtractionComponent component : extraction.getExtractionComponents())
336        {
337            component.prepareComponentExecution(context);
338            component.execute(contentHandler, context);
339        }
340        
341        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
342        contentHandler.endDocument();
343    }
344    
345    private Set<Path> _doExecuteForPathWithVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, Path basePath, String unresolvedPath, PipelineDescriptor pipelineDescriptor) throws Exception
346    {
347        Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent = new HashMap<>();
348        for (TwoStepsExecutingExtractionComponent component : components)
349        {
350            component.prepareComponentExecution(context);
351            List<Content> firstLevelResults = StreamSupport.stream(component.computeFirstLevelResults(context).spliterator(), false)
352                    .collect(Collectors.toList());
353            firstLevelResultsByComponent.put(component, firstLevelResults);
354        }
355        
356        List<Content> allContents = firstLevelResultsByComponent.values()
357                .stream()
358                .flatMap(List::stream)
359                .collect(Collectors.toList());
360        
361        String unresolvedFilePath = _unresolvedFilePathWhenVar(unresolvedPath, pipelineDescriptor);
362        Map<Path, List<Content>> paths = _resultPathResolver.resolvePath(unresolvedFilePath, allContents, extraction, basePath);
363        for (Path filePath : paths.keySet())
364        {
365            List<Content> involvedContentsForPath = paths.get(filePath);
366            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);
367                 Pipeline pipeline = pipelineDescriptor.newPipeline(fileOs);)
368            {
369                ContentHandler contentHandler = pipeline.getHandler();
370                _withVarExecute(contentHandler, attributes, context, components, involvedContentsForPath, firstLevelResultsByComponent);
371                pipeline.serialize();
372            }
373        }
374        
375        return paths.keySet();
376    }
377    
378    private String _unresolvedFilePathWhenVar(String unresolvedFolderOrFilePath, PipelineDescriptor pipelineDescriptor)
379    {
380        if (_resultPathResolver.isFolder(unresolvedFolderOrFilePath))
381        {
382            return unresolvedFolderOrFilePath + "/${title}." + pipelineDescriptor.getDefaultExtension();
383        }
384        else
385        {
386            return unresolvedFolderOrFilePath;
387        }
388    }
389    
390    private void _withVarExecute(ContentHandler contentHandler, AttributesImpl attributes, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, List<Content> involvedContentsForPath, Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent) throws Exception
391    {
392        contentHandler.startDocument();
393        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
394        
395        for (TwoStepsExecutingExtractionComponent component : components)
396        {
397            List<Content> firstLevelResults = firstLevelResultsByComponent.get(component);
398            List<Content> involvedFirstLevelResults = ListUtils.intersection(firstLevelResults, involvedContentsForPath); 
399            component.executeFor(contentHandler, involvedFirstLevelResults, context);
400        }
401        
402        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
403        contentHandler.endDocument();
404    }
405}