Source code

001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.extraction.execution;
017
018import java.io.File;
019import java.io.OutputStream;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.Set;
028import java.util.stream.Collectors;
029import java.util.stream.StreamSupport;
030
031import org.apache.avalon.framework.component.Component;
032import org.apache.avalon.framework.service.ServiceException;
033import org.apache.avalon.framework.service.ServiceManager;
034import org.apache.avalon.framework.service.Serviceable;
035import org.apache.cocoon.xml.AttributesImpl;
036import org.apache.cocoon.xml.XMLUtils;
037import org.apache.commons.collections4.ListUtils;
038import org.apache.commons.lang3.StringUtils;
039import org.apache.commons.lang3.tuple.Pair;
040import org.apache.excalibur.source.Source;
041import org.apache.excalibur.source.SourceResolver;
042import org.apache.excalibur.source.impl.FileSource;
043import org.xml.sax.ContentHandler;
044
045import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
046import org.ametys.cms.contenttype.ContentTypesHelper;
047import org.ametys.cms.repository.Content;
048import org.ametys.core.right.RightManager;
049import org.ametys.core.right.RightManager.RightResult;
050import org.ametys.core.user.CurrentUserProvider;
051import org.ametys.core.util.DateUtils;
052import org.ametys.plugins.extraction.ExtractionConstants;
053import org.ametys.plugins.extraction.component.ExtractionComponent;
054import org.ametys.plugins.extraction.component.TwoStepsExecutingExtractionComponent;
055import org.ametys.plugins.extraction.execution.pipeline.Pipeline;
056import org.ametys.plugins.extraction.execution.pipeline.PipelineDescriptor;
057import org.ametys.plugins.extraction.execution.pipeline.Pipelines;
058import org.ametys.plugins.repository.AmetysObjectResolver;
059import org.ametys.runtime.plugin.component.AbstractLogEnabled;
060import org.ametys.runtime.util.AmetysHomeHelper;
061
062import com.google.common.base.Predicates;
063
064/**
065 * Extracts query results form a XML definition file
066 */
067public class ExtractionExecutor extends AbstractLogEnabled implements Component, Serviceable
068{
069    /** The Avalon role. */
070    public static final String ROLE = ExtractionExecutor.class.getName();
071    
072    private RightManager _rightManager;
073    private ExtractionDefinitionReader _reader;
074    private CurrentUserProvider _currentUserProvider;
075    private ContentTypeExtensionPoint _contentTypeExtensionPoint;
076    private AmetysObjectResolver _resolver;
077    private ContentTypesHelper _contentTypesHelper;
078    private SourceResolver _sourceResolver;
079    private PathResolver _resultPathResolver;
080
081    @Override
082    public void service(ServiceManager serviceManager) throws ServiceException
083    {
084        _rightManager = (RightManager) serviceManager.lookup(RightManager.ROLE);
085        _reader = (ExtractionDefinitionReader) serviceManager.lookup(ExtractionDefinitionReader.ROLE);
086        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
087        _contentTypeExtensionPoint = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
088        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
089        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
090        _sourceResolver = (SourceResolver) serviceManager.lookup(SourceResolver.ROLE);
091        _resultPathResolver = (PathResolver) serviceManager.lookup(PathResolver.ROLE);
092    }
093    
094    /**
095     * Executes the given extraction
096     * @param relativeFilePath The path of the extraction file to execute
097     * @param defaultResultFileName The default file name for the result (it can be unused, for instance if resultSubFolder is a file, i.e. contains a '.' in its last element)
098     * @param lang The language
099     * @param parameters The parameters
100     * @param pipeline The execution pipeline
101     * @return The set of the result files path
102     * @throws Exception if an errors occurred
103     */
104    public Set<Path> execute(String relativeFilePath, String defaultResultFileName, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
105    {
106        return _execute(relativeFilePath, defaultResultFileName, null, lang, parameters, pipeline);
107    }
108    
109    /**
110     * Executes the given extraction
111     * @param relativeFilePath The path of the extraction file to execute
112     * @param resultOutputStream The result output stream
113     * @param lang The language
114     * @param parameters The parameters
115     * @param pipeline The execution pipeline
116     * @throws Exception if an errors occurred
117     */
118    public void execute(String relativeFilePath, OutputStream resultOutputStream, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
119    {
120        _execute(relativeFilePath, null, resultOutputStream, lang, parameters, pipeline);
121    }
122    
123    private Set<Path> _execute(
124            String relativeFilePath,
125            String defaultResultFileName, 
126            OutputStream resultOutputStream, 
127            String lang, 
128            Map<String, Object> parameters, 
129            PipelineDescriptor pipeline) throws Exception
130    {
131        _checkRights();
132        
133        Pair<Extraction, String> extractionAndName = _getExtraction(relativeFilePath);
134        Extraction extraction = extractionAndName.getLeft();
135        String definitionFilename = extractionAndName.getRight();
136        
137        AttributesImpl attributes = _getAttrs(definitionFilename);
138
139        ExtractionExecutionContext context = _getContext(extraction, lang, parameters);
140        
141        if (resultOutputStream != null)
142        {
143            _doExecuteForPathWithNoVar(attributes, extraction, context, resultOutputStream, pipeline);
144            return Set.of();
145        }
146        else
147        {
148            return _doExecute(attributes, extraction, relativeFilePath, context, defaultResultFileName, pipeline);
149        }
150    }
151    
152    private void _checkRights()
153    {
154        if (_rightManager.hasRight(_currentUserProvider.getUser(), ExtractionConstants.EXECUTE_EXTRACTION_RIGHT_ID, "/admin") != RightResult.RIGHT_ALLOW)
155        {
156            String errorMessage = "User " + _currentUserProvider.getUser() + " tried to execute extraction with no sufficient rights"; 
157            getLogger().error(errorMessage);
158            throw new IllegalStateException(errorMessage);
159        }
160    }
161    
162    private Pair<Extraction, String> _getExtraction(String relativeFilePath) throws Exception
163    {
164        String absoluteFilePath = ExtractionConstants.DEFINITIONS_DIR + relativeFilePath;
165        Source src = _sourceResolver.resolveURI(absoluteFilePath);
166
167        try
168        {
169            File file = ((FileSource) src).getFile();
170            
171            if (!file.exists())
172            {
173                throw new IllegalArgumentException("The file " + relativeFilePath + " does not exist.");
174            }
175            
176            Extraction extraction = _reader.readExtractionDefinitionFile(file);
177            return Pair.of(extraction, file.getName());
178        }
179        catch (Exception e)
180        {
181            throw new IllegalStateException("An unexpected error occured.", e);
182        }
183        finally
184        {
185            _sourceResolver.release(src);
186        }
187    }
188    
189    private AttributesImpl _getAttrs(String definitionFilename)
190    {
191        AttributesImpl attributes = new AttributesImpl();
192        attributes.addCDATAAttribute("user", _currentUserProvider.getUser().getLogin());
193        attributes.addCDATAAttribute("date", ZonedDateTime.now().format(DateUtils.getISODateTimeFormatter()));
194        attributes.addCDATAAttribute("name", definitionFilename);
195        return attributes;
196    }
197    
198    private ExtractionExecutionContext _getContext(Extraction extraction, String lang, Map<String, Object> parameters)
199    {
200        ExtractionExecutionContext context = new ExtractionExecutionContext();
201        if (StringUtils.isNotEmpty(lang))
202        {
203            context.setDefaultLocale(new Locale(lang));
204        }
205        context.setDisplayOptionalColumns(_getDisplayOptionalColumns(extraction.getDisplayOptionalColumnsNames(), parameters));
206        context.setClauseVariables(_getQueryVariables(extraction.getQueryVariablesNamesAndContentTypes(), parameters));
207        return context;
208    }
209
210    Map<String, Boolean> _getDisplayOptionalColumns(List<String> displayOptionalColumnsNames, Map<String, Object> parameters)
211    {
212        Map<String, Boolean> result = new HashMap<>();
213        for (String name : displayOptionalColumnsNames)
214        {
215            Boolean value = _getDipslayOptionalColumn(name, parameters);
216            if (value == null)
217            {
218                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
219            }
220            else
221            {
222                result.put(name, value);
223            }
224        }
225        return result;
226    }
227    
228    private Boolean _getDipslayOptionalColumn(String optionalColumnName, Map<String, Object> parameters)
229    {
230        Object value = parameters.get(optionalColumnName);
231        
232        if (value instanceof Boolean)
233        {
234            return (Boolean) value;
235        }
236        else if (value instanceof String)
237        {
238            return Boolean.valueOf((String) value);
239        }
240        else
241        {
242            return null;
243        }
244    }
245
246    private Map<String, List<String>> _getQueryVariables(Map<String, String> queryVariablesNamesAndContentTypes, Map<String, Object> parameters)
247    {
248        Map<String, List<String>> result = new HashMap<>();
249        for (Map.Entry<String, String> entry : queryVariablesNamesAndContentTypes.entrySet())
250        {
251            String name = entry.getKey();
252            String contentTypeId = entry.getValue();
253            if (!_contentTypeExtensionPoint.hasExtension(contentTypeId))
254            {
255                throw new IllegalArgumentException("Extraction - content type '" + contentTypeId + "' used in variable '" + name + "' definition does not exist");
256            }
257            
258            @SuppressWarnings("unchecked")
259            List<String> contentIds = (List<String>) parameters.get(name);
260            if (contentIds == null || contentIds.isEmpty())
261            {
262                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
263            }
264            
265            for (String contentId : contentIds)
266            {
267                Content content = _resolver.resolveById(contentId);
268                if (!_contentTypesHelper.isInstanceOf(content, contentTypeId))
269                {
270                    throw new IllegalArgumentException("Extraction - content '" + contentId + "' is not an instance of content type '" + contentTypeId + "', defined by the variable named '" + name + "'");
271                }
272            }
273            
274            result.put(name, contentIds);
275        }
276        return result;
277    }
278    
279    private Set<Path> _doExecute(AttributesImpl attributes, Extraction extraction, String extractionFilePath, ExtractionExecutionContext context, String defaultResultFileName, PipelineDescriptor pipeline) throws Exception
280    {
281        String unresolvedPath = pipeline.getResultSubfolder();
282        Path basePath = Paths.get(AmetysHomeHelper.getAmetysHomeData().toPath().toString(), ExtractionConstants.RESULT_EXTRACTION_DIR_NAME);
283       
284        if (_resultPathResolver.hasVariable(unresolvedPath))
285        {
286            List<ExtractionComponent> extractionComponents = extraction.getExtractionComponents();
287            boolean allAreNotTwoStepsComponent = extractionComponents.stream()
288                    .filter(Predicates.not(TwoStepsExecutingExtractionComponent.class::isInstance))
289                    .findFirst()
290                    .isPresent();
291            if (allAreNotTwoStepsComponent)
292            {
293                throw new IllegalArgumentException("The extraction " + extractionFilePath + " has an invalid component at first level which does not support a subfolder containing variables.");
294            }
295            
296            List<TwoStepsExecutingExtractionComponent> components = extractionComponents.stream()
297                    .map(TwoStepsExecutingExtractionComponent.class::cast)
298                    .collect(Collectors.toList());
299            return _doExecuteForPathWithVar(attributes, extraction, context, components, basePath, unresolvedPath, pipeline);
300        }
301        else
302        {
303            Path fileOrFolderPath = _resultPathResolver.resolvePath(unresolvedPath, null, extraction, basePath).keySet().iterator().next();
304            Path filePath = _filePathWhenNoVar(fileOrFolderPath, defaultResultFileName);
305            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);)
306            {
307                _doExecuteForPathWithNoVar(attributes, extraction, context, fileOs, pipeline);
308                return Set.of(filePath);
309            }
310        }
311    }
312    
313    private Path _filePathWhenNoVar(Path fileOrFolderPath, String defaultFileName)
314    {
315        Path fileName = fileOrFolderPath.getFileName();
316        if (fileName.toString().contains("."))
317        {
318            // is already a file, do not use default file name
319            return fileOrFolderPath;
320        }
321        return Paths.get(fileOrFolderPath.toString(), defaultFileName);
322    }
323    
324    private void _doExecuteForPathWithNoVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, OutputStream outputStream, PipelineDescriptor pipelineDescriptor) throws Exception
325    {
326        try (Pipeline pipeline = pipelineDescriptor.newPipeline(outputStream);)
327        {
328            ContentHandler contentHandler = pipeline.getHandler();
329            _noVarExecute(contentHandler, attributes, extraction, context);
330            pipeline.serialize();
331        }
332    }
333    
334    private void _noVarExecute(ContentHandler contentHandler, AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context) throws Exception
335    {
336        contentHandler.startDocument();
337        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
338        
339        for (ExtractionComponent component : extraction.getExtractionComponents())
340        {
341            component.prepareComponentExecution(context);
342            component.execute(contentHandler, context);
343        }
344        
345        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
346        contentHandler.endDocument();
347    }
348    
349    private Set<Path> _doExecuteForPathWithVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, Path basePath, String unresolvedPath, PipelineDescriptor pipelineDescriptor) throws Exception
350    {
351        Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent = new HashMap<>();
352        for (TwoStepsExecutingExtractionComponent component : components)
353        {
354            component.prepareComponentExecution(context);
355            List<Content> firstLevelResults = StreamSupport.stream(component.computeFirstLevelResults(context).spliterator(), false)
356                    .collect(Collectors.toList());
357            firstLevelResultsByComponent.put(component, firstLevelResults);
358        }
359        
360        List<Content> allContents = firstLevelResultsByComponent.values()
361                .stream()
362                .flatMap(List::stream)
363                .collect(Collectors.toList());
364        
365        String unresolvedFilePath = _unresolvedFilePathWhenVar(unresolvedPath, pipelineDescriptor);
366        Map<Path, List<Content>> paths = _resultPathResolver.resolvePath(unresolvedFilePath, allContents, extraction, basePath);
367        for (Path filePath : paths.keySet())
368        {
369            List<Content> involvedContentsForPath = paths.get(filePath);
370            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);
371                 Pipeline pipeline = pipelineDescriptor.newPipeline(fileOs);)
372            {
373                ContentHandler contentHandler = pipeline.getHandler();
374                _withVarExecute(contentHandler, attributes, context, components, involvedContentsForPath, firstLevelResultsByComponent);
375                pipeline.serialize();
376            }
377        }
378        
379        return paths.keySet();
380    }
381    
382    private String _unresolvedFilePathWhenVar(String unresolvedFolderOrFilePath, PipelineDescriptor pipelineDescriptor)
383    {
384        if (_resultPathResolver.isFolder(unresolvedFolderOrFilePath))
385        {
386            return unresolvedFolderOrFilePath + "/${title}." + pipelineDescriptor.getDefaultExtension();
387        }
388        else
389        {
390            return unresolvedFolderOrFilePath;
391        }
392    }
393    
394    private void _withVarExecute(ContentHandler contentHandler, AttributesImpl attributes, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, List<Content> involvedContentsForPath, Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent) throws Exception
395    {
396        contentHandler.startDocument();
397        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
398        
399        for (TwoStepsExecutingExtractionComponent component : components)
400        {
401            List<Content> firstLevelResults = firstLevelResultsByComponent.get(component);
402            List<Content> involvedFirstLevelResults = ListUtils.intersection(firstLevelResults, involvedContentsForPath); 
403            component.executeFor(contentHandler, involvedFirstLevelResults, context);
404        }
405        
406        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
407        contentHandler.endDocument();
408    }
409}