Source code

001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.extraction.execution;
017
018import java.io.File;
019import java.io.OutputStream;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.Optional;
028import java.util.Set;
029import java.util.stream.Collectors;
030import java.util.stream.StreamSupport;
031
032import org.apache.avalon.framework.component.Component;
033import org.apache.avalon.framework.service.ServiceException;
034import org.apache.avalon.framework.service.ServiceManager;
035import org.apache.avalon.framework.service.Serviceable;
036import org.apache.cocoon.xml.AttributesImpl;
037import org.apache.cocoon.xml.XMLUtils;
038import org.apache.commons.collections4.ListUtils;
039import org.apache.commons.lang3.StringUtils;
040import org.apache.commons.lang3.tuple.Pair;
041import org.apache.excalibur.source.Source;
042import org.apache.excalibur.source.SourceResolver;
043import org.apache.excalibur.source.impl.FileSource;
044import org.xml.sax.ContentHandler;
045
046import org.ametys.cms.contenttype.ContentTypesHelper;
047import org.ametys.cms.repository.Content;
048import org.ametys.core.right.RightManager;
049import org.ametys.core.right.RightManager.RightResult;
050import org.ametys.core.user.CurrentUserProvider;
051import org.ametys.core.util.DateUtils;
052import org.ametys.plugins.extraction.ExtractionConstants;
053import org.ametys.plugins.extraction.component.ExtractionComponent;
054import org.ametys.plugins.extraction.component.TwoStepsExecutingExtractionComponent;
055import org.ametys.plugins.extraction.execution.Extraction.ClausesVariable;
056import org.ametys.plugins.extraction.execution.Extraction.ClausesVariableType;
057import org.ametys.plugins.extraction.execution.pipeline.Pipeline;
058import org.ametys.plugins.extraction.execution.pipeline.PipelineDescriptor;
059import org.ametys.plugins.extraction.execution.pipeline.Pipelines;
060import org.ametys.plugins.repository.AmetysObjectResolver;
061import org.ametys.runtime.plugin.component.AbstractLogEnabled;
062import org.ametys.runtime.util.AmetysHomeHelper;
063
064import com.google.common.base.Predicates;
065
066/**
067 * Extracts query results form a XML definition file
068 */
069public class ExtractionExecutor extends AbstractLogEnabled implements Component, Serviceable
070{
071    /** The Avalon role. */
072    public static final String ROLE = ExtractionExecutor.class.getName();
073    
074    private RightManager _rightManager;
075    private ExtractionDefinitionReader _reader;
076    private CurrentUserProvider _currentUserProvider;
077    private AmetysObjectResolver _resolver;
078    private ContentTypesHelper _contentTypesHelper;
079    private SourceResolver _sourceResolver;
080    private PathResolver _resultPathResolver;
081
082    @Override
083    public void service(ServiceManager serviceManager) throws ServiceException
084    {
085        _rightManager = (RightManager) serviceManager.lookup(RightManager.ROLE);
086        _reader = (ExtractionDefinitionReader) serviceManager.lookup(ExtractionDefinitionReader.ROLE);
087        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
088        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
089        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
090        _sourceResolver = (SourceResolver) serviceManager.lookup(SourceResolver.ROLE);
091        _resultPathResolver = (PathResolver) serviceManager.lookup(PathResolver.ROLE);
092    }
093    
094    /**
095     * Executes the given extraction
096     * @param relativeFilePath The path of the extraction file to execute
097     * @param defaultResultFileName The default file name for the result (it can be unused, for instance if resultSubFolder is a file, i.e. contains a '.' in its last element)
098     * @param lang The language
099     * @param parameters The parameters
100     * @param pipeline The execution pipeline
101     * @return The set of the result files path
102     * @throws Exception if an errors occurred
103     */
104    public Set<Path> execute(String relativeFilePath, String defaultResultFileName, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
105    {
106        return _execute(relativeFilePath, defaultResultFileName, null, lang, parameters, pipeline);
107    }
108    
109    /**
110     * Executes the given extraction
111     * @param relativeFilePath The path of the extraction file to execute
112     * @param resultOutputStream The result output stream
113     * @param lang The language
114     * @param parameters The parameters
115     * @param pipeline The execution pipeline
116     * @throws Exception if an errors occurred
117     */
118    public void execute(String relativeFilePath, OutputStream resultOutputStream, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
119    {
120        _execute(relativeFilePath, null, resultOutputStream, lang, parameters, pipeline);
121    }
122    
123    private Set<Path> _execute(
124            String relativeFilePath,
125            String defaultResultFileName, 
126            OutputStream resultOutputStream, 
127            String lang, 
128            Map<String, Object> parameters, 
129            PipelineDescriptor pipeline) throws Exception
130    {
131        _checkRights();
132        
133        Pair<Extraction, String> extractionAndName = _getExtraction(relativeFilePath);
134        Extraction extraction = extractionAndName.getLeft();
135        String definitionFilename = extractionAndName.getRight();
136        
137        AttributesImpl attributes = _getAttrs(definitionFilename);
138
139        ExtractionExecutionContext context = _getContext(extraction, lang, parameters);
140        
141        if (resultOutputStream != null)
142        {
143            _doExecuteForPathWithNoVar(attributes, extraction, context, resultOutputStream, pipeline);
144            return Set.of();
145        }
146        else
147        {
148            return _doExecute(attributes, extraction, relativeFilePath, context, defaultResultFileName, pipeline);
149        }
150    }
151    
152    private void _checkRights()
153    {
154        if (_rightManager.hasRight(_currentUserProvider.getUser(), ExtractionConstants.EXECUTE_EXTRACTION_RIGHT_ID, "/admin") != RightResult.RIGHT_ALLOW)
155        {
156            String errorMessage = "User " + _currentUserProvider.getUser() + " tried to execute extraction with no sufficient rights"; 
157            getLogger().error(errorMessage);
158            throw new IllegalStateException(errorMessage);
159        }
160    }
161    
162    private Pair<Extraction, String> _getExtraction(String relativeFilePath) throws Exception
163    {
164        String absoluteFilePath = ExtractionConstants.DEFINITIONS_DIR + relativeFilePath;
165        Source src = _sourceResolver.resolveURI(absoluteFilePath);
166
167        try
168        {
169            File file = ((FileSource) src).getFile();
170            
171            if (!file.exists())
172            {
173                throw new IllegalArgumentException("The file " + relativeFilePath + " does not exist.");
174            }
175            
176            Extraction extraction = _reader.readExtractionDefinitionFile(file);
177            return Pair.of(extraction, file.getName());
178        }
179        catch (Exception e)
180        {
181            throw new IllegalStateException("An unexpected error occured.", e);
182        }
183        finally
184        {
185            _sourceResolver.release(src);
186        }
187    }
188    
189    private AttributesImpl _getAttrs(String definitionFilename)
190    {
191        AttributesImpl attributes = new AttributesImpl();
192        attributes.addCDATAAttribute("user", _currentUserProvider.getUser().getLogin());
193        attributes.addCDATAAttribute("date", ZonedDateTime.now().format(DateUtils.getISODateTimeFormatter()));
194        attributes.addCDATAAttribute("name", definitionFilename);
195        return attributes;
196    }
197    
198    private ExtractionExecutionContext _getContext(Extraction extraction, String lang, Map<String, Object> parameters)
199    {
200        ExtractionExecutionContext context = new ExtractionExecutionContext();
201        if (StringUtils.isNotEmpty(lang))
202        {
203            context.setDefaultLocale(new Locale(lang));
204        }
205        context.setDisplayOptionalColumns(_getDisplayOptionalColumns(extraction.getDisplayOptionalColumnsNames(), parameters));
206        context.setClausesVariablesValues(_getClausesVariablesValues(extraction.getClausesVariables(), parameters));
207        return context;
208    }
209
210    Map<String, Boolean> _getDisplayOptionalColumns(List<String> displayOptionalColumnsNames, Map<String, Object> parameters)
211    {
212        Map<String, Boolean> result = new HashMap<>();
213        for (String name : displayOptionalColumnsNames)
214        {
215            Boolean value = _getDipslayOptionalColumn(name, parameters);
216            if (value == null)
217            {
218                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
219            }
220            else
221            {
222                result.put(name, value);
223            }
224        }
225        return result;
226    }
227    
228    private Boolean _getDipslayOptionalColumn(String optionalColumnName, Map<String, Object> parameters)
229    {
230        Object value = parameters.get(optionalColumnName);
231        
232        if (value instanceof Boolean)
233        {
234            return (Boolean) value;
235        }
236        else if (value instanceof String)
237        {
238            return Boolean.valueOf((String) value);
239        }
240        else
241        {
242            return null;
243        }
244    }
245
246    private Map<ClausesVariable, List<String>> _getClausesVariablesValues(List<ClausesVariable> clausesVariables, Map<String, Object> parameters)
247    {
248        Map<ClausesVariable, List<String>> result = new HashMap<>();
249        for (ClausesVariable clausesVariable : clausesVariables)
250        {
251            if (ClausesVariableType.SELECT_CONTENTS.equals(clausesVariable.type()))
252            {
253                // Only one content type for SELECT_CONTENTS clauses variables
254                Optional<String> contentTypeId = clausesVariable.contentTypeIds()
255                                                                .stream()
256                                                                .findFirst();
257                
258                @SuppressWarnings("unchecked")
259                List<String> contentIds = (List<String>) parameters.get(clausesVariable.name());
260                if (contentIds == null || contentIds.isEmpty())
261                {
262                    throw new IllegalArgumentException("Extraction - There is a variable named '" + clausesVariable.name() + "' but there is no corresponding value");
263                }
264                
265                // If a content type is provided, check all contents' types
266                if (contentTypeId.isPresent())
267                {
268                    for (String contentId : contentIds)
269                    {
270                        Content content = _resolver.resolveById(contentId);
271                        if (!_contentTypesHelper.isInstanceOf(content, contentTypeId.get()))
272                        {
273                            throw new IllegalArgumentException("Extraction - content '" + contentId + "' is not an instance of content type '" + contentTypeId.get() + "', defined by the variable named '" + clausesVariable.name() + "'");
274                        }
275                    }
276                }
277                
278                result.put(clausesVariable, contentIds);
279            }
280            else
281            {
282                String solrRequest = (String) parameters.get(clausesVariable.name());
283                if (StringUtils.isEmpty(solrRequest))
284                {
285                    throw new IllegalArgumentException("Extraction - There is a variable named '" + clausesVariable.name() + "' but there is no corresponding value");
286                }
287                
288                result.put(clausesVariable, List.of(solrRequest));
289            }
290        }
291        return result;
292    }
293    
294    private Set<Path> _doExecute(AttributesImpl attributes, Extraction extraction, String extractionFilePath, ExtractionExecutionContext context, String defaultResultFileName, PipelineDescriptor pipeline) throws Exception
295    {
296        String unresolvedPath = pipeline.getResultSubfolder();
297        Path basePath = Paths.get(AmetysHomeHelper.getAmetysHomeData().toPath().toString(), ExtractionConstants.RESULT_EXTRACTION_DIR_NAME);
298       
299        if (_resultPathResolver.hasVariable(unresolvedPath))
300        {
301            List<ExtractionComponent> extractionComponents = extraction.getExtractionComponents();
302            boolean allAreNotTwoStepsComponent = extractionComponents.stream()
303                    .filter(Predicates.not(TwoStepsExecutingExtractionComponent.class::isInstance))
304                    .findFirst()
305                    .isPresent();
306            if (allAreNotTwoStepsComponent)
307            {
308                throw new IllegalArgumentException("The extraction " + extractionFilePath + " has an invalid component at first level which does not support a subfolder containing variables.");
309            }
310            
311            List<TwoStepsExecutingExtractionComponent> components = extractionComponents.stream()
312                    .map(TwoStepsExecutingExtractionComponent.class::cast)
313                    .collect(Collectors.toList());
314            return _doExecuteForPathWithVar(attributes, extraction, context, components, basePath, unresolvedPath, pipeline);
315        }
316        else
317        {
318            Path fileOrFolderPath = _resultPathResolver.resolvePath(unresolvedPath, null, extraction, basePath).keySet().iterator().next();
319            Path filePath = _filePathWhenNoVar(fileOrFolderPath, defaultResultFileName);
320            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);)
321            {
322                _doExecuteForPathWithNoVar(attributes, extraction, context, fileOs, pipeline);
323                return Set.of(filePath);
324            }
325        }
326    }
327    
328    private Path _filePathWhenNoVar(Path fileOrFolderPath, String defaultFileName)
329    {
330        Path fileName = fileOrFolderPath.getFileName();
331        if (fileName.toString().contains("."))
332        {
333            // is already a file, do not use default file name
334            return fileOrFolderPath;
335        }
336        return Paths.get(fileOrFolderPath.toString(), defaultFileName);
337    }
338    
339    private void _doExecuteForPathWithNoVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, OutputStream outputStream, PipelineDescriptor pipelineDescriptor) throws Exception
340    {
341        try (Pipeline pipeline = pipelineDescriptor.newPipeline(outputStream);)
342        {
343            ContentHandler contentHandler = pipeline.getHandler();
344            _noVarExecute(contentHandler, attributes, extraction, context);
345            pipeline.serialize();
346        }
347    }
348    
349    private void _noVarExecute(ContentHandler contentHandler, AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context) throws Exception
350    {
351        contentHandler.startDocument();
352        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
353        
354        for (ExtractionComponent component : extraction.getExtractionComponents())
355        {
356            component.prepareComponentExecution(context);
357            component.execute(contentHandler, context);
358        }
359        
360        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
361        contentHandler.endDocument();
362    }
363    
364    private Set<Path> _doExecuteForPathWithVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, Path basePath, String unresolvedPath, PipelineDescriptor pipelineDescriptor) throws Exception
365    {
366        Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent = new HashMap<>();
367        for (TwoStepsExecutingExtractionComponent component : components)
368        {
369            component.prepareComponentExecution(context);
370            List<Content> firstLevelResults = StreamSupport.stream(component.computeFirstLevelResults(context).spliterator(), false)
371                    .collect(Collectors.toList());
372            firstLevelResultsByComponent.put(component, firstLevelResults);
373        }
374        
375        List<Content> allContents = firstLevelResultsByComponent.values()
376                .stream()
377                .flatMap(List::stream)
378                .collect(Collectors.toList());
379        
380        String unresolvedFilePath = _unresolvedFilePathWhenVar(unresolvedPath, pipelineDescriptor);
381        Map<Path, List<Content>> paths = _resultPathResolver.resolvePath(unresolvedFilePath, allContents, extraction, basePath);
382        for (Path filePath : paths.keySet())
383        {
384            List<Content> involvedContentsForPath = paths.get(filePath);
385            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);
386                 Pipeline pipeline = pipelineDescriptor.newPipeline(fileOs);)
387            {
388                ContentHandler contentHandler = pipeline.getHandler();
389                _withVarExecute(contentHandler, attributes, context, components, involvedContentsForPath, firstLevelResultsByComponent);
390                pipeline.serialize();
391            }
392        }
393        
394        return paths.keySet();
395    }
396    
397    private String _unresolvedFilePathWhenVar(String unresolvedFolderOrFilePath, PipelineDescriptor pipelineDescriptor)
398    {
399        if (_resultPathResolver.isFolder(unresolvedFolderOrFilePath))
400        {
401            return unresolvedFolderOrFilePath + "/${title}." + pipelineDescriptor.getDefaultExtension();
402        }
403        else
404        {
405            return unresolvedFolderOrFilePath;
406        }
407    }
408    
409    private void _withVarExecute(ContentHandler contentHandler, AttributesImpl attributes, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, List<Content> involvedContentsForPath, Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent) throws Exception
410    {
411        contentHandler.startDocument();
412        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
413        
414        for (TwoStepsExecutingExtractionComponent component : components)
415        {
416            List<Content> firstLevelResults = firstLevelResultsByComponent.get(component);
417            List<Content> involvedFirstLevelResults = ListUtils.intersection(firstLevelResults, involvedContentsForPath); 
418            component.executeFor(contentHandler, involvedFirstLevelResults, context);
419        }
420        
421        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
422        contentHandler.endDocument();
423    }
424}