Source code

001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.extraction.execution;
017
018import java.io.File;
019import java.io.OutputStream;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.stream.Collectors;
028import java.util.stream.StreamSupport;
029
030import org.apache.avalon.framework.component.Component;
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.avalon.framework.service.Serviceable;
034import org.apache.cocoon.xml.AttributesImpl;
035import org.apache.cocoon.xml.XMLUtils;
036import org.apache.commons.collections4.ListUtils;
037import org.apache.commons.lang3.StringUtils;
038import org.apache.commons.lang3.tuple.Pair;
039import org.apache.excalibur.source.Source;
040import org.apache.excalibur.source.SourceResolver;
041import org.apache.excalibur.source.impl.FileSource;
042import org.xml.sax.ContentHandler;
043
044import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
045import org.ametys.cms.contenttype.ContentTypesHelper;
046import org.ametys.cms.repository.Content;
047import org.ametys.core.right.RightManager;
048import org.ametys.core.right.RightManager.RightResult;
049import org.ametys.core.user.CurrentUserProvider;
050import org.ametys.core.util.DateUtils;
051import org.ametys.plugins.extraction.ExtractionConstants;
052import org.ametys.plugins.extraction.component.ExtractionComponent;
053import org.ametys.plugins.extraction.component.TwoStepsExecutingExtractionComponent;
054import org.ametys.plugins.extraction.execution.pipeline.Pipeline;
055import org.ametys.plugins.extraction.execution.pipeline.PipelineDescriptor;
056import org.ametys.plugins.extraction.execution.pipeline.Pipelines;
057import org.ametys.plugins.repository.AmetysObjectResolver;
058import org.ametys.runtime.plugin.component.AbstractLogEnabled;
059import org.ametys.runtime.util.AmetysHomeHelper;
060
061import com.google.common.base.Predicates;
062
063/**
064 * Extracts query results form a XML definition file
065 */
066public class ExtractionExecutor extends AbstractLogEnabled implements Component, Serviceable
067{
068    /** The Avalon role. */
069    public static final String ROLE = ExtractionExecutor.class.getName();
070    
071    /** The directory under ametys home data directory for extraction results */
072    public static final String EXTRACTION_DIR_NAME = "extraction";
073
074    private RightManager _rightManager;
075    private ExtractionDefinitionReader _reader;
076    private CurrentUserProvider _currentUserProvider;
077    private ContentTypeExtensionPoint _contentTypeExtensionPoint;
078    private AmetysObjectResolver _resolver;
079    private ContentTypesHelper _contentTypesHelper;
080    private SourceResolver _sourceResolver;
081    private PathResolver _resultPathResolver;
082
083    @Override
084    public void service(ServiceManager serviceManager) throws ServiceException
085    {
086        _rightManager = (RightManager) serviceManager.lookup(RightManager.ROLE);
087        _reader = (ExtractionDefinitionReader) serviceManager.lookup(ExtractionDefinitionReader.ROLE);
088        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
089        _contentTypeExtensionPoint = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
090        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
091        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
092        _sourceResolver = (SourceResolver) serviceManager.lookup(SourceResolver.ROLE);
093        _resultPathResolver = (PathResolver) serviceManager.lookup(PathResolver.ROLE);
094    }
095    
096    /**
097     * Executes the given extraction
098     * @param fileName The name of the extraction file to execute
099     * @param defaultResultFileName The default file name for the result (it can be unused, for instance if resultSubFolder is a file, i.e. contains a '.' in its last element)
100     * @param lang The lang
101     * @param parameters The parameters
102     * @param pipeline The execution pipeline
103     * @throws Exception if an errors occured
104     */
105    public void execute(String fileName, String defaultResultFileName, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
106    {
107        _execute(fileName, defaultResultFileName, null, lang, parameters, pipeline);
108    }
109    
110    /**
111     * Executes the given extraction
112     * @param fileName The name of the extraction file to execute
113     * @param resultOutputStream The result output stream
114     * @param lang The lang
115     * @param parameters The parameters
116     * @param pipeline The execution pipeline
117     * @throws Exception if an errors occured
118     */
119    public void execute(String fileName, OutputStream resultOutputStream, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
120    {
121        _execute(fileName, null, resultOutputStream, lang, parameters, pipeline);
122    }
123    
124    private void _execute(
125            String fileName,
126            String defaultResultFileName, 
127            OutputStream resultOutputStream, 
128            String lang, 
129            Map<String, Object> parameters, 
130            PipelineDescriptor pipeline) throws Exception
131    {
132        _checkRights();
133        
134        Pair<Extraction, String> extractionAndName = _getExtraction(fileName);
135        Extraction extraction = extractionAndName.getLeft();
136        String definitionFilename = extractionAndName.getRight();
137        
138        AttributesImpl attributes = _getAttrs(definitionFilename);
139
140        ExtractionExecutionContext context = _getContext(extraction, lang, parameters);
141        
142        if (resultOutputStream != null)
143        {
144            _doExecuteForPathWithNoVar(attributes, extraction, context, resultOutputStream, pipeline);
145        }
146        else
147        {
148            _doExecute(attributes, extraction, fileName, context, defaultResultFileName, pipeline);
149        }
150    }
151    
152    private void _checkRights()
153    {
154        if (_rightManager.hasRight(_currentUserProvider.getUser(), ExtractionConstants.EXECUTE_EXTRACTION_RIGHT_ID, "/admin") != RightResult.RIGHT_ALLOW)
155        {
156            String errorMessage = "User " + _currentUserProvider.getUser() + " tried to execute extraction with no sufficient rights"; 
157            getLogger().error(errorMessage);
158            throw new IllegalStateException(errorMessage);
159        }
160    }
161    
162    private Pair<Extraction, String> _getExtraction(String fileName) throws Exception
163    {
164        String filePath = ExtractionConstants.DEFINITIONS_DIR + fileName;
165        
166        Source src = _sourceResolver.resolveURI(filePath);
167        Extraction extraction = null;
168        String definitionFilename = null;
169        try
170        {
171            File file = ((FileSource) src).getFile();
172            
173            if (!file.exists())
174            {
175                throw new IllegalArgumentException("The file " + filePath + " does not exist.");
176            }
177            
178            extraction = _reader.readExtractionDefinitionFile(file);
179            definitionFilename = file.getName();
180            return Pair.of(extraction, definitionFilename);
181        }
182        catch (Exception e)
183        {
184            throw new IllegalStateException("An unexpected error occured.", e);
185        }
186        finally
187        {
188            _sourceResolver.release(src);
189        }
190    }
191    
192    private AttributesImpl _getAttrs(String definitionFilename)
193    {
194        AttributesImpl attributes = new AttributesImpl();
195        attributes.addCDATAAttribute("user", _currentUserProvider.getUser().getLogin());
196        attributes.addCDATAAttribute("date", ZonedDateTime.now().format(DateUtils.getISODateTimeFormatter()));
197        attributes.addCDATAAttribute("name", definitionFilename);
198        return attributes;
199    }
200    
201    private ExtractionExecutionContext _getContext(Extraction extraction, String lang, Map<String, Object> parameters)
202    {
203        ExtractionExecutionContext context = new ExtractionExecutionContext();
204        if (StringUtils.isNotEmpty(lang))
205        {
206            context.setDefaultLocale(new Locale(lang));
207        }
208        context.setDisplayOptionalColumns(_getDisplayOptionalColumns(extraction.getDisplayOptionalColumnsNames(), parameters));
209        context.setClauseVariables(_getQueryVariables(extraction.getQueryVariablesNamesAndContentTypes(), parameters));
210        return context;
211    }
212
213    Map<String, Boolean> _getDisplayOptionalColumns(List<String> displayOptionalColumnsNames, Map<String, Object> parameters)
214    {
215        Map<String, Boolean> result = new HashMap<>();
216        for (String name : displayOptionalColumnsNames)
217        {
218            Boolean value = _getDipslayOptionalColumn(name, parameters);
219            if (value == null)
220            {
221                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
222            }
223            else
224            {
225                result.put(name, value);
226            }
227        }
228        return result;
229    }
230    
231    private Boolean _getDipslayOptionalColumn(String optionalColumnName, Map<String, Object> parameters)
232    {
233        Object value = parameters.get(optionalColumnName);
234        
235        if (value instanceof Boolean)
236        {
237            return (Boolean) value;
238        }
239        else if (value instanceof String)
240        {
241            return Boolean.valueOf((String) value);
242        }
243        else
244        {
245            return null;
246        }
247    }
248
249    private Map<String, String> _getQueryVariables(Map<String, String> queryVariablesNamesAndContentTypes, Map<String, Object> parameters)
250    {
251        Map<String, String> result = new HashMap<>();
252        for (Map.Entry<String, String> entry : queryVariablesNamesAndContentTypes.entrySet())
253        {
254            String name = entry.getKey();
255            String contentTypeId = entry.getValue();
256            if (!_contentTypeExtensionPoint.hasExtension(contentTypeId))
257            {
258                throw new IllegalArgumentException("Extraction - content type '" + contentTypeId + "' used in variable '" + name + "' definition does not exist");
259            }
260            
261            String contentId = (String) parameters.get(name);
262            if (null == contentId)
263            {
264                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
265            }
266            
267            Content content = _resolver.resolveById(contentId);
268            if (!_contentTypesHelper.isInstanceOf(content, contentTypeId))
269            {
270                throw new IllegalArgumentException("Extraction - content '" + contentId + "' is not an instance of content type '" + contentTypeId + "', defined by the variable named '" + name + "'");
271            }
272            
273            result.put(name, contentId);
274        }
275        return result;
276    }
277    
278    private void _doExecute(AttributesImpl attributes, Extraction extraction, String extractionName, ExtractionExecutionContext context, String defaultResultFileName, PipelineDescriptor pipeline) throws Exception
279    {
280        String unresolvedPath = pipeline.getResultSubfolder();
281        Path basePath = Paths.get(AmetysHomeHelper.getAmetysHomeData().toPath().toString(), EXTRACTION_DIR_NAME);
282        if (_resultPathResolver.hasVariable(unresolvedPath))
283        {
284            List<ExtractionComponent> extractionComponents = extraction.getExtractionComponents();
285            boolean allAreNotTwoStepsComponent = extractionComponents.stream()
286                    .filter(Predicates.not(TwoStepsExecutingExtractionComponent.class::isInstance))
287                    .findFirst()
288                    .isPresent();
289            if (allAreNotTwoStepsComponent)
290            {
291                throw new IllegalArgumentException("The extraction " + extractionName + " has an invalid component at first level which does not support a subfolder containing variables.");
292            }
293            
294            List<TwoStepsExecutingExtractionComponent> components = extractionComponents.stream()
295                    .map(TwoStepsExecutingExtractionComponent.class::cast)
296                    .collect(Collectors.toList());
297            _doExecuteForPathWithVar(attributes, extraction, context, components, basePath, unresolvedPath, pipeline);
298        }
299        else
300        {
301            Path fileOrFolderPath = _resultPathResolver.resolvePath(unresolvedPath, null, extraction, basePath).keySet().iterator().next();
302            Path filePath = _filePathWhenNoVar(fileOrFolderPath, defaultResultFileName);
303            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);)
304            {
305                _doExecuteForPathWithNoVar(attributes, extraction, context, fileOs, pipeline);
306            }
307        }
308    }
309    
310    private Path _filePathWhenNoVar(Path fileOrFolderPath, String defaultFileName)
311    {
312        Path fileName = fileOrFolderPath.getFileName();
313        if (fileName.toString().contains("."))
314        {
315            // is already a file, do not use default file name
316            return fileOrFolderPath;
317        }
318        return Paths.get(fileOrFolderPath.toString(), defaultFileName);
319    }
320    
321    private void _doExecuteForPathWithNoVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, OutputStream outputStream, PipelineDescriptor pipelineDescriptor) throws Exception
322    {
323        try (Pipeline pipeline = pipelineDescriptor.newPipeline(outputStream);)
324        {
325            ContentHandler contentHandler = pipeline.getHandler();
326            _noVarExecute(contentHandler, attributes, extraction, context);
327            pipeline.serialize();
328        }
329    }
330    
331    private void _noVarExecute(ContentHandler contentHandler, AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context) throws Exception
332    {
333        contentHandler.startDocument();
334        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
335        
336        for (ExtractionComponent component : extraction.getExtractionComponents())
337        {
338            component.prepareComponentExecution(context);
339            component.execute(contentHandler, context);
340        }
341        
342        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
343        contentHandler.endDocument();
344    }
345    
346    private void _doExecuteForPathWithVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, Path basePath, String unresolvedPath, PipelineDescriptor pipelineDescriptor) throws Exception
347    {
348        Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent = new HashMap<>();
349        for (TwoStepsExecutingExtractionComponent component : components)
350        {
351            component.prepareComponentExecution(context);
352            List<Content> firstLevelResults = StreamSupport.stream(component.computeFirstLevelResults(context).spliterator(), false)
353                    .collect(Collectors.toList());
354            firstLevelResultsByComponent.put(component, firstLevelResults);
355        }
356        
357        List<Content> allContents = firstLevelResultsByComponent.values()
358                .stream()
359                .flatMap(List::stream)
360                .collect(Collectors.toList());
361        
362        String unresolvedFilePath = _unresolvedFilePathWhenVar(unresolvedPath, pipelineDescriptor);
363        Map<Path, List<Content>> paths = _resultPathResolver.resolvePath(unresolvedFilePath, allContents, extraction, basePath);
364        for (Path filePath : paths.keySet())
365        {
366            List<Content> involvedContentsForPath = paths.get(filePath);
367            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);
368                 Pipeline pipeline = pipelineDescriptor.newPipeline(fileOs);)
369            {
370                ContentHandler contentHandler = pipeline.getHandler();
371                _withVarExecute(contentHandler, attributes, context, components, involvedContentsForPath, firstLevelResultsByComponent);
372                pipeline.serialize();
373            }
374        }
375    }
376    
377    private String _unresolvedFilePathWhenVar(String unresolvedFolderOrFilePath, PipelineDescriptor pipelineDescriptor)
378    {
379        if (_resultPathResolver.isFolder(unresolvedFolderOrFilePath))
380        {
381            return unresolvedFolderOrFilePath + "/${title}." + pipelineDescriptor.getDefaultExtension();
382        }
383        else
384        {
385            return unresolvedFolderOrFilePath;
386        }
387    }
388    
389    private void _withVarExecute(ContentHandler contentHandler, AttributesImpl attributes, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, List<Content> involvedContentsForPath, Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent) throws Exception
390    {
391        contentHandler.startDocument();
392        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
393        
394        for (TwoStepsExecutingExtractionComponent component : components)
395        {
396            List<Content> firstLevelResults = firstLevelResultsByComponent.get(component);
397            List<Content> involvedFirstLevelResults = ListUtils.intersection(firstLevelResults, involvedContentsForPath); 
398            component.executeFor(contentHandler, involvedFirstLevelResults, context);
399        }
400        
401        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
402        contentHandler.endDocument();
403    }
404}