Source code

001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.extraction.execution;
017
018import java.io.File;
019import java.io.OutputStream;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.stream.Collectors;
028import java.util.stream.StreamSupport;
029
030import org.apache.avalon.framework.component.Component;
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.avalon.framework.service.Serviceable;
034import org.apache.cocoon.xml.AttributesImpl;
035import org.apache.cocoon.xml.XMLUtils;
036import org.apache.commons.collections4.ListUtils;
037import org.apache.commons.lang3.StringUtils;
038import org.apache.commons.lang3.tuple.Pair;
039import org.apache.excalibur.source.Source;
040import org.apache.excalibur.source.SourceResolver;
041import org.apache.excalibur.source.impl.FileSource;
042import org.xml.sax.ContentHandler;
043
044import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
045import org.ametys.cms.contenttype.ContentTypesHelper;
046import org.ametys.cms.repository.Content;
047import org.ametys.core.right.RightManager;
048import org.ametys.core.right.RightManager.RightResult;
049import org.ametys.core.user.CurrentUserProvider;
050import org.ametys.core.util.DateUtils;
051import org.ametys.plugins.extraction.ExtractionConstants;
052import org.ametys.plugins.extraction.component.ExtractionComponent;
053import org.ametys.plugins.extraction.component.TwoStepsExecutingExtractionComponent;
054import org.ametys.plugins.extraction.execution.pipeline.Pipeline;
055import org.ametys.plugins.extraction.execution.pipeline.PipelineDescriptor;
056import org.ametys.plugins.extraction.execution.pipeline.Pipelines;
057import org.ametys.plugins.repository.AmetysObjectResolver;
058import org.ametys.runtime.plugin.component.AbstractLogEnabled;
059import org.ametys.runtime.util.AmetysHomeHelper;
060
061import com.google.common.base.Predicates;
062
063/**
064 * Extracts query results form a XML definition file
065 */
066public class ExtractionExecutor extends AbstractLogEnabled implements Component, Serviceable
067{
068    /** The Avalon role. */
069    public static final String ROLE = ExtractionExecutor.class.getName();
070    
071    /** The directory under ametys home data directory for extraction results */
072    public static final String EXTRACTION_DIR_NAME = "extraction";
073
074    private RightManager _rightManager;
075    private ExtractionDefinitionReader _reader;
076    private CurrentUserProvider _currentUserProvider;
077    private ContentTypeExtensionPoint _contentTypeExtensionPoint;
078    private AmetysObjectResolver _resolver;
079    private ContentTypesHelper _contentTypesHelper;
080    private SourceResolver _sourceResolver;
081    private PathResolver _resultPathResolver;
082
083    @Override
084    public void service(ServiceManager serviceManager) throws ServiceException
085    {
086        _rightManager = (RightManager) serviceManager.lookup(RightManager.ROLE);
087        _reader = (ExtractionDefinitionReader) serviceManager.lookup(ExtractionDefinitionReader.ROLE);
088        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
089        _contentTypeExtensionPoint = (ContentTypeExtensionPoint) serviceManager.lookup(ContentTypeExtensionPoint.ROLE);
090        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
091        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
092        _sourceResolver = (SourceResolver) serviceManager.lookup(SourceResolver.ROLE);
093        _resultPathResolver = (PathResolver) serviceManager.lookup(PathResolver.ROLE);
094    }
095    
096    /**
097     * Executes the given extraction
098     * @param fileName The name of the extraction file to execute
099     * @param defaultResultFileName The default file name for the result (it can be unused, for instance if resultSubFolder is a file, i.e. contains a '.' in its last element)
100     * @param lang The lang
101     * @param parameters The parameters
102     * @param pipeline The execution pipeline
103     * @throws Exception if an errors occured
104     */
105    public void execute(String fileName, String defaultResultFileName, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
106    {
107        _execute(fileName, defaultResultFileName, null, lang, parameters, pipeline);
108    }
109    
110    /**
111     * Executes the given extraction
112     * @param fileName The name of the extraction file to execute
113     * @param resultOutputStream The result output stream
114     * @param lang The lang
115     * @param parameters The parameters
116     * @param pipeline The execution pipeline
117     * @throws Exception if an errors occured
118     */
119    public void execute(String fileName, OutputStream resultOutputStream, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
120    {
121        _execute(fileName, null, resultOutputStream, lang, parameters, pipeline);
122    }
123    
124    private void _execute(
125            String fileName,
126            String defaultResultFileName, 
127            OutputStream resultOutputStream, 
128            String lang, 
129            Map<String, Object> parameters, 
130            PipelineDescriptor pipeline) throws Exception
131    {
132        _checkRights();
133        
134        Pair<Extraction, String> extractionAndName = _getExtraction(fileName);
135        Extraction extraction = extractionAndName.getLeft();
136        String definitionFilename = extractionAndName.getRight();
137        
138        AttributesImpl attributes = _getAttrs(definitionFilename);
139
140        ExtractionExecutionContext context = _getContext(extraction, lang, parameters);
141        
142        if (resultOutputStream != null)
143        {
144            _doExecuteForPathWithNoVar(attributes, extraction, context, resultOutputStream, pipeline);
145        }
146        else
147        {
148            _doExecute(attributes, extraction, fileName, context, defaultResultFileName, pipeline);
149        }
150    }
151    
152    private void _checkRights()
153    {
154        if (_rightManager.hasRight(_currentUserProvider.getUser(), "Extraction_Rights_ExecuteExtraction", "/admin") != RightResult.RIGHT_ALLOW)
155        {
156            String errorMessage = "User " + _currentUserProvider.getUser() + " tried to execute extraction with no sufficient rights"; 
157            getLogger().error(errorMessage);
158            throw new IllegalStateException(errorMessage);
159        }
160    }
161    
162    private Pair<Extraction, String> _getExtraction(String fileName) throws Exception
163    {
164        String filePath = ExtractionConstants.DEFINITIONS_DIR + fileName;
165        
166        Source src = _sourceResolver.resolveURI(filePath);
167        Extraction extraction = null;
168        String definitionFilename = null;
169        try
170        {
171            File file = ((FileSource) src).getFile();
172            
173            if (!file.exists())
174            {
175                throw new IllegalArgumentException("The file " + filePath + " does not exist.");
176            }
177            
178            extraction = _reader.readExtractionDefinitionFile(file);
179            definitionFilename = file.getName();
180            return Pair.of(extraction, definitionFilename);
181        }
182        catch (Exception e)
183        {
184            throw new IllegalStateException("An unexpected error occured.", e);
185        }
186        finally
187        {
188            _sourceResolver.release(src);
189        }
190    }
191    
192    private AttributesImpl _getAttrs(String definitionFilename)
193    {
194        AttributesImpl attributes = new AttributesImpl();
195        attributes.addCDATAAttribute("user", _currentUserProvider.getUser().getLogin());
196        attributes.addCDATAAttribute("date", ZonedDateTime.now().format(DateUtils.getISODateTimeFormatter()));
197        attributes.addCDATAAttribute("name", definitionFilename);
198        return attributes;
199    }
200    
201    private ExtractionExecutionContext _getContext(Extraction extraction, String lang, Map<String, Object> parameters)
202    {
203        ExtractionExecutionContext context = new ExtractionExecutionContext();
204        if (StringUtils.isNotEmpty(lang))
205        {
206            context.setDefaultLocale(new Locale(lang));
207        }
208        context.setDisplayOptionalColumns(_getDisplayOptionalColumns(extraction.getDisplayOptionalColumnsNames(), parameters));
209        context.setClauseVariables(_getQueryVariables(extraction.getQueryVariablesNamesAndContentTypes(), parameters));
210        return context;
211    }
212
213    Map<String, Boolean> _getDisplayOptionalColumns(List<String> displayOptionalColumnsNames, Map<String, Object> parameters)
214    {
215        Map<String, Boolean> result = new HashMap<>();
216        for (String name : displayOptionalColumnsNames)
217        {
218            Boolean value = (Boolean) parameters.get(name);
219            if (null == value)
220            {
221                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
222            }
223            result.put(name, value);
224        }
225        return result;
226    }
227
228    private Map<String, String> _getQueryVariables(Map<String, String> queryVariablesNamesAndContentTypes, Map<String, Object> parameters)
229    {
230        Map<String, String> result = new HashMap<>();
231        for (Map.Entry<String, String> entry : queryVariablesNamesAndContentTypes.entrySet())
232        {
233            String name = entry.getKey();
234            String contentTypeId = entry.getValue();
235            if (!_contentTypeExtensionPoint.hasExtension(contentTypeId))
236            {
237                throw new IllegalArgumentException("Extraction - content type '" + contentTypeId + "' used in variable '" + name + "' definition does not exist");
238            }
239            
240            String contentId = (String) parameters.get(name);
241            if (null == contentId)
242            {
243                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
244            }
245            
246            Content content = _resolver.resolveById(contentId);
247            if (!_contentTypesHelper.isInstanceOf(content, contentTypeId))
248            {
249                throw new IllegalArgumentException("Extraction - content '" + contentId + "' is not an instance of content type '" + contentTypeId + "', defined by the variable named '" + name + "'");
250            }
251            
252            result.put(name, contentId);
253        }
254        return result;
255    }
256    
257    private void _doExecute(AttributesImpl attributes, Extraction extraction, String extractionName, ExtractionExecutionContext context, String defaultResultFileName, PipelineDescriptor pipeline) throws Exception
258    {
259        String unresolvedPath = pipeline.getResultSubfolder();
260        Path basePath = Paths.get(AmetysHomeHelper.getAmetysHomeData().toPath().toString(), EXTRACTION_DIR_NAME);
261        if (_resultPathResolver.hasVariable(unresolvedPath))
262        {
263            List<ExtractionComponent> extractionComponents = extraction.getExtractionComponents();
264            boolean allAreNotTwoStepsComponent = extractionComponents.stream()
265                    .filter(Predicates.not(TwoStepsExecutingExtractionComponent.class::isInstance))
266                    .findFirst()
267                    .isPresent();
268            if (allAreNotTwoStepsComponent)
269            {
270                throw new IllegalArgumentException("The extraction " + extractionName + " has an invalid component at first level which does not support a subfolder containing variables.");
271            }
272            
273            List<TwoStepsExecutingExtractionComponent> components = extractionComponents.stream()
274                    .map(TwoStepsExecutingExtractionComponent.class::cast)
275                    .collect(Collectors.toList());
276            _doExecuteForPathWithVar(attributes, extraction, context, components, basePath, unresolvedPath, pipeline);
277        }
278        else
279        {
280            Path fileOrFolderPath = _resultPathResolver.resolvePath(unresolvedPath, null, extraction, basePath).keySet().iterator().next();
281            Path filePath = _filePathWhenNoVar(fileOrFolderPath, defaultResultFileName);
282            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);)
283            {
284                _doExecuteForPathWithNoVar(attributes, extraction, context, fileOs, pipeline);
285            }
286        }
287    }
288    
289    private Path _filePathWhenNoVar(Path fileOrFolderPath, String defaultFileName)
290    {
291        Path fileName = fileOrFolderPath.getFileName();
292        if (fileName.toString().contains("."))
293        {
294            // is already a file, do not use default file name
295            return fileOrFolderPath;
296        }
297        return Paths.get(fileOrFolderPath.toString(), defaultFileName);
298    }
299    
300    private void _doExecuteForPathWithNoVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, OutputStream outputStream, PipelineDescriptor pipelineDescriptor) throws Exception
301    {
302        try (Pipeline pipeline = pipelineDescriptor.newPipeline(outputStream);)
303        {
304            ContentHandler contentHandler = pipeline.getHandler();
305            _noVarExecute(contentHandler, attributes, extraction, context);
306            pipeline.serialize();
307        }
308    }
309    
310    private void _noVarExecute(ContentHandler contentHandler, AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context) throws Exception
311    {
312        contentHandler.startDocument();
313        XMLUtils.startElement(contentHandler, "extraction", attributes);
314        
315        for (ExtractionComponent component : extraction.getExtractionComponents())
316        {
317            component.prepareComponentExecution(context);
318            component.execute(contentHandler, context);
319        }
320        
321        XMLUtils.endElement(contentHandler, "extraction");
322        contentHandler.endDocument();
323    }
324    
325    private void _doExecuteForPathWithVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, Path basePath, String unresolvedPath, PipelineDescriptor pipelineDescriptor) throws Exception
326    {
327        Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent = new HashMap<>();
328        for (TwoStepsExecutingExtractionComponent component : components)
329        {
330            component.prepareComponentExecution(context);
331            List<Content> firstLevelResults = StreamSupport.stream(component.computeFirstLevelResults(context).spliterator(), false)
332                    .collect(Collectors.toList());
333            firstLevelResultsByComponent.put(component, firstLevelResults);
334        }
335        
336        List<Content> allContents = firstLevelResultsByComponent.values()
337                .stream()
338                .flatMap(List::stream)
339                .collect(Collectors.toList());
340        
341        String unresolvedFilePath = _unresolvedFilePathWhenVar(unresolvedPath, pipelineDescriptor);
342        Map<Path, List<Content>> paths = _resultPathResolver.resolvePath(unresolvedFilePath, allContents, extraction, basePath);
343        for (Path filePath : paths.keySet())
344        {
345            List<Content> involvedContentsForPath = paths.get(filePath);
346            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);
347                 Pipeline pipeline = pipelineDescriptor.newPipeline(fileOs);)
348            {
349                ContentHandler contentHandler = pipeline.getHandler();
350                _withVarExecute(contentHandler, attributes, context, components, involvedContentsForPath, firstLevelResultsByComponent);
351                pipeline.serialize();
352            }
353        }
354    }
355    
356    private String _unresolvedFilePathWhenVar(String unresolvedFolderOrFilePath, PipelineDescriptor pipelineDescriptor)
357    {
358        if (_resultPathResolver.isFolder(unresolvedFolderOrFilePath))
359        {
360            return unresolvedFolderOrFilePath + "/${title}." + pipelineDescriptor.getDefaultExtension();
361        }
362        else
363        {
364            return unresolvedFolderOrFilePath;
365        }
366    }
367    
368    private void _withVarExecute(ContentHandler contentHandler, AttributesImpl attributes, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, List<Content> involvedContentsForPath, Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent) throws Exception
369    {
370        contentHandler.startDocument();
371        XMLUtils.startElement(contentHandler, "extraction", attributes);
372        
373        for (TwoStepsExecutingExtractionComponent component : components)
374        {
375            List<Content> firstLevelResults = firstLevelResultsByComponent.get(component);
376            List<Content> involvedFirstLevelResults = ListUtils.intersection(firstLevelResults, involvedContentsForPath); 
377            component.executeFor(contentHandler, involvedFirstLevelResults, context);
378        }
379        
380        XMLUtils.endElement(contentHandler, "extraction");
381        contentHandler.endDocument();
382    }
383}