Source code

001/*
002 *  Copyright 2018 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.extraction.execution;
017
018import java.io.File;
019import java.io.OutputStream;
020import java.nio.file.Path;
021import java.nio.file.Paths;
022import java.time.ZonedDateTime;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.Optional;
027import java.util.Set;
028import java.util.stream.Collectors;
029import java.util.stream.StreamSupport;
030
031import org.apache.avalon.framework.component.Component;
032import org.apache.avalon.framework.service.ServiceException;
033import org.apache.avalon.framework.service.ServiceManager;
034import org.apache.avalon.framework.service.Serviceable;
035import org.apache.cocoon.xml.AttributesImpl;
036import org.apache.cocoon.xml.XMLUtils;
037import org.apache.commons.collections4.ListUtils;
038import org.apache.commons.lang3.LocaleUtils;
039import org.apache.commons.lang3.StringUtils;
040import org.apache.excalibur.source.SourceResolver;
041import org.apache.excalibur.source.impl.FileSource;
042import org.xml.sax.ContentHandler;
043
044import org.ametys.cms.contenttype.ContentTypesHelper;
045import org.ametys.cms.repository.Content;
046import org.ametys.core.right.RightManager;
047import org.ametys.core.right.RightManager.RightResult;
048import org.ametys.core.user.CurrentUserProvider;
049import org.ametys.core.user.UserIdentity;
050import org.ametys.core.util.DateUtils;
051import org.ametys.plugins.extraction.ExtractionConstants;
052import org.ametys.plugins.extraction.component.ExtractionComponent;
053import org.ametys.plugins.extraction.component.TwoStepsExecutingExtractionComponent;
054import org.ametys.plugins.extraction.execution.Extraction.ClausesVariable;
055import org.ametys.plugins.extraction.execution.Extraction.ClausesVariableType;
056import org.ametys.plugins.extraction.execution.pipeline.Pipeline;
057import org.ametys.plugins.extraction.execution.pipeline.PipelineDescriptor;
058import org.ametys.plugins.extraction.execution.pipeline.Pipelines;
059import org.ametys.plugins.repository.AmetysObjectResolver;
060import org.ametys.runtime.authentication.AccessDeniedException;
061import org.ametys.runtime.plugin.component.AbstractLogEnabled;
062import org.ametys.runtime.util.AmetysHomeHelper;
063
064import com.google.common.base.Predicates;
065
066/**
067 * Extracts query results form a XML definition file
068 */
069public class ExtractionExecutor extends AbstractLogEnabled implements Component, Serviceable
070{
071    /** The Avalon role. */
072    public static final String ROLE = ExtractionExecutor.class.getName();
073    
074    private RightManager _rightManager;
075    private ExtractionDefinitionReader _reader;
076    private CurrentUserProvider _currentUserProvider;
077    private AmetysObjectResolver _resolver;
078    private ContentTypesHelper _contentTypesHelper;
079    private SourceResolver _sourceResolver;
080    private PathResolver _resultPathResolver;
081
082    @Override
083    public void service(ServiceManager serviceManager) throws ServiceException
084    {
085        _rightManager = (RightManager) serviceManager.lookup(RightManager.ROLE);
086        _reader = (ExtractionDefinitionReader) serviceManager.lookup(ExtractionDefinitionReader.ROLE);
087        _currentUserProvider = (CurrentUserProvider) serviceManager.lookup(CurrentUserProvider.ROLE);
088        _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE);
089        _contentTypesHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
090        _sourceResolver = (SourceResolver) serviceManager.lookup(SourceResolver.ROLE);
091        _resultPathResolver = (PathResolver) serviceManager.lookup(PathResolver.ROLE);
092    }
093    
094    /**
095     * Executes the given extraction
096     * @param relativeFilePath The path of the extraction file to execute
097     * @param defaultResultFileName The default file name for the result (it can be unused, for instance if resultSubFolder is a file, i.e. contains a '.' in its last element)
098     * @param lang The language
099     * @param parameters The parameters
100     * @param pipeline The execution pipeline
101     * @return The set of the result files path
102     * @throws Exception if an errors occurred
103     */
104    public Set<Path> execute(String relativeFilePath, String defaultResultFileName, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
105    {
106        return _execute(relativeFilePath, defaultResultFileName, null, lang, parameters, pipeline);
107    }
108    
109    /**
110     * Executes the given extraction
111     * @param relativeFilePath The path of the extraction file to execute
112     * @param resultOutputStream The result output stream
113     * @param lang The language
114     * @param parameters The parameters
115     * @param pipeline The execution pipeline
116     * @throws Exception if an errors occurred
117     */
118    public void execute(String relativeFilePath, OutputStream resultOutputStream, String lang, Map<String, Object> parameters, PipelineDescriptor pipeline) throws Exception
119    {
120        _execute(relativeFilePath, null, resultOutputStream, lang, parameters, pipeline);
121    }
122    
123    private Set<Path> _execute(
124            String relativeFilePath,
125            String defaultResultFileName,
126            OutputStream resultOutputStream,
127            String lang,
128            Map<String, Object> parameters,
129            PipelineDescriptor pipeline) throws Exception
130    {
131        String absoluteFilePath = ExtractionConstants.DEFINITIONS_DIR + relativeFilePath;
132
133        FileSource src = null;
134        try
135        {
136            src = (FileSource) _sourceResolver.resolveURI(absoluteFilePath);
137            File file = src.getFile();
138            
139            if (!file.exists())
140            {
141                throw new IllegalArgumentException("The file " + relativeFilePath + " does not exist.");
142            }
143            
144            _checkRights(src);
145
146            Extraction extraction = _reader.readExtractionDefinitionFile(file);
147
148            String definitionFilename = file.getName();
149
150            AttributesImpl attributes = _getAttrs(definitionFilename);
151
152            ExtractionExecutionContext context = _getContext(extraction, lang, parameters);
153            
154            if (resultOutputStream != null)
155            {
156                _doExecuteForPathWithNoVar(attributes, extraction, context, resultOutputStream, pipeline);
157                return Set.of();
158            }
159            else
160            {
161                return _doExecute(attributes, extraction, relativeFilePath, context, defaultResultFileName, pipeline);
162            }
163        }
164        finally
165        {
166            _sourceResolver.release(src);
167        }
168    }
169    
170    private void _checkRights(FileSource src)
171    {
172        UserIdentity currentUser = _currentUserProvider.getUser();
173        
174        if (_rightManager.hasReadAccess(currentUser, src)
175                || _rightManager.hasRight(currentUser, ExtractionConstants.EXECUTE_EXTRACTION_RIGHT_ID, src) == RightResult.RIGHT_ALLOW)
176        {
177            return;
178        }
179        
180        String errorMessage = "User " + _currentUserProvider.getUser() + " tried to execute extraction with no sufficient rights";
181        getLogger().error(errorMessage);
182        throw new AccessDeniedException(errorMessage);
183    }
184    
185    private AttributesImpl _getAttrs(String definitionFilename)
186    {
187        AttributesImpl attributes = new AttributesImpl();
188        attributes.addCDATAAttribute("user", _currentUserProvider.getUser().getLogin());
189        attributes.addCDATAAttribute("date", ZonedDateTime.now().format(DateUtils.getISODateTimeFormatter()));
190        attributes.addCDATAAttribute("name", definitionFilename);
191        return attributes;
192    }
193    
194    private ExtractionExecutionContext _getContext(Extraction extraction, String lang, Map<String, Object> parameters)
195    {
196        ExtractionExecutionContext context = new ExtractionExecutionContext();
197        if (StringUtils.isNotEmpty(lang))
198        {
199            context.setDefaultLocale(LocaleUtils.toLocale(lang));
200        }
201        context.setDisplayOptionalColumns(_getDisplayOptionalColumns(extraction.getDisplayOptionalColumnsNames(), parameters));
202        context.setClausesVariablesValues(_getClausesVariablesValues(extraction.getClausesVariables(), parameters));
203        return context;
204    }
205
206    Map<String, Boolean> _getDisplayOptionalColumns(List<String> displayOptionalColumnsNames, Map<String, Object> parameters)
207    {
208        Map<String, Boolean> result = new HashMap<>();
209        for (String name : displayOptionalColumnsNames)
210        {
211            Boolean value = _getDipslayOptionalColumn(name, parameters);
212            if (value == null)
213            {
214                throw new IllegalArgumentException("Extraction - There is a variable named '" + name + "' but there is no corresponding value");
215            }
216            else
217            {
218                result.put(name, value);
219            }
220        }
221        return result;
222    }
223    
224    private Boolean _getDipslayOptionalColumn(String optionalColumnName, Map<String, Object> parameters)
225    {
226        Object value = parameters.get(optionalColumnName);
227        
228        if (value instanceof Boolean)
229        {
230            return (Boolean) value;
231        }
232        else if (value instanceof String)
233        {
234            return Boolean.valueOf((String) value);
235        }
236        else
237        {
238            return null;
239        }
240    }
241
242    private Map<ClausesVariable, List<String>> _getClausesVariablesValues(List<ClausesVariable> clausesVariables, Map<String, Object> parameters)
243    {
244        Map<ClausesVariable, List<String>> result = new HashMap<>();
245        for (ClausesVariable clausesVariable : clausesVariables)
246        {
247            if (ClausesVariableType.SELECT_CONTENTS.equals(clausesVariable.type()))
248            {
249                // Only one content type for SELECT_CONTENTS clauses variables
250                Optional<String> contentTypeId = clausesVariable.contentTypeIds()
251                                                                .stream()
252                                                                .findFirst();
253                
254                @SuppressWarnings("unchecked")
255                List<String> contentIds = (List<String>) parameters.get(clausesVariable.name());
256                if (contentIds == null || contentIds.isEmpty())
257                {
258                    throw new IllegalArgumentException("Extraction - There is a variable named '" + clausesVariable.name() + "' but there is no corresponding value");
259                }
260                
261                // If a content type is provided, check all contents' types
262                if (contentTypeId.isPresent())
263                {
264                    for (String contentId : contentIds)
265                    {
266                        Content content = _resolver.resolveById(contentId);
267                        if (!_contentTypesHelper.isInstanceOf(content, contentTypeId.get()))
268                        {
269                            throw new IllegalArgumentException("Extraction - content '" + contentId + "' is not an instance of content type '" + contentTypeId.get() + "', defined by the variable named '" + clausesVariable.name() + "'");
270                        }
271                    }
272                }
273                
274                result.put(clausesVariable, contentIds);
275            }
276            else
277            {
278                String solrRequest = (String) parameters.get(clausesVariable.name());
279                if (StringUtils.isEmpty(solrRequest))
280                {
281                    throw new IllegalArgumentException("Extraction - There is a variable named '" + clausesVariable.name() + "' but there is no corresponding value");
282                }
283                
284                result.put(clausesVariable, List.of(solrRequest));
285            }
286        }
287        return result;
288    }
289    
290    private Set<Path> _doExecute(AttributesImpl attributes, Extraction extraction, String extractionFilePath, ExtractionExecutionContext context, String defaultResultFileName, PipelineDescriptor pipeline) throws Exception
291    {
292        String unresolvedPath = pipeline.getResultSubfolder();
293        Path basePath = Paths.get(AmetysHomeHelper.getAmetysHomeData().toPath().toString(), ExtractionConstants.RESULT_EXTRACTION_DIR_NAME);
294       
295        if (_resultPathResolver.hasVariable(unresolvedPath))
296        {
297            List<ExtractionComponent> extractionComponents = extraction.getExtractionComponents();
298            boolean allAreNotTwoStepsComponent = extractionComponents.stream()
299                    .filter(Predicates.not(TwoStepsExecutingExtractionComponent.class::isInstance))
300                    .findFirst()
301                    .isPresent();
302            if (allAreNotTwoStepsComponent)
303            {
304                throw new IllegalArgumentException("The extraction " + extractionFilePath + " has an invalid component at first level which does not support a subfolder containing variables.");
305            }
306            
307            List<TwoStepsExecutingExtractionComponent> components = extractionComponents.stream()
308                    .map(TwoStepsExecutingExtractionComponent.class::cast)
309                    .collect(Collectors.toList());
310            return _doExecuteForPathWithVar(attributes, extraction, context, components, basePath, unresolvedPath, pipeline);
311        }
312        else
313        {
314            Path fileOrFolderPath = _resultPathResolver.resolvePath(unresolvedPath, null, extraction, basePath).keySet().iterator().next();
315            Path filePath = _filePathWhenNoVar(fileOrFolderPath, defaultResultFileName);
316            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);)
317            {
318                _doExecuteForPathWithNoVar(attributes, extraction, context, fileOs, pipeline);
319                return Set.of(filePath);
320            }
321        }
322    }
323    
324    private Path _filePathWhenNoVar(Path fileOrFolderPath, String defaultFileName)
325    {
326        Path fileName = fileOrFolderPath.getFileName();
327        if (fileName.toString().contains("."))
328        {
329            // is already a file, do not use default file name
330            return fileOrFolderPath;
331        }
332        return Paths.get(fileOrFolderPath.toString(), defaultFileName);
333    }
334    
335    private void _doExecuteForPathWithNoVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, OutputStream outputStream, PipelineDescriptor pipelineDescriptor) throws Exception
336    {
337        try (Pipeline pipeline = pipelineDescriptor.newPipeline(outputStream);)
338        {
339            ContentHandler contentHandler = pipeline.getHandler();
340            _noVarExecute(contentHandler, attributes, extraction, context);
341            pipeline.serialize();
342        }
343    }
344    
345    private void _noVarExecute(ContentHandler contentHandler, AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context) throws Exception
346    {
347        contentHandler.startDocument();
348        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
349        
350        for (ExtractionComponent component : extraction.getExtractionComponents())
351        {
352            component.prepareComponentExecution(context);
353            component.execute(contentHandler, context);
354        }
355        
356        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
357        contentHandler.endDocument();
358    }
359    
360    private Set<Path> _doExecuteForPathWithVar(AttributesImpl attributes, Extraction extraction, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, Path basePath, String unresolvedPath, PipelineDescriptor pipelineDescriptor) throws Exception
361    {
362        Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent = new HashMap<>();
363        for (TwoStepsExecutingExtractionComponent component : components)
364        {
365            component.prepareComponentExecution(context);
366            List<Content> firstLevelResults = StreamSupport.stream(component.computeFirstLevelResults(context).spliterator(), false)
367                    .collect(Collectors.toList());
368            firstLevelResultsByComponent.put(component, firstLevelResults);
369        }
370        
371        List<Content> allContents = firstLevelResultsByComponent.values()
372                .stream()
373                .flatMap(List::stream)
374                .collect(Collectors.toList());
375        
376        String unresolvedFilePath = _unresolvedFilePathWhenVar(unresolvedPath, pipelineDescriptor);
377        Map<Path, List<Content>> paths = _resultPathResolver.resolvePath(unresolvedFilePath, allContents, extraction, basePath);
378        for (Path filePath : paths.keySet())
379        {
380            List<Content> involvedContentsForPath = paths.get(filePath);
381            try (OutputStream fileOs = Pipelines.getOutputStream(filePath);
382                 Pipeline pipeline = pipelineDescriptor.newPipeline(fileOs);)
383            {
384                ContentHandler contentHandler = pipeline.getHandler();
385                _withVarExecute(contentHandler, attributes, context, components, involvedContentsForPath, firstLevelResultsByComponent);
386                pipeline.serialize();
387            }
388        }
389        
390        return paths.keySet();
391    }
392    
393    private String _unresolvedFilePathWhenVar(String unresolvedFolderOrFilePath, PipelineDescriptor pipelineDescriptor)
394    {
395        if (_resultPathResolver.isFolder(unresolvedFolderOrFilePath))
396        {
397            return unresolvedFolderOrFilePath + "/${title}." + pipelineDescriptor.getDefaultExtension();
398        }
399        else
400        {
401            return unresolvedFolderOrFilePath;
402        }
403    }
404    
405    private void _withVarExecute(ContentHandler contentHandler, AttributesImpl attributes, ExtractionExecutionContext context, List<TwoStepsExecutingExtractionComponent> components, List<Content> involvedContentsForPath, Map<TwoStepsExecutingExtractionComponent, List<Content>> firstLevelResultsByComponent) throws Exception
406    {
407        contentHandler.startDocument();
408        XMLUtils.startElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG, attributes);
409        
410        for (TwoStepsExecutingExtractionComponent component : components)
411        {
412            List<Content> firstLevelResults = firstLevelResultsByComponent.get(component);
413            List<Content> involvedFirstLevelResults = ListUtils.intersection(firstLevelResults, involvedContentsForPath);
414            component.executeFor(contentHandler, involvedFirstLevelResults, context);
415        }
416        
417        XMLUtils.endElement(contentHandler, ExtractionConstants.RESULT_EXTRACTION_TAG);
418        contentHandler.endDocument();
419    }
420}