Source code

001/*
002 *  Copyright 2022 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.odf.schedulable;
017
018import java.io.IOException;
019import java.io.InputStream;
020import java.time.Duration;
021import java.time.ZonedDateTime;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.Objects;
027import java.util.Optional;
028import java.util.concurrent.atomic.AtomicInteger;
029import java.util.stream.Collectors;
030
031import org.apache.avalon.framework.service.ServiceException;
032import org.apache.avalon.framework.service.ServiceManager;
033import org.apache.cocoon.components.ContextHelper;
034import org.apache.cocoon.environment.Request;
035import org.apache.commons.io.IOUtils;
036import org.apache.commons.lang.StringUtils;
037import org.apache.commons.lang3.exception.ExceptionUtils;
038import org.apache.commons.lang3.time.DurationFormatUtils;
039import org.apache.excalibur.source.Source;
040import org.apache.excalibur.source.SourceResolver;
041import org.quartz.JobDataMap;
042import org.quartz.JobExecutionContext;
043
044import org.ametys.cms.contenttype.ContentType;
045import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
046import org.ametys.cms.schedule.AbstractSendingMailSchedulable;
047import org.ametys.core.schedule.progression.ContainerProgressionTracker;
048import org.ametys.core.ui.mail.StandardMailBodyHelper;
049import org.ametys.odf.init.OdfRefTableDataExtensionPoint;
050import org.ametys.odf.init.OdfRefTableDataSynchronizationAccessController;
051import org.ametys.plugins.contentio.csv.ImportCSVFileHelper;
052import org.ametys.plugins.contentio.csv.SynchronizeModeEnumerator.ImportMode;
053import org.ametys.plugins.core.schedule.Scheduler;
054import org.ametys.runtime.config.Config;
055import org.ametys.runtime.i18n.I18nizableText;
056
057/**
058 * Import reference table data described by {@link OdfRefTableDataExtensionPoint}.
059 */
060public class OdfRefTableDataSynchronizationSchedulable extends AbstractSendingMailSchedulable
061{
062    private static final String __CONTEXT_KEY_SYNC_REPORT = OdfRefTableDataSynchronizationSchedulable.class.getName() + "$syncReport";
063    
064    private static final String __RESULT_IMPORTED_COUNT = "importedCount";
065    private static final String __RESULT_DETAILS = "details";
066    private static final String __RESULT_ERROR = "error";
067    private static final String __RESULT_PARTIAL = "partial";
068    private static final String __RESULT_SUCCESS = "success";
069    private static final String __RESULT_DURATION = "duration";
070    private static final String __IMPORT_MODE = "importMode";
071
072    private OdfRefTableDataExtensionPoint _odfRefTableDataEP;
073    private ImportCSVFileHelper _importCSVFileHelper;
074    private ContentTypeExtensionPoint _contentTypeEP;
075    private SourceResolver _srcResolver;
076    
077    @Override
078    public void service(ServiceManager manager) throws ServiceException
079    {
080        super.service(manager);
081        _odfRefTableDataEP = (OdfRefTableDataExtensionPoint) manager.lookup(OdfRefTableDataExtensionPoint.ROLE);
082        _importCSVFileHelper = (ImportCSVFileHelper) manager.lookup(ImportCSVFileHelper.ROLE);
083        _contentTypeEP = (ContentTypeExtensionPoint) manager.lookup(ContentTypeExtensionPoint.ROLE);
084        _srcResolver = (SourceResolver) manager.lookup(SourceResolver.ROLE);
085    }
086    
087    @SuppressWarnings("unchecked")
088    @Override
089    protected void _doExecute(JobExecutionContext context, ContainerProgressionTracker progressionTracker) throws Exception
090    {
091        JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
092        ZonedDateTime begin = ZonedDateTime.now();
093        getLogger().info("[BEGIN] ODF reference tables synchronization");
094        
095        Map<String, Object> result = new HashMap<>();
096        result.put(__RESULT_IMPORTED_COUNT, 0);
097        
098        Request request = ContextHelper.getRequest(_context);
099        
100        try
101        {
102            request.setAttribute(OdfRefTableDataSynchronizationAccessController.ODF_REF_TABLE_SYNCHRONIZATION, true);
103            
104            String language = Config.getInstance().getValue("odf.programs.lang");
105            ImportMode importMode = ImportMode.valueOf((String) jobDataMap.get(Scheduler.PARAM_VALUES_PREFIX + __IMPORT_MODE));
106            
107            Map<String, String> dataToImport = _odfRefTableDataEP.getDataToImport();
108            
109            if (getLogger().isInfoEnabled())
110            {
111                getLogger().info("All CSV files to import: {}", dataToImport.toString());
112            }
113            
114            AtomicInteger count = new AtomicInteger();
115            Integer total = dataToImport.size();
116            for (String contentTypeId : dataToImport.keySet())
117            {
118                getLogger().info("[{}/{}] Synchronizing contents of type {}...", count.incrementAndGet(), total, contentTypeId);
119                
120                Map<String, Object> details = (Map<String, Object>) result.computeIfAbsent(__RESULT_DETAILS, __ -> new HashMap<>());
121                
122                if (_contentTypeEP.hasExtension(contentTypeId))
123                {
124                    ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
125                    String dataURI = dataToImport.get(contentTypeId);
126                    Source source = null;
127                    
128                    try
129                    {
130                        source = _srcResolver.resolveURI(dataURI);
131
132                        try (
133                            InputStream is = source.getInputStream();
134                            InputStream data = IOUtils.buffer(is);
135                        )
136                        {
137                            Map<String, Object> csvResult = _importCSVFileHelper.importContents(data, contentType, language, importMode);
138                            details.put(contentTypeId, csvResult);
139    
140                            if (csvResult.containsKey(ImportCSVFileHelper.RESULT_ERROR))
141                            {
142                                _addToListInMap(result, __RESULT_ERROR, contentTypeId);
143                                getLogger().error("Error while importing ODF reference table data for content type {} with the following reason: '{}'.", contentTypeId, csvResult.get(ImportCSVFileHelper.RESULT_ERROR));
144                            }
145                            else
146                            {
147                                Integer nbImported = (Integer) csvResult.getOrDefault(ImportCSVFileHelper.RESULT_IMPORTED_COUNT, 0);
148                                result.put(__RESULT_IMPORTED_COUNT, (Integer) result.get(__RESULT_IMPORTED_COUNT) + nbImported);
149                                Integer nbErrors = (Integer) csvResult.getOrDefault(ImportCSVFileHelper.RESULT_NB_ERRORS, 0);
150                                Integer nbWarnings = (Integer) csvResult.getOrDefault(ImportCSVFileHelper.RESULT_NB_WARNINGS, 0);
151                                if (nbErrors > 0 || nbWarnings > 0)
152                                {
153                                    _addToListInMap(result, __RESULT_PARTIAL, contentTypeId);
154                                    getLogger().warn("Some errors and warnings while importing ODF reference table data for content type {} with {} synchronized entries, {} errors and {} warnings.", contentTypeId, nbImported, nbErrors, nbWarnings);
155                                }
156                                else
157                                {
158                                    _addToListInMap(result, __RESULT_SUCCESS, contentTypeId);
159                                    getLogger().info("Success while importing ODF reference table data for content type {} with {} synchronized entries.", contentTypeId, nbImported);
160                                }
161                            }
162                        }
163                    }
164                    catch (IOException e)
165                    {
166                        getLogger().error("Error while importing ODF reference table data of content type {} from file {}.", contentTypeId, dataURI, e);
167                        details.put(contentTypeId, Map.of("error", "exception", "message", e.getMessage()));
168                        _addToListInMap(result, __RESULT_ERROR, contentTypeId);
169                    }
170                    finally
171                    {
172                        if (source != null)
173                        {
174                            _srcResolver.release(source);
175                        }
176                    }
177                }
178                else
179                {
180                    getLogger().warn("The content type {} is not defined.", contentTypeId);
181                    details.put(contentTypeId, Map.of("error", "unexisting"));
182                    _addToListInMap(result, __RESULT_ERROR, contentTypeId);
183                }
184            }
185        }
186        catch (Exception e)
187        {
188            getLogger().error("Error during ODF reference tables synchronization", e);
189            throw e;
190        }
191        finally
192        {
193            request.removeAttribute(OdfRefTableDataSynchronizationAccessController.ODF_REF_TABLE_SYNCHRONIZATION);
194            String duration = Optional.of(ZonedDateTime.now())
195                .map(end -> Duration.between(begin, end))
196                .map(Duration::toMillis)
197                .map(DurationFormatUtils::formatDurationHMS)
198                .orElse("undefined");
199            
200            if (getLogger().isInfoEnabled())
201            {
202                StringBuilder resume = new StringBuilder("Resume: ");
203                resume.append(result.get(__RESULT_IMPORTED_COUNT));
204                resume.append(" synchronized entries");
205                if (result.containsKey(__RESULT_SUCCESS))
206                {
207                    resume.append(", ");
208                    resume.append(((List<String>) result.get(__RESULT_SUCCESS)).size());
209                    resume.append(" successful reference tables");
210                }
211                if (result.containsKey(__RESULT_PARTIAL))
212                {
213                    resume.append(", ");
214                    resume.append(((List<String>) result.get(__RESULT_PARTIAL)).size());
215                    resume.append(" partial successful reference tables");
216                }
217                if (result.containsKey(__RESULT_ERROR))
218                {
219                    resume.append(", ");
220                    resume.append(((List<String>) result.get(__RESULT_ERROR)).size());
221                    resume.append(" failed reference tables");
222                }
223                getLogger().info(resume.toString());
224                getLogger().info("[END] ODF reference tables synchronization in {}", duration);
225            }
226            
227            result.put(__RESULT_DURATION, duration);
228            context.put(__CONTEXT_KEY_SYNC_REPORT, result);
229        }
230    }
231
232    @Override
233    protected boolean _isMailBodyInHTML(JobExecutionContext context) throws Exception
234    {
235        return true;
236    }
237
238    @SuppressWarnings("unchecked")
239    @Override
240    protected I18nizableText _getSuccessMailSubject(JobExecutionContext context) throws Exception
241    {
242        String subjectKey = "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_SUBJECT";
243        Map<String, Object> result = (Map<String, Object>) context.get(__CONTEXT_KEY_SYNC_REPORT);
244        if (result.containsKey(__RESULT_PARTIAL) || result.containsKey(__RESULT_ERROR))
245        {
246            subjectKey += "_WITH_ERRORS";
247        }
248        return new I18nizableText("plugin.odf", subjectKey);
249    }
250
251    @SuppressWarnings("unchecked")
252    @Override
253    protected String _getSuccessMailBody(JobExecutionContext context) throws Exception
254    {
255        try
256        {
257            Map<String, Object> result = (Map<String, Object>) context.get(__CONTEXT_KEY_SYNC_REPORT);
258            
259            StringBuilder sb = new StringBuilder();
260            
261            // Duration
262            sb.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_DURATION", List.of(result.get(__RESULT_DURATION).toString()))));
263            
264            // Imported count
265            sb.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_COUNT", List.of(result.get(__RESULT_IMPORTED_COUNT).toString()))));
266            
267            // Success
268            if (result.containsKey(__RESULT_SUCCESS))
269            {
270                sb.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_SUCCESS", List.of(String.valueOf(((List<String>) result.get(__RESULT_SUCCESS)).size())))));
271            }
272            
273            // Partial
274            if (result.containsKey(__RESULT_PARTIAL))
275            {
276                List<String> partialCTypes = (List<String>) result.get(__RESULT_PARTIAL);
277                
278                List<String> partialCTypeLabels = partialCTypes.stream()
279                    .map(id -> _contentTypeEP.getExtension(id))
280                    .filter(Objects::nonNull)
281                    .map(ContentType::getLabel)
282                    .map(_i18nUtils::translate)
283                    .collect(Collectors.toList());
284                
285                sb.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_PARTIAL", List.of(String.valueOf(partialCTypes.size()), StringUtils.join(partialCTypeLabels, ", ")))));
286            }
287            
288            // Fail
289            if (result.containsKey(__RESULT_ERROR))
290            {
291                List<String> errorCTypes = (List<String>) result.get(__RESULT_PARTIAL);
292                
293                List<String> errorCTypeLabels = errorCTypes.stream()
294                    .map(id -> _contentTypeEP.getExtension(id))
295                    .filter(Objects::nonNull)
296                    .map(ContentType::getLabel)
297                    .map(_i18nUtils::translate)
298                    .collect(Collectors.toList());
299                
300                sb.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_FAIL", List.of(String.valueOf(errorCTypeLabels.size()), StringUtils.join(errorCTypeLabels, ", ")))));
301            }
302            
303            String details = sb.toString();
304            
305            return StandardMailBodyHelper.newHTMLBody()
306                    .withTitle(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_SUBJECT"))
307                    .addMessage(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME"))
308                    .addMessage(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_ASK_DETAILS"))
309                    .withDetails(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_REPORT"), details, false)
310                    .build();
311        }
312        catch (IOException e)
313        {
314            getLogger().error("Failed to build HTML email body for reference table synchronization report", e);
315            return null;
316        }
317    }
318
319    @Override
320    protected I18nizableText _getErrorMailSubject(JobExecutionContext context) throws Exception
321    {
322        return new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_ERROR_SUBJECT");
323    }
324
325    @Override
326    protected String _getErrorMailBody(JobExecutionContext context, Throwable throwable) throws Exception
327    {
328        String error = StringUtils.join(ExceptionUtils.getStackFrames(throwable), "<br/>");
329        
330        return StandardMailBodyHelper.newHTMLBody()
331                .withTitle(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_ERROR_SUBJECT"))
332                .addMessage(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_ERROR_BODY"))
333                .withDetails(null, error, false)
334                .build();
335    }
336    
337    private void _addToListInMap(Map<String, Object> map, String itemName, String itemToAdd)
338    {
339        @SuppressWarnings("unchecked")
340        List<String> list = (List<String>) map.computeIfAbsent(itemName, __ -> new ArrayList<>());
341        list.add(itemToAdd);
342    }
343}