Source code

001/*
002 *  Copyright 2022 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.odf.schedulable;
017
018import java.io.IOException;
019import java.io.InputStream;
020import java.time.Duration;
021import java.time.ZonedDateTime;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.Optional;
027import java.util.concurrent.atomic.AtomicInteger;
028
029import org.apache.avalon.framework.service.ServiceException;
030import org.apache.avalon.framework.service.ServiceManager;
031import org.apache.cocoon.components.ContextHelper;
032import org.apache.cocoon.environment.Request;
033import org.apache.commons.io.IOUtils;
034import org.apache.commons.lang.StringUtils;
035import org.apache.commons.lang3.exception.ExceptionUtils;
036import org.apache.commons.lang3.time.DurationFormatUtils;
037import org.apache.excalibur.source.Source;
038import org.apache.excalibur.source.SourceResolver;
039import org.quartz.JobDataMap;
040import org.quartz.JobExecutionContext;
041
042import org.ametys.cms.contenttype.ContentType;
043import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
044import org.ametys.cms.schedule.AbstractSendingMailSchedulable;
045import org.ametys.odf.init.OdfRefTableDataExtensionPoint;
046import org.ametys.odf.init.OdfRefTableDataSynchronizationAccessController;
047import org.ametys.plugins.contentio.csv.ImportCSVFileHelper;
048import org.ametys.plugins.contentio.csv.SynchronizeModeEnumerator.ImportMode;
049import org.ametys.plugins.core.schedule.Scheduler;
050import org.ametys.runtime.config.Config;
051import org.ametys.runtime.i18n.I18nizableText;
052
053/**
054 * Import reference table data described by {@link OdfRefTableDataExtensionPoint}.
055 */
056public class OdfRefTableDataSynchronizationSchedulable extends AbstractSendingMailSchedulable
057{
058    private static final String __CONTEXT_KEY_SYNC_REPORT = OdfRefTableDataSynchronizationSchedulable.class.getName() + "$syncReport";
059    
060    private static final String __RESULT_IMPORTED_COUNT = "importedCount";
061    private static final String __RESULT_DETAILS = "details";
062    private static final String __RESULT_ERROR = "error";
063    private static final String __RESULT_PARTIAL = "partial";
064    private static final String __RESULT_SUCCESS = "success";
065    private static final String __RESULT_DURATION = "duration";
066    private static final String __IMPORT_MODE = "importMode";
067
068    private OdfRefTableDataExtensionPoint _odfRefTableDataEP;
069    private ImportCSVFileHelper _importCSVFileHelper;
070    private ContentTypeExtensionPoint _contentTypeEP;
071    private SourceResolver _srcResolver;
072    
073    @Override
074    public void service(ServiceManager manager) throws ServiceException
075    {
076        super.service(manager);
077        _odfRefTableDataEP = (OdfRefTableDataExtensionPoint) manager.lookup(OdfRefTableDataExtensionPoint.ROLE);
078        _importCSVFileHelper = (ImportCSVFileHelper) manager.lookup(ImportCSVFileHelper.ROLE);
079        _contentTypeEP = (ContentTypeExtensionPoint) manager.lookup(ContentTypeExtensionPoint.ROLE);
080        _srcResolver = (SourceResolver) manager.lookup(SourceResolver.ROLE);
081    }
082    
083    @SuppressWarnings("unchecked")
084    @Override
085    protected void _doExecute(JobExecutionContext context) throws Exception
086    {
087        JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
088        ZonedDateTime begin = ZonedDateTime.now();
089        getLogger().info("[BEGIN] ODF reference tables synchronization");
090        
091        Map<String, Object> result = new HashMap<>();
092        result.put(__RESULT_IMPORTED_COUNT, 0);
093        
094        Request request = ContextHelper.getRequest(_context);
095        
096        try
097        {
098            request.setAttribute(OdfRefTableDataSynchronizationAccessController.ODF_REF_TABLE_SYNCHRONIZATION, true);
099            
100            String language = Config.getInstance().getValue("odf.programs.lang");
101            ImportMode importMode = ImportMode.valueOf((String) jobDataMap.get(Scheduler.PARAM_VALUES_PREFIX + __IMPORT_MODE));
102            
103            Map<String, String> dataToImport = _odfRefTableDataEP.getDataToImport();
104            
105            if (getLogger().isInfoEnabled())
106            {
107                getLogger().info("All CSV files to import: {}", dataToImport.toString());
108            }
109            
110            AtomicInteger count = new AtomicInteger();
111            Integer total = dataToImport.size();
112            for (String contentTypeId : dataToImport.keySet())
113            {
114                getLogger().info("[{}/{}] Synchronizing contents of type {}...", count.incrementAndGet(), total, contentTypeId);
115                
116                Map<String, Object> details = (Map<String, Object>) result.computeIfAbsent(__RESULT_DETAILS, __ -> new HashMap<>());
117                
118                if (_contentTypeEP.hasExtension(contentTypeId))
119                {
120                    ContentType contentType = _contentTypeEP.getExtension(contentTypeId);
121                    String dataURI = dataToImport.get(contentTypeId);
122                    Source source = null;
123                    
124                    try
125                    {
126                        source = _srcResolver.resolveURI(dataURI);
127
128                        try (
129                            InputStream is = source.getInputStream();
130                            InputStream data = IOUtils.buffer(is);
131                        )
132                        {
133                            Map<String, Object> csvResult = _importCSVFileHelper.importContents(data, contentType, language, importMode);
134                            details.put(contentTypeId, csvResult);
135    
136                            if (csvResult.containsKey(ImportCSVFileHelper.RESULT_ERROR))
137                            {
138                                _addToListInMap(result, __RESULT_ERROR, contentTypeId);
139                                getLogger().error("Error while importing ODF reference table data for content type {} with the following reason: '{}'.", contentTypeId, csvResult.get(ImportCSVFileHelper.RESULT_ERROR));
140                            }
141                            else
142                            {
143                                Integer nbImported = (Integer) csvResult.getOrDefault(ImportCSVFileHelper.RESULT_IMPORTED_COUNT, 0);
144                                result.put(__RESULT_IMPORTED_COUNT, (Integer) result.get(__RESULT_IMPORTED_COUNT) + nbImported);
145                                Integer nbErrors = (Integer) csvResult.getOrDefault(ImportCSVFileHelper.RESULT_NB_ERRORS, 0);
146                                Integer nbWarnings = (Integer) csvResult.getOrDefault(ImportCSVFileHelper.RESULT_NB_WARNINGS, 0);
147                                if (nbErrors > 0 || nbWarnings > 0)
148                                {
149                                    _addToListInMap(result, __RESULT_PARTIAL, contentTypeId);
150                                    getLogger().warn("Some errors and warnings while importing ODF reference table data for content type {} with {} synchronized entries, {} errors and {} warnings.", contentTypeId, nbImported, nbErrors, nbWarnings);
151                                }
152                                else
153                                {
154                                    _addToListInMap(result, __RESULT_SUCCESS, contentTypeId);
155                                    getLogger().info("Success while importing ODF reference table data for content type {} with {} synchronized entries.", contentTypeId, nbImported);
156                                }
157                            }
158                        }
159                    }
160                    catch (IOException e)
161                    {
162                        getLogger().error("Error while importing ODF reference table data of content type {} from file {}.", contentTypeId, dataURI, e);
163                        details.put(contentTypeId, Map.of("error", "exception", "message", e.getMessage()));
164                        _addToListInMap(result, __RESULT_ERROR, contentTypeId);
165                    }
166                    finally
167                    {
168                        if (source != null)
169                        {
170                            _srcResolver.release(source);
171                        }
172                    }
173                }
174                else
175                {
176                    getLogger().warn("The content type {} is not defined.", contentTypeId);
177                    details.put(contentTypeId, Map.of("error", "unexisting"));
178                    _addToListInMap(result, __RESULT_ERROR, contentTypeId);
179                }
180            }
181        }
182        catch (Exception e)
183        {
184            getLogger().error("Error during ODF reference tables synchronization", e);
185            throw e;
186        }
187        finally
188        {
189            request.removeAttribute(OdfRefTableDataSynchronizationAccessController.ODF_REF_TABLE_SYNCHRONIZATION);
190            String duration = Optional.of(ZonedDateTime.now())
191                .map(end -> Duration.between(begin, end))
192                .map(Duration::toMillis)
193                .map(DurationFormatUtils::formatDurationHMS)
194                .orElse("undefined");
195            
196            if (getLogger().isInfoEnabled())
197            {
198                StringBuilder resume = new StringBuilder("Resume: ");
199                resume.append(result.get(__RESULT_IMPORTED_COUNT));
200                resume.append(" synchronized entries");
201                if (result.containsKey(__RESULT_SUCCESS))
202                {
203                    resume.append(", ");
204                    resume.append(((List<String>) result.get(__RESULT_SUCCESS)).size());
205                    resume.append(" successful reference tables");
206                }
207                if (result.containsKey(__RESULT_PARTIAL))
208                {
209                    resume.append(", ");
210                    resume.append(((List<String>) result.get(__RESULT_PARTIAL)).size());
211                    resume.append(" partial successful reference tables");
212                }
213                if (result.containsKey(__RESULT_ERROR))
214                {
215                    resume.append(", ");
216                    resume.append(((List<String>) result.get(__RESULT_ERROR)).size());
217                    resume.append(" failed reference tables");
218                }
219                getLogger().info(resume.toString());
220                getLogger().info("[END] ODF reference tables synchronization in {}", duration);
221            }
222            
223            result.put(__RESULT_DURATION, duration);
224            context.put(__CONTEXT_KEY_SYNC_REPORT, result);
225        }
226    }
227
228    @Override
229    protected boolean _isMailBodyInHTML(JobExecutionContext context) throws Exception
230    {
231        return true;
232    }
233
234    @SuppressWarnings("unchecked")
235    @Override
236    protected I18nizableText _getSuccessMailSubject(JobExecutionContext context) throws Exception
237    {
238        String subjectKey = "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_SUBJECT";
239        Map<String, Object> result = (Map<String, Object>) context.get(__CONTEXT_KEY_SYNC_REPORT);
240        if (result.containsKey(__RESULT_PARTIAL) || result.containsKey(__RESULT_ERROR))
241        {
242            subjectKey += "_WITH_ERRORS";
243        }
244        return new I18nizableText("plugin.odf", subjectKey);
245    }
246
247    @SuppressWarnings("unchecked")
248    @Override
249    protected I18nizableText _getSuccessMailBody(JobExecutionContext context) throws Exception
250    {
251        Map<String, Object> result = (Map<String, Object>) context.get(__CONTEXT_KEY_SYNC_REPORT);
252        
253        StringBuilder body = new StringBuilder();
254        body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME")));
255        body.append("<ul>");
256        
257        // Count
258        body.append("<li>");
259        body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_DURATION", List.of(result.get(__RESULT_DURATION).toString()))));
260        body.append("</li>");
261        
262        // Count
263        body.append("<li>");
264        body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_COUNT", List.of(result.get(__RESULT_IMPORTED_COUNT).toString()))));
265        body.append("</li>");
266        
267        // Success
268        if (result.containsKey(__RESULT_SUCCESS))
269        {
270            body.append("<li>");
271            body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_SUCCESS", List.of(String.valueOf(((List<String>) result.get(__RESULT_SUCCESS)).size())))));
272            body.append("</li>");
273        }
274        
275        // Partial
276        if (result.containsKey(__RESULT_PARTIAL))
277        {
278            body.append("<li>");
279            body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_PARTIAL", List.of(String.valueOf(((List<String>) result.get(__RESULT_PARTIAL)).size())))));
280            body.append("</li>");
281        }
282        
283        // Fail
284        if (result.containsKey(__RESULT_ERROR))
285        {
286            body.append("<li>");
287            body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_FAIL", List.of(String.valueOf(((List<String>) result.get(__RESULT_ERROR)).size())))));
288            body.append("</li>");
289        }
290        
291        body.append("</ul>");
292        body.append(_i18nUtils.translate(new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_SUCCESS_BODY_RESUME_ASK_DETAILS")));
293
294        return new I18nizableText(body.toString());
295    }
296
297    @Override
298    protected I18nizableText _getErrorMailSubject(JobExecutionContext context) throws Exception
299    {
300        return new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_ERROR_SUBJECT");
301    }
302
303    @Override
304    protected I18nizableText _getErrorMailBody(JobExecutionContext context, Throwable throwable) throws Exception
305    {
306        String error = StringUtils.join(ExceptionUtils.getStackFrames(throwable), "<br/>");
307        return new I18nizableText("plugin.odf", "PLUGINS_ODF_TABLEREF_SYNC_MAIL_ERROR_BODY", List.of(error));
308    }
309    
310    private void _addToListInMap(Map<String, Object> map, String itemName, String itemToAdd)
311    {
312        @SuppressWarnings("unchecked")
313        List<String> list = (List<String>) map.computeIfAbsent(itemName, __ -> new ArrayList<>());
314        list.add(itemToAdd);
315    }
316}