Source code

001/*
002 *  Copyright 2017 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.core.user;
017
018import java.io.File;
019import java.io.FileInputStream;
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.nio.charset.StandardCharsets;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.Collection;
026import java.util.HashMap;
027import java.util.List;
028import java.util.Locale;
029import java.util.Map;
030import java.util.Map.Entry;
031
032import org.apache.avalon.framework.component.Component;
033import org.apache.avalon.framework.service.ServiceException;
034import org.apache.avalon.framework.service.ServiceManager;
035import org.apache.avalon.framework.service.Serviceable;
036import org.apache.cocoon.servlet.multipart.Part;
037import org.apache.cocoon.servlet.multipart.PartOnDisk;
038import org.apache.cocoon.servlet.multipart.RejectedPart;
039import org.apache.commons.collections4.ListUtils;
040import org.apache.commons.io.FilenameUtils;
041import org.apache.commons.io.IOUtils;
042import org.apache.commons.io.input.BOMInputStream;
043import org.apache.commons.lang3.ArrayUtils;
044import org.apache.commons.lang3.StringUtils;
045
046import org.ametys.core.user.InvalidModificationException;
047import org.ametys.core.user.User;
048import org.ametys.core.user.UserIdentity;
049import org.ametys.core.user.UserManager;
050import org.ametys.core.util.I18nUtils;
051import org.ametys.runtime.i18n.I18nizableText;
052import org.ametys.runtime.parameter.Errors;
053import org.ametys.runtime.plugin.component.AbstractLogEnabled;
054
055/**
056 * Import users from a CSV or text file.
057 */
058public class ImportUsers extends AbstractLogEnabled implements Component, Serviceable
059{
060    /** Avalon Role */
061    public static final String ROLE = ImportUsers.class.getName();
062    
063    private static final String[] _ALLOWED_EXTENSIONS = new String[] {"txt", "csv"};
064    private static final String[] _COLUMNS = new String[] {"login", "password", "firstname", "lastname", "email", "salt"};
065    private static final String _COLUMN_LOGIN = "login";
066    private static final String _COLUMN_PASSWORD = "password";
067    private static final String _COLUMN_FIRSTNAME = "firstname";
068    private static final String _COLUMN_LASTNAME = "lastname";
069    private static final String _COLUMN_EMAIL = "email";
070    private static final String _COLUMN_SALT = "salt";
071
072    /** The subscribers DAO. */
073    protected UserDAO _usersDao;
074    /** User Manager. */
075    protected UserManager _userManager;
076    /** I18n utils */
077    protected I18nUtils _i18nUtils;
078    
079    @Override
080    public void service(ServiceManager smanager) throws ServiceException
081    {
082        _usersDao = (UserDAO) smanager.lookup(UserDAO.ROLE);
083        _userManager = (UserManager) smanager.lookup(UserManager.ROLE);
084        _i18nUtils = (I18nUtils) smanager.lookup(I18nUtils.ROLE);
085    }
086
087    /**
088     * Read a file from disk and add users inside in the requested population/directory
089     * @param filePart file
090     * @param populationId population Id
091     * @param userDirectoryId user directory
092     * @param cleanDirectory true to clear every user that was not in the import file
093     * @param hashedPasswords true if the passwords are already hashed in the file (salt must be in the file too, and password hashed with "saltPassword" with nothing in-between)
094     * @return a map containing success, addedCount, existingCount, errorCount, deletedCount, message
095     * success : true if success
096     * addedCount, existingCount, errorCount : number of imported user added, modified, an error occured (see server log)
097     * message : error message if something wrong happened (usually malformed file)
098     * @throws FileNotFoundException error reading the file
099     * @throws InvalidModificationException error modifying the directory
100     */
101    public Map<String, Object> importUsers(Part filePart, String populationId, String userDirectoryId, boolean cleanDirectory, boolean hashedPasswords) throws FileNotFoundException, InvalidModificationException
102    {
103        Map<String, Object> result = new HashMap<>();
104
105        if (filePart instanceof RejectedPart)
106        {
107            result.put("success", false);
108            result.put("error", "rejected-file");
109            result.put("message", new I18nizableText("plugin.core", "PLUGINS_CORE_USERS_IMPORT_ERROR_REJECTED_FILE"));
110            return result;
111        }
112
113        PartOnDisk uploadedFilePart = (PartOnDisk) filePart;
114        File uploadedFile = (uploadedFilePart != null) ? uploadedFilePart.getFile() : null;
115        String filename = (uploadedFilePart != null) ? uploadedFilePart.getFileName().toLowerCase() : null;
116
117        if (!FilenameUtils.isExtension(filename, _ALLOWED_EXTENSIONS))
118        {
119            result.put("success", false);
120            result.put("error", "invalid-extension");
121            result.put("message", new I18nizableText("plugin.core", "PLUGINS_CORE_USERS_IMPORT_ERROR_INVALID_EXTENSION"));
122            return result;
123        }
124
125        try (FileInputStream fileIS =  new FileInputStream(uploadedFile); 
126             BOMInputStream bomIS = new BOMInputStream(fileIS))
127        {
128            Map<String, List<UserIdentity>> parsedUsers = parseFile(bomIS, populationId, userDirectoryId, hashedPasswords);
129            
130            List<UserIdentity> addedUsers = parsedUsers.get("added");
131            List<UserIdentity> editedUsers = parsedUsers.get("edited");
132            
133            List<Map<String, String>> removedUsers;
134            if (cleanDirectory)
135            {
136                List<UserIdentity> usersToSave = ListUtils.union(addedUsers, editedUsers);
137                removedUsers = clearUserDirectory(populationId, userDirectoryId, usersToSave);
138            }
139            else
140            {
141                removedUsers = new ArrayList<>();
142            }
143    
144            result.put("success", true);
145            result.put("addedCount", addedUsers.size());
146            result.put("existingCount", editedUsers.size());
147            result.put("errorCount", parsedUsers.get("error").size());
148            result.put("deletedCount", removedUsers.size());
149        }
150        catch (IOException ioe)
151        {
152            result.put("success", false);
153            result.put("error", IOException.class.getName());
154            result.put("message", ioe.getLocalizedMessage());
155        }
156        catch (ImportUserActionException iuae)
157        {
158            result.put("success", false);
159            result.put("error", iuae.getErrorName());
160            result.put("message", iuae.getI18nizableText());
161        }
162
163        return result;
164    }
165    
166    /**
167     * Remove all users in a userDirectory, avoiding those in usersToSave list
168     * @param populationId population Id
169     * @param userDirectoryId directory Id
170     * @param usersToSave list of users that need to be saved
171     * @return list of deleted users (containing login and populationId)
172     * @throws InvalidModificationException If modification is not possible
173     */
174    protected List<Map<String, String>> clearUserDirectory(String populationId, String userDirectoryId, List<UserIdentity> usersToSave) throws InvalidModificationException
175    {
176        Map<String, Object> parameters =  new HashMap<>();
177        Collection<User> existingUsersInDirectory = _userManager.getUsersByDirectory(populationId, userDirectoryId, Integer.MAX_VALUE, 0, parameters);
178        List<Map<String, String>> usersToDelete = new ArrayList<>();
179        UserIdentity currentUser = _usersDao._getCurrentUser();
180        for (User user : existingUsersInDirectory)
181        {
182            UserIdentity ui = user.getIdentity();
183            // If the user is not in the "save" list, and is not the current user, add it to the delete list
184            if (!usersToSave.contains(ui) && (currentUser == null || !currentUser.equals(ui)))
185            {
186                Map<String, String> userTodelete = new HashMap<>();
187                userTodelete.put("login", ui.getLogin());
188                userTodelete.put("populationId", ui.getPopulationId());
189                usersToDelete.add(userTodelete);
190            }
191        }
192        _usersDao.deleteUsers(usersToDelete);
193        return usersToDelete;
194    }
195    
196    /**
197     * Parse the file to add new users in the directory
198     * @param bomIS input stream
199     * @param populationId population Id
200     * @param userDirectoryId directory Id
201     * @param hashedPassword true if the password is hashed
202     * @return a map, containing added/edited/error, each containing the logins in each list
203     * @throws IOException if an error occurs
204     * @throws ImportUserActionException file parsing failed
205     */
206    @SuppressWarnings("unchecked")
207    protected Map<String, List<UserIdentity>> parseFile(BOMInputStream bomIS, String populationId, String userDirectoryId, boolean hashedPassword) throws IOException, ImportUserActionException
208    {
209        Map<String, Integer> order = new HashMap<>();
210
211        List<UserIdentity> addedUsers = new ArrayList<>();
212        List<UserIdentity> editedUsers = new ArrayList<>();
213        List<UserIdentity> errors = new ArrayList<>();
214
215        boolean headerRead = false;
216        
217        for (String line : IOUtils.readLines(bomIS, StandardCharsets.UTF_8))
218        {
219            String[] columns = Arrays.stream(line.split(";")).map(StringUtils::normalizeSpace).toArray(String[]::new);
220            if (!headerRead)
221            {
222                _checkCsvHeader(columns, hashedPassword);
223                for (int i = 0; i < columns.length; i++)
224                {
225                    String string = columns[i];
226                    order.put(string, i);
227                }
228                headerRead = true;
229            }
230            else
231            {
232                Map<String, String> untypedValues = new HashMap<>();
233                String login = columns[order.get(_COLUMN_LOGIN)];
234                UserIdentity userIdentity = new UserIdentity(login, populationId);
235                untypedValues.put("login", login);
236                untypedValues.put("password", columns[order.get(_COLUMN_PASSWORD)]);
237                untypedValues.put("firstname", columns[order.get(_COLUMN_FIRSTNAME)]);
238                untypedValues.put("lastname", columns[order.get(_COLUMN_LASTNAME)]);
239                untypedValues.put("email", columns[order.get(_COLUMN_EMAIL)]);
240                if (hashedPassword)
241                {
242                    untypedValues.put("salt", columns[order.get(_COLUMN_SALT)]);
243                    untypedValues.put("clearText", "false");
244                }
245                else
246                {
247                    untypedValues.put("clearText", "true");
248                }
249                try
250                {
251                    User user = _userManager.getUser(populationId, login);
252                    if (user == null)
253                    {
254                        Map<String, Object> addedUser = _usersDao.addUser(populationId, userDirectoryId, untypedValues);
255                        if (addedUser.containsKey("errors"))
256                        {
257                            Map<String, Errors> fieldErrors = (Map<String, Errors>) addedUser.get("errors");
258                            errors.add(userIdentity);
259                            getLogger().error("Failed to import user from inputs [{}] for population '{}' and user directory '{}' with the following error(s):\n{}", line, populationId, userDirectoryId, _parseErrors(fieldErrors));
260                        }
261                        else
262                        {
263                            addedUsers.add(userIdentity);
264                        }
265                    }
266                    else
267                    {
268                        Map<String, Object> editedUser = _usersDao.editUser(populationId, untypedValues);
269                        if (editedUser.containsKey("errors"))
270                        {
271                            Map<String, Errors> fieldErrors = (Map<String, Errors>) editedUser.get("errors");
272                            errors.add(userIdentity);
273                            getLogger().error("Failed to edit user from inputs [{}] for population '{}' and user directory '{}' with the following error(s):\n{}", line, populationId, userDirectoryId, _parseErrors(fieldErrors));
274                        }
275                        else
276                        {
277                            editedUsers.add(userIdentity);
278                        }
279                    }
280                }
281                catch (InvalidModificationException e)
282                {
283                    errors.add(userIdentity);
284                    getLogger().error("Unable to import user from inputs [{}] for population '{}' and user directory '{}'", line, populationId, userDirectoryId, e);
285                }
286            }
287        }
288        
289        Map<String, List<UserIdentity>> result = new HashMap<>();
290        result.put("added", addedUsers);
291        result.put("edited", editedUsers);
292        result.put("error", errors);
293        return result;
294    }
295    
296    private String _parseErrors(Map<String, Errors> errors)
297    {
298        StringBuilder sb = new StringBuilder();
299        
300        for (Entry<String, Errors> entry : errors.entrySet())
301        {
302            sb.append("[" + entry.getKey() + "] ");
303            List<I18nizableText> errorLabels = entry.getValue().getErrors();
304            for (I18nizableText errorLabel : errorLabels)
305            {
306                sb.append(_i18nUtils.translate(errorLabel, Locale.ENGLISH.getLanguage())).append("; ");
307            }
308        }
309        return sb.toString();
310    }
311    
312    /**
313     * Check a line (the header) and throw an exception if not correct
314     * @param columns list of fields
315     * @param hashedPassword is the password in clear text ? (special case for 'salt' column)
316     * @throws ImportUserActionException thrown if error
317     */
318    private void _checkCsvHeader(String[] columns, boolean hashedPassword) throws ImportUserActionException
319    {
320        for (String expectedColumn : _COLUMNS)
321        {
322            // Check if salt column is present even if the password is not hashed
323            if (!hashedPassword && expectedColumn.equals(_COLUMN_SALT) && ArrayUtils.contains(columns, expectedColumn))
324            {
325                throw new ImportUserActionException("PLUGINS_CORE_USERS_IMPORT_ERROR_COLUMN_SALT_PRESENT", "column-salt");
326            }
327            // If column not found (unless this is salt and password is not hashed)
328            if (!ArrayUtils.contains(columns, expectedColumn) && (hashedPassword || !expectedColumn.equals(_COLUMN_SALT)))
329            {
330                List<String> params = new ArrayList<>(1);
331                params.add(expectedColumn);
332                throw new ImportUserActionException("PLUGINS_CORE_USERS_IMPORT_ERROR_COLUMN_MISSING", "column-missing", params);
333            }
334            // If column is present, but multiple times
335            if (ArrayUtils.contains(columns, expectedColumn) && ArrayUtils.indexOf(columns, expectedColumn) != ArrayUtils.lastIndexOf(columns, expectedColumn))
336            {
337                List<String> params = new ArrayList<>(1);
338                params.add(expectedColumn);
339                throw new ImportUserActionException("PLUGINS_CORE_USERS_IMPORT_ERROR_COLUMN_DUPLICATE", "column-duplicate", params);
340            }
341        }
342    }
343
344    /**
345     * Internal exception to throw with a i18n message
346     */
347    private static class ImportUserActionException extends Exception
348    {
349        private I18nizableText _i18nizableText;
350        private String _errorName;
351
352        /**
353         * Create an exception with a parametrized message
354         * @param i18nkey i18n key
355         * @param errorName technical error name
356         * @param params parameters
357         */
358        public ImportUserActionException(String i18nkey, String errorName, List<String> params)
359        {
360            this._i18nizableText = new I18nizableText("plugin.core", i18nkey, params);
361            this._errorName = errorName;
362        }
363        /**
364         * Create an exception with a message
365         * @param i18nkey i18n key
366         * @param errorName technical error name
367         */
368        public ImportUserActionException(String i18nkey, String errorName)
369        {
370            this._i18nizableText = new I18nizableText("plugin.core", i18nkey);
371            this._errorName = errorName;
372        }
373        /**
374         * Get the I18nizableText
375         * @return I18nizableText representing the error message
376         */
377        public I18nizableText getI18nizableText()
378        {
379            return this._i18nizableText;
380        }
381        /**
382         * Get the technical error name
383         * @return technical error name
384         */
385        public String getErrorName()
386        {
387            return this._errorName;
388        }
389    }
390}