/*
 *  Copyright 2014 Anyware Services
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.ametys.plugins.contentio.in.csv;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.service.ServiceException;
import org.apache.avalon.framework.service.ServiceManager;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.tika.parser.txt.CharsetDetector;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;
import org.supercsv.io.CsvMapReader;
import org.supercsv.io.ICsvMapReader;
import org.supercsv.io.Tokenizer;
import org.supercsv.prefs.CsvPreference;

import org.ametys.cms.contenttype.ContentConstants;
import org.ametys.cms.contenttype.ContentTypesHelper;
import org.ametys.cms.contenttype.MetadataDefinition;
import org.ametys.cms.contenttype.MetadataType;
import org.ametys.cms.repository.Content;
import org.ametys.cms.repository.ModifiableContent;
import org.ametys.plugins.contentio.AbstractContentImporter;
import org.ametys.plugins.contentio.ContentImporter;
import org.ametys.plugins.contentio.ContentImporterHelper;
import org.ametys.plugins.repository.metadata.ModifiableCompositeMetadata;
import org.ametys.plugins.repository.version.VersionableAmetysObject;

/**
 * {@link ContentImporter} importing contents from a CSV file.
 * Each CSV record (line) contains content properties.
 * Configuration options:
 * <ul>
 *   <li>The CSV file charset (default: auto-detect)</li>
 *   <li>The CSV delimiter character (default: auto-detect from the header)</li>
 *   <li>The CSV quote character (default to the double-quote: <code>"</code>)</li>
 *   <li>Structure of the header line: fixed pattern or column list.</li>
 *   <li>Mapping from CSV columns to content metadatas.</li>
 * </ul>
 */
public class CsvContentImporter extends AbstractContentImporter
{
    
    /** The content type helper. */
    protected ContentTypesHelper _cTypeHelper;
    
    /** The file charset. */
    protected Charset _charset;
    
    /** The CSV delimiter character. */
    protected Character _delimiterChar;
    
    /** The CSV quote character. */
    protected Character _quoteChar;
    
    /** True if the supported CSV files have a header formed of the columns (this is not always the case). */
    protected boolean _columnHeaderLine;
    
    /** Determine if the file is supported by matching the header line against this pattern. */
    protected Pattern _matchPattern;
    
    /** Determine if the file is supported by detecting the following columns in the header. */
    protected Set<String> _matchColumns;
    
    /** Contains mapping from CSV column to content metadata path. */
    protected Map<String, String> _columnToMetadata;
    
    @Override
    public void service(ServiceManager serviceManager) throws ServiceException
    {
        super.service(serviceManager);
        _cTypeHelper = (ContentTypesHelper) serviceManager.lookup(ContentTypesHelper.ROLE);
    }
    
    @Override
    public void configure(Configuration configuration) throws ConfigurationException
    {
        // Configure priority, allowed extensions, content creation parameters.
        super.configure(configuration);
        
        // Configure CSV parsing and mapping properties.
        configureCsvProperties(configuration.getChild("csv"));
    }
    
    /**
     * Configure CSV parsing and mapping properties.
     * @param configuration the CSV configuration.
     * @throws ConfigurationException if an error occurs.
     */
    protected void configureCsvProperties(Configuration configuration) throws ConfigurationException
    {
        String charsetName = configuration.getAttribute("charset", null);
        if (StringUtils.isNotEmpty(charsetName))
        {
            try
            {
                _charset = Charset.forName(charsetName);
            }
            catch (Exception e)
            {
                throw new ConfigurationException("Invalid charset: " + charsetName, e);
            }
        }
        
        String delimiter = configuration.getAttribute("delimiter", null);
        if (StringUtils.isNotEmpty(delimiter))
        {
            _delimiterChar = delimiter.charAt(0);
        }
        
        String quote = configuration.getAttribute("quote", null);
        if (StringUtils.isNotEmpty(quote))
        {
            _quoteChar = quote.charAt(0);
        }
        
        _columnHeaderLine = configuration.getAttributeAsBoolean("columnHeader", true);
        
        // Match pattern or column list.
        String matchPattern = configuration.getChild("match").getAttribute("pattern", null);
        String matchColumns = configuration.getChild("match").getAttribute("columns", null);
        
        if (matchPattern != null && matchColumns == null)
        {
            _matchPattern = Pattern.compile(matchPattern);
        }
        else if (matchPattern == null && matchColumns != null)
        {
            _matchColumns = new HashSet<>();
            for (String column : StringUtils.split(matchColumns, ", "))
            {
                _matchColumns.add(column.trim());
            }
        }
        else
        {
            throw new ConfigurationException("A CSV content importer must match a pattern or a column list, but not both.", configuration);
        }
        
        // Configure mappings.
        _columnToMetadata = new HashMap<>();
        
        for (Configuration mappingConf : configuration.getChild("mappings").getChildren("mapping"))
        {
            String column = mappingConf.getAttribute("column");
            String metadata = mappingConf.getAttribute("metadata");
            
            _columnToMetadata.put(column, metadata);
        }
    }
    
    @Override
    protected Collection<String> getDefaultExtensions()
    {
        return Arrays.asList("csv", "tsv");
    }
    
    @Override
    public boolean supports(InputStream is, String name) throws IOException
    {
        if (name == null || isExtensionValid(name))
        {
            if (_matchPattern != null)
            {
                return matchHeaderPattern(is);
            }
            else // if (_matchColumns != null)
            {
                return matchColumns(is);
            }
        }
        return false;
    }
    
    /**
     * Test if the importer supports the given file by matching its first line against the configured pattern.
     * @param is an input stream on the data to test.
     * @return true if the data's first line matches the pattern, false otherwise.
     * @throws IOException if a read error occurs.
     */
    protected boolean matchHeaderPattern(InputStream is) throws IOException
    {
        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
        String header = reader.readLine();
        
        return _matchPattern.matcher(header).matches();
    }
    
    /**
     * Test if the importer supports the given file by testing if it contains the configured columns in its header.
     * @param is an input stream on the data to test.
     * @return true if the CSV columns contain all the configured columns, false otherwise.
     * @throws IOException if a read error occurs.
     */
    protected boolean matchColumns(InputStream is) throws IOException
    {
        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
        String header = reader.readLine();
        
        Map<String, Object> params = new HashMap<>();
        
        CsvPreference preference = getCsvPreference(header, params);
        
        try (Tokenizer tok = new Tokenizer(new StringReader(header + "\n"), preference);)
        {
            List<String> columns = new ArrayList<>();
            tok.readColumns(columns);
            
            return columns.containsAll(_matchColumns);
        }
    }
    
    private ICsvMapReader _getMapReader(BufferedReader reader, CsvPreference preference) throws IOException
    {
        if (_columnHeaderLine)
        {
            // Reset the reader (go back to the beginning of the file).
            reader.reset();
            return new CsvMapReader(reader, preference);
        }
        else
        {
            // No named columns: use a custom CSV map reader which uses the column number as the map index.
            return new CsvColNumberMapReader(reader, preference);
        }
    }
    
    @Override
    public Set<String> importContents(InputStream is, Map<String, Object> params) throws IOException
    {
        Set<String> contentIds = new HashSet<>();
        
        // Get a reader using the right charset and wrap it in a buffered reader.
        BufferedReader reader = new BufferedReader(getReader(is), 8192);
        
        // Mark the start of file, to be able to reset it.
        reader.mark(8192);
        String headerLine = reader.readLine();
        
        if (headerLine != null)
        {
            CsvPreference preference = getCsvPreference(headerLine, params);
            
            try (ICsvMapReader mapReader = _getMapReader(reader, preference))
            {
                // Get the columns from the header line, if applicable.
                String[] columns = _columnHeaderLine ? mapReader.getHeader(true) : new String[0];
                
                Map<String, String> properties;
                while ((properties = mapReader.read(columns)) != null)
                {
                    String contentId = importContent(properties, params, mapReader.getLineNumber());
                    
                    if (contentId != null)
                    {
                        contentIds.add(contentId);
                    }
                }
            }
        }
        
        return contentIds;
    }
    
    /**
     * Get a reader on the data stream, optionally detecting the charset.
     * @param in the data stream.
     * @return the reader with the correct character set.
     */
    protected Reader getReader(InputStream in)
    {
        if (_charset != null)
        {
            // Return an InputStreamReader with the configured charset.
            return new InputStreamReader(in, _charset);
        }
        else
        {
            // Use Tika/ICU to detect the file charset.
            BufferedInputStream buffIs = new BufferedInputStream(in);
            
            CharsetDetector detector = new CharsetDetector();
            return detector.getReader(buffIs, Charset.defaultCharset().name());
        }
    }
    
    /**
     * Get the CSV preference.
     * @param header the CSV first line.
     * @param params the import parameters.
     * @return a {@link CsvPreference} object.
     */
    protected CsvPreference getCsvPreference(String header, Map<String, Object> params)
    {
        char delimiter = getDelimiter(header, params);
        char quoteChar = getQuoteChar(params);
        
        return new CsvPreference.Builder(quoteChar, delimiter, "\r\n").build();
    }
    
    /**
     * Get the CSV character delimiter.
     * @param header the CSV first line.
     * @param params the import parameters.
     * @return the CSV character delimiter.
     */
    protected char getDelimiter(String header, Map<String, Object> params)
    {
        char delimiter = ',';
        
        if (_delimiterChar != null) // The delimiter char is specified
        {
            delimiter = _delimiterChar;
        }
        else if (header.contains("\t")) // Else, try to auto-detect.
        {
            delimiter = '\t';
        }
        else if (header.contains(";"))
        {
            delimiter = ';';
        }
        else if (header.contains(","))
        {
            delimiter = ',';
        }
        
        return delimiter;
    }
    
    /**
     * Get the CSV quote character.
     * @param params the import parameters.
     * @return the CSV quote character.
     */
    protected char getQuoteChar(Map<String, Object> params)
    {
        char quote = '"';
        
        if (_quoteChar != null)
        {
            quote = _quoteChar;
        }
        
        return quote;
    }
    
    /**
     * Import a content from a CSV record.
     * @param properties the CSV record as a Map of values, indexed by column name or number.
     * @param params the import parameters.
     * @param lineNumber the line number of the record being imported, for logging purposes.
     * @return the content ID or null if the content was not created.
     */
    protected String importContent(Map<String, String> properties, Map<String, Object> params, int lineNumber)
    {
        try
        {
            // Map properties to metadata.
            Map<String, String> metadata = getMetadataFromProperties(properties);
            
            String title = metadata.get("title");
            
            Content content = createContent(title, params);
            
            if (content instanceof ModifiableContent)
            {
                setMetadatas((ModifiableContent) content, metadata, params);
            }
            else
            {
                getLogger().error("Import from CSV file: the content on line " + lineNumber + " was imported as a read-only content, it could not be modified.");
            }
            
            return content.getId();
        }
        catch (Exception e)
        {
            getLogger().error("Import from CSV file: error importing the content on line " + lineNumber, e);
        }
        
        return null;
    }
    
    /**
     * Get the content metadata from a CSV record.
     * @param properties the CSV record as a Map of values, indexed by column name or number.
     * @return a Map of metadata values, indexed by metadata path.
     */
    protected Map<String, String> getMetadataFromProperties(Map<String, String> properties)
    {
        Map<String, String> metadata = new HashMap<>();
        
        for (String propName : properties.keySet())
        {
            String value = properties.get(propName);
            String metaName = null;
            if (_columnToMetadata.containsKey(propName))
            {
                metaName = _columnToMetadata.get(propName);
            }
            else if (_columnHeaderLine)
            {
                metaName = propName;
            }
            
            if (metaName != null)
            {
                metadata.put(metaName, value);
            }
        }
        
        return metadata;
    }
    
    /**
     * Set the content metadatas from the CSV values.
     * @param content the content to populate.
     * @param metaValues the metadata values, extracted from the CSV record.
     * @param params the import parameters.
     */
    protected void setMetadatas(ModifiableContent content, Map<String, String> metaValues, Map<String, Object> params)
    {
        for (String path : metaValues.keySet())
        {
            String value = metaValues.get(path);
            
            if (value != null)
            {
                setMetadata(content, path, value, params);
            }
        }
        
        // Save changes and create a version.
        content.saveChanges();
        if (content instanceof VersionableAmetysObject)
        {
            ((VersionableAmetysObject) content).checkpoint();
        }
    }

    /**
     * Set a metadata from its string value.
     * @param content the content to populate.
     * @param path the metadata path.
     * @param value the metadata string value.
     * @param params the import parameters.
     */
    protected void setMetadata(ModifiableContent content, String path, String value, Map<String, Object> params)
    {
        ModifiableCompositeMetadata metaHolder = content.getMetadataHolder();
        MetadataDefinition metaDef = null;
        
        // Iterate over path parts while they are composites.
        String[] pathElements = StringUtils.split(path, ContentConstants.METADATA_PATH_SEPARATOR);
        for (int i = 0; i < (pathElements.length - 1); i++)
        {
            String compositeName = pathElements[i];
            
            // Get metadata definition and metadata holder for this level.
            metaDef = getMetadataDefinition(content, metaDef, compositeName);
            if (metaDef != null && metaDef.getType() == MetadataType.COMPOSITE)
            {
                metaHolder = metaHolder.getCompositeMetadata(compositeName, true);
            }
        }
        
        // Last path element: get metadata name and definition.
        String metaName = pathElements[pathElements.length - 1];
        metaDef = getMetadataDefinition(content, metaDef, metaName);
        
        if (metaDef != null)
        {
            try
            {
//                if (metaDef.isMultiple())
//                {
//                    setMultipleMetadata(metaHolder, metaDef, metaName, value, params);
//                }
//                else
//                {
                setMetadata(metaHolder, metaDef, metaName, value, params);
//                }
            }
            catch (Exception e)
            {
                String message = "The value for metadata '" + metaName + "' is invalid and will be ignored: " + value;
                getLogger().warn(message, e);
            }
        }
    }

    /**
     * Get a metadata definition, either from the parent metadata definition or from the content itself.
     * @param content the imported content.
     * @param parentMetaDef the parent metadata definition.
     * @param name the metadata name.
     * @return the metadata definition.
     */
    protected MetadataDefinition getMetadataDefinition(Content content, MetadataDefinition parentMetaDef, String name)
    {
        MetadataDefinition metaDef = null;
        
        if (parentMetaDef == null)
        {
            metaDef = _cTypeHelper.getMetadataDefinition(name, content.getTypes(), content.getMixinTypes());
        }
        else
        {
            metaDef = parentMetaDef.getMetadataDefinition(name);
        }
        
        return metaDef;
    }
    
    /**
     * Set a single metadata.
     * @param meta the metadata holder.
     * @param metaDef the metadata definition.
     * @param name the metadata name.
     * @param value the metadata value as a String.
     * @param params the import parameters.
     * @throws IOException if an error occurs.
     */
    protected void setMetadata(ModifiableCompositeMetadata meta, MetadataDefinition metaDef, String name, String value, Map<String, Object> params) throws IOException
    {
        switch (metaDef.getType())
        {
            case STRING:
                setStringMetadata(meta, name, metaDef, new String [] {value});
                break;
            case BOOLEAN:
                setBooleanMetadata(meta, name, metaDef, new String [] {value});
                break;
            case LONG:
                setLongMetadata(meta, name, metaDef, new String [] {value});
                break;
            case DOUBLE:
                setDoubleMetadata(meta, name, metaDef, new String [] {value});
                break;
            case DATE:
            case DATETIME:
                setDateMetadata(meta, name, metaDef, new String [] {value});
                break;
            case GEOCODE:
                break;
            case RICH_TEXT:
                setRichText(meta, name, value);
                break;
            case BINARY:
            case FILE:
                setBinaryMetadata(meta, name, metaDef, value);
                break;
            case COMPOSITE:
            case USER:
            case REFERENCE:
            case CONTENT:
            case SUB_CONTENT:
            case MULTILINGUAL_STRING:
                // Not supported
            default:
                break;
        }
    }
    
    /**
     * Set a RichText metadata from a String value.
     * @param meta the metadata holder.
     * @param name the metadata name.
     * @param value the String value.
     * @throws IOException if something goes wrong when manipulating files
     */
    protected void setRichText(ModifiableCompositeMetadata meta, String name, String value) throws IOException
    {
        String cleanValue = Jsoup.clean(value, Whitelist.none());
        
        cleanValue = StringEscapeUtils.escapeXml10(cleanValue);
        
        String[] lines = StringUtils.split(cleanValue, "\r\n");
        
        String docbook = ContentImporterHelper.textToDocbook(lines);
        setRichText(meta, name, new ByteArrayInputStream(docbook.getBytes("UTF-8")));
    }
    
}