Source code

001/*
002 *  Copyright 2020 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.plugins.webcontentio.archive;
017
018import java.io.IOException;
019import java.io.InputStream;
020import java.nio.file.DirectoryStream;
021import java.nio.file.Files;
022import java.nio.file.Path;
023import java.util.ArrayList;
024import java.util.Collection;
025import java.util.Collections;
026import java.util.List;
027import java.util.Map;
028import java.util.Optional;
029import java.util.function.Function;
030import java.util.stream.Collectors;
031import java.util.stream.Stream;
032import java.util.stream.StreamSupport;
033
034import javax.jcr.Node;
035import javax.jcr.RepositoryException;
036import javax.xml.parsers.DocumentBuilder;
037import javax.xml.parsers.DocumentBuilderFactory;
038import javax.xml.parsers.ParserConfigurationException;
039import javax.xml.transform.TransformerException;
040
041import org.apache.commons.lang3.StringUtils;
042import org.apache.xpath.XPathAPI;
043import org.slf4j.Logger;
044import org.w3c.dom.Document;
045import org.w3c.dom.Element;
046import org.xml.sax.SAXException;
047
048import org.ametys.cms.repository.DefaultContent;
049import org.ametys.core.util.LambdaUtils;
050import org.ametys.plugins.contentio.archive.Archivers;
051import org.ametys.plugins.contentio.archive.Archivers.AmetysObjectNotImportedException;
052import org.ametys.plugins.contentio.archive.ContentsArchiverHelper.ContentFiller;
053import org.ametys.plugins.contentio.archive.ImportGlobalFailException;
054import org.ametys.plugins.contentio.archive.ImportReport;
055import org.ametys.plugins.contentio.archive.Merger;
056import org.ametys.plugins.contentio.archive.ResourcesAdditionalDataGetter;
057import org.ametys.plugins.contentio.archive.UnitaryImporter;
058import org.ametys.plugins.contentio.archive.ZipEntryHelper;
059import org.ametys.plugins.repository.ModifiableTraversableAmetysObject;
060import org.ametys.plugins.repository.collection.AmetysObjectCollection;
061import org.ametys.plugins.repository.data.extractor.xml.ModelAwareXMLValuesExtractor;
062import org.ametys.plugins.repository.data.extractor.xml.XMLValuesExtractorAdditionalDataGetter;
063import org.ametys.runtime.model.View;
064import org.ametys.web.repository.ModifiableSiteAwareAmetysObject;
065import org.ametys.web.repository.site.Site;
066import org.ametys.web.repository.site.SiteType;
067
068class SiteImporter
069{
070    final ImportReport _report = new ImportReport();
071    private final SitesArchiver _siteArchiver;
072    private final ModifiableTraversableAmetysObject _root;
073    private final Path _zipArchivePath;
074    private final Merger _merger;
075    private final Logger _logger;
076    private final DocumentBuilder _builder;
077    
078    SiteImporter(SitesArchiver siteArchiver, ModifiableTraversableAmetysObject root, Path zipArchivePath, Merger merger, Logger logger) throws ParserConfigurationException
079    {
080        _siteArchiver = siteArchiver;
081        _root = root;
082        _zipArchivePath = zipArchivePath;
083        _merger = merger;
084        _logger = logger;
085        _builder = DocumentBuilderFactory.newInstance()
086                .newDocumentBuilder();
087    }
088    
089    private class UnitarySiteImporter implements UnitaryImporter<Site>
090    {
091        private String _siteName;
092        
093        UnitarySiteImporter(String siteName)
094        {
095            _siteName = siteName;
096        }
097        
098        @Override
099        public String objectNameForLogs()
100        {
101            return "Site";
102        }
103
104        @Override
105        public Document getPropertiesXml(Path zipEntryPath) throws Exception
106        {
107            return _getSitePropertiesXml(zipEntryPath);
108        }
109
110        @Override
111        public String retrieveId(Document propertiesXml) throws Exception
112        {
113            return Archivers.xpathEvalNonEmpty("site/@id", propertiesXml);
114        }
115
116        @Override
117        public Site create(Path zipEntryPath, String id, Document propertiesXml) throws AmetysObjectNotImportedException, Exception
118        {
119            return _createSite(zipEntryPath, id, _siteName, propertiesXml);
120        }
121        
122        @Override
123        public ImportReport getReport()
124        {
125            return _report;
126        }
127        
128        @Override
129        public void unitaryImportFinalize()
130        {
131            _siteArchiver._siteManager.clearCache();
132            UnitaryImporter.super.unitaryImportFinalize();
133        }
134    }
135    
136    void importSites() throws IOException
137    {
138        Collection<Path> sitePaths = _retrieveSites();
139        for (Path sitePath : sitePaths)
140        {
141            String siteName = sitePath.getFileName().toString();
142            _importSite(sitePath, siteName);
143        }
144    }
145    
146    void importSite(String siteName) throws IOException
147    {
148        String path = Archivers.getHashedPath(siteName);
149        String prefix = SitesArchiver.__COMMON_PREFIX + path + "/" + siteName;
150        Path sitePath = ZipEntryHelper.zipFileRoot(_zipArchivePath)
151                .resolve(prefix);
152        _importSite(sitePath, siteName);
153    }
154    
155    private Collection<Path> _retrieveSites() throws IOException
156    {
157        Collection<Path> sitePaths = new SiteRetriever().getThreeDepthPaths();
158        return sitePaths;
159    }
160    
161    /**
162     * Inner class to retrieve from:
163     * 
164     * sites
165     * |_45
166     * __|_7d
167     * ____|_www
168     * |_7b
169     * __|_d7
170     * ____|_site1
171     * 
172     * the result paths:
173     * # /sites/45/7d/www
174     * # /sites/7b/d7/site1
175     * 
176     */
177    private class SiteRetriever
178    {
179        Collection<Path> getThreeDepthPaths() throws IOException
180        {
181            return _getThreeDepthPaths(SitesArchiver.__COMMON_PREFIX, 0)
182                    .collect(Collectors.toList());
183        }
184        
185        private Stream<Path> _getThreeDepthPaths(String currentPath, int count) throws IOException
186        {
187            DirectoryStream<Path> nextLevels = ZipEntryHelper.children(
188                _zipArchivePath, 
189                Optional.of(currentPath), 
190                Files::isDirectory);
191            return _getNextLevelPathDirectories(nextLevels, count);
192        }
193        
194        private Stream<Path> _getThreeDepthPaths(Path currentPath, int count) throws IOException
195        {
196            if (count == 3)
197            {
198                return Stream.of(currentPath);
199            }
200            else
201            {
202                return _getThreeDepthPaths(currentPath.toString(), count);
203            }
204        }
205        
206        private Stream<Path> _getNextLevelPathDirectories(DirectoryStream<Path> currentLevels, int count)
207        {
208            return StreamSupport.stream(currentLevels.spliterator(), false)
209                    .map(LambdaUtils.wrap(currentLevel -> _getThreeDepthPaths(currentLevel, count + 1)))
210                    .flatMap(Function.identity());
211        }
212    }
213    
214    private void _importSite(Path sitePath, String siteName) throws IOException
215    {
216        new UnitarySiteImporter(siteName)
217                .unitaryImport(_zipArchivePath, sitePath, _merger, _logger);
218    }
219    
220    private Document _getSitePropertiesXml(Path sitePath) throws IOException
221    {
222        String propertiesFileName = sitePath.getFileName() + ".xml";
223        String zipEntryPath = sitePath.resolve(propertiesFileName).toString();
224        try (InputStream stream = ZipEntryHelper.zipEntryFileInputStream(_zipArchivePath, zipEntryPath))
225        {
226            Document doc = _builder.parse(stream);
227            return doc;
228        }
229        catch (SAXException e)
230        {
231            throw new IOException(e);
232        }
233    }
234    
235    private Site _createSite(Path sitePath, String id, String siteName, Document propertiesXml) throws ImportGlobalFailException, AmetysObjectNotImportedException, RepositoryException, TransformerException, IOException, Exception
236    {
237        _logger.info("Importing site from '{}!{}' ...", _zipArchivePath, sitePath);
238        Site site = _createChildSite(id, siteName);
239        site.setType(Archivers.xpathEvalNonEmpty("site/@type", propertiesXml));
240        
241        XMLValuesExtractorAdditionalDataGetter additionalDataGetter = new ResourcesAdditionalDataGetter(_zipArchivePath, sitePath);
242        _setSiteProperties(site, propertiesXml, additionalDataGetter);
243        
244        Archivers.unitarySave(site.getNode(), _logger);
245        
246        ImportReport importResourceReport = _importResources(site, sitePath);
247        _report.addFrom(importResourceReport);
248        ImportReport importContentReport = _importContents(site, sitePath, siteName);
249        _report.addFrom(importContentReport);
250        ImportReport importPluginReport = _importPlugins(site, sitePath);
251        _report.addFrom(importPluginReport);
252        ImportReport importSitemapReport = _importSitemaps(site, sitePath);
253        _report.addFrom(importSitemapReport);
254        
255        return site;
256    }
257    
258    private Site _createChildSite(String id, String siteName) throws RepositoryException
259    {
260        String uuid = StringUtils.substringAfter(id, "://");
261        // Create a Node with JCR primary type "ametys:site"
262        // But then call '_replaceNodeWithDesiredUuid' to have it with the desired UUID (srcNode will be removed)
263        Node srcNode = ((Site) _root.createChild(siteName, "ametys:site")).getNode();
264        Node siteNode = Archivers.replaceNodeWithDesiredUuid(srcNode, uuid);
265        return _siteArchiver._resolver.resolve(_root.getPath(), siteNode, null, false);
266    }
267    
268    private void _setSiteProperties(Site site, Document propertiesXml, XMLValuesExtractorAdditionalDataGetter additionalDataGetter) throws TransformerException, Exception
269    {
270        Element siteElement = (Element) XPathAPI.selectSingleNode(propertiesXml, "site");
271        SiteType siteType = _siteArchiver._siteTypeEP.getExtension(site.getType());
272        View view = View.of(siteType);
273        Map<String, Object> values = new ModelAwareXMLValuesExtractor(siteElement, additionalDataGetter, siteType)
274                .extractValues(view);
275        
276        site.getDataHolder().synchronizeValues(view, values);
277    }
278    
279    private ImportReport _importResources(Site site, Path sitePath) throws IOException
280    {
281        String baseImportResourcePath = sitePath.resolve("resources")
282                .toString();
283        if (ZipEntryHelper.zipEntryFolderExists(_zipArchivePath, baseImportResourcePath))
284        {
285            Node parentOfRootResources = site.getNode();
286            return _siteArchiver._resourcesArchiverHelper.importCollection(baseImportResourcePath + "/", parentOfRootResources, _zipArchivePath, _merger);
287        }
288        else
289        {
290            _logger.info("No resource to be imported for Site '{}', the path '{}!{}' does not exist", site, _zipArchivePath, baseImportResourcePath);
291            return new ImportReport();
292        }
293    }
294    
295    private ImportReport _importContents(Site site, Path sitePath, String siteName) throws IOException
296    {
297        AmetysObjectCollection rootContents = site.hasChild(SitesArchiver.__SITE_CONTENTS_JCR_NODE_NAME)
298                ? site.getChild(SitesArchiver.__SITE_CONTENTS_JCR_NODE_NAME)
299                : site.createChild(SitesArchiver.__SITE_CONTENTS_JCR_NODE_NAME, "ametys:collection");
300        String baseImportContentPath = sitePath.resolve("contents")
301                .toString();
302        if (ZipEntryHelper.zipEntryFolderExists(_zipArchivePath, baseImportContentPath))
303        {
304            Collection<ContentFiller> contentFillers = Collections.singleton(content -> _setWebAttributes(content, siteName));
305            return _siteArchiver._contentsArchiverHelper.importContents(baseImportContentPath + "/", rootContents, _zipArchivePath, _merger, contentFillers);
306        }
307        else
308        {
309            _logger.info("No content to be imported for Site '{}', the path '{}!{}' does not exist", site, _zipArchivePath, baseImportContentPath);
310            return new ImportReport();
311        }
312    }
313    
314    private void _setWebAttributes(DefaultContent content, String siteName)
315    {
316        if (content instanceof ModifiableSiteAwareAmetysObject)
317        {
318            _logger.debug("Setting site name '{}' to freshly imported content {}", siteName, content);
319            ((ModifiableSiteAwareAmetysObject) content).setSiteName(siteName);
320        }
321    }
322    
323    private ImportReport _importPlugins(Site site, Path sitePath) throws IOException, RepositoryException
324    {
325        String baseImportPluginPath = sitePath.resolve("plugins")
326                .toString();
327        if (!ZipEntryHelper.zipEntryFolderExists(_zipArchivePath, baseImportPluginPath))
328        {
329            _logger.info("No plugin to be imported for Site '{}', the path '{}!{}' does not exist", site, _zipArchivePath, baseImportPluginPath);
330            return new ImportReport();
331        }
332        
333        List<ImportReport> reports = new ArrayList<>();
334        
335        DirectoryStream<Path> childPlugins = ZipEntryHelper.children(
336                _zipArchivePath, 
337                Optional.of(baseImportPluginPath), 
338                Files::isDirectory);
339        
340        try (childPlugins)
341        {
342            for (Path childPlugin : childPlugins)
343            {
344                ImportReport importPluginReport = _importPlugin(site, childPlugin);
345                reports.add(importPluginReport);
346            }
347        }
348        
349        return ImportReport.union(reports);
350    }
351    
352    private ImportReport _importPlugin(Site site, Path zipPluginEntryPath) throws IOException, RepositoryException
353    {
354        String pluginName = zipPluginEntryPath.getFileName().toString();
355        SitePluginArchiver sitePluginArchiver = _siteArchiver._retrieveSitePluginArchiver(pluginName);
356        Node allPluginsNode = SitesArchiver._getAllPluginsNode(site);
357        _logger.info("Importing site plugin '{}' ({}!{}) from archive with archiver '{}'", pluginName, _zipArchivePath, zipPluginEntryPath, sitePluginArchiver.getClass().getName());
358        String zipPluginEntryPathStr = zipPluginEntryPath.toString();
359        return sitePluginArchiver.partialImport(site, pluginName, allPluginsNode, _zipArchivePath, zipPluginEntryPathStr, _merger);
360    }
361    
362    private ImportReport _importSitemaps(Site site, Path sitePath) throws RepositoryException, IOException
363    {
364        String baseImportSitemapPath = sitePath.resolve("pages")
365                .toString();
366        if (!ZipEntryHelper.zipEntryFolderExists(_zipArchivePath, baseImportSitemapPath))
367        {
368            _logger.info("No sitemap to be imported for Site '{}', the path '{}!{}' does not exist", site, _zipArchivePath, baseImportSitemapPath);
369            return new ImportReport();
370        }
371        
372        List<ImportReport> reports = new ArrayList<>();
373        
374        DirectoryStream<Path> sitemaps = ZipEntryHelper.children(
375                _zipArchivePath, 
376                Optional.of(baseImportSitemapPath), 
377                Files::isDirectory);
378        
379        try (sitemaps)
380        {
381            for (Path sitemap : sitemaps)
382            {
383                var sitemapImporter = new SitemapImporter(_siteArchiver, site, sitemap, _zipArchivePath, _merger, _logger, _builder);
384                sitemapImporter.importSitemap();
385                reports.add(sitemapImporter._report);
386            }
387        }
388        
389        return ImportReport.union(reports);
390    }
391}