001/*
002 *  Copyright 2015 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.web.frontoffice;
017
018import java.io.IOException;
019import java.io.InputStream;
020import java.io.UnsupportedEncodingException;
021import java.net.URLEncoder;
022import java.util.ArrayList;
023import java.util.Arrays;
024import java.util.Collection;
025import java.util.Date;
026import java.util.Enumeration;
027import java.util.HashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.Objects;
031import java.util.Set;
032import java.util.regex.Pattern;
033import java.util.stream.Collectors;
034
035import org.apache.avalon.framework.context.Context;
036import org.apache.avalon.framework.context.ContextException;
037import org.apache.avalon.framework.context.Contextualizable;
038import org.apache.avalon.framework.service.ServiceException;
039import org.apache.avalon.framework.service.ServiceManager;
040import org.apache.cocoon.ProcessingException;
041import org.apache.cocoon.environment.ObjectModelHelper;
042import org.apache.cocoon.environment.Request;
043import org.apache.cocoon.generation.ServiceableGenerator;
044import org.apache.cocoon.xml.AttributesImpl;
045import org.apache.cocoon.xml.XMLUtils;
046import org.apache.commons.lang.StringUtils;
047import org.apache.excalibur.xml.sax.SAXParser;
048import org.apache.solr.client.solrj.util.ClientUtils;
049import org.apache.tika.Tika;
050import org.xml.sax.InputSource;
051import org.xml.sax.SAXException;
052
053import org.ametys.cms.content.ContentHelper;
054import org.ametys.cms.content.RichTextHandler;
055import org.ametys.cms.contenttype.ContentType;
056import org.ametys.cms.contenttype.ContentTypeExtensionPoint;
057import org.ametys.cms.contenttype.ContentTypesHelper;
058import org.ametys.cms.contenttype.MetadataDefinition;
059import org.ametys.cms.contenttype.MetadataManager;
060import org.ametys.cms.contenttype.MetadataSet;
061import org.ametys.cms.contenttype.MetadataType;
062import org.ametys.cms.repository.Content;
063import org.ametys.cms.repository.RequestAttributeWorkspaceSelector;
064import org.ametys.cms.search.SearchResults;
065import org.ametys.cms.search.Sort;
066import org.ametys.cms.search.query.DocumentTypeQuery;
067import org.ametys.cms.search.query.OrQuery;
068import org.ametys.cms.search.query.Query;
069import org.ametys.cms.search.solr.SearcherFactory;
070import org.ametys.cms.search.solr.SearcherFactory.Searcher;
071import org.ametys.cms.tag.TagProviderExtensionPoint;
072import org.ametys.cms.transformation.URIResolverExtensionPoint;
073import org.ametys.core.util.LambdaUtils;
074import org.ametys.plugins.explorer.dublincore.DublinCoreMetadataProvider;
075import org.ametys.plugins.explorer.resources.Resource;
076import org.ametys.plugins.explorer.resources.metadata.TikaProvider;
077import org.ametys.plugins.repository.AmetysObject;
078import org.ametys.plugins.repository.AmetysObjectResolver;
079import org.ametys.plugins.repository.AmetysRepositoryException;
080import org.ametys.plugins.repository.metadata.RichText;
081import org.ametys.runtime.i18n.I18nizableText;
082import org.ametys.runtime.parameter.ParameterHelper;
083import org.ametys.web.indexing.solr.SolrWebFieldNames;
084import org.ametys.web.repository.page.Page;
085import org.ametys.web.repository.page.Page.PageType;
086import org.ametys.web.repository.page.Zone;
087import org.ametys.web.repository.page.ZoneItem;
088import org.ametys.web.repository.page.ZoneItem.ZoneType;
089import org.ametys.web.repository.site.Site;
090import org.ametys.web.repository.site.SiteManager;
091
092/**
093 * Abstract class for solr search
094 */
095public abstract class AbstractSearchGenerator extends ServiceableGenerator implements Contextualizable, SolrWebFieldNames
096{
097    /** Textfield pattern */
098    protected static final Pattern _TEXTFIELD_PATTERN = Pattern.compile("^[^?*].*$");
099    
100    /** The {@link ContentType} manager */
101    protected ContentTypeExtensionPoint _cTypeExtPt;
102    /** The sites manager */
103    protected SiteManager _siteManager;
104    /** The cocoon context */
105    protected org.apache.cocoon.environment.Context _context;
106    /** The tag extension point */
107    protected TagProviderExtensionPoint _tagExtPt;
108    /** The Ametys resolver */
109    protected AmetysObjectResolver _resolver;
110    /** DublinCore provider */
111    protected DublinCoreMetadataProvider _dcProvider;
112    /** The helper to handler content types */
113    protected ContentTypesHelper _contentTypesHelper;
114    /** The helper to handler contents */
115    protected ContentHelper _contentHelper;
116    /** The uri resolver extension point */
117    protected URIResolverExtensionPoint _uriResolverEP;
118    /** Metadata manager. */
119    protected MetadataManager _metadataManager;
120    /** The sax parser */
121    protected SAXParser _parser;
122    /** The searcher factory */
123    protected SearcherFactory _searcherFactory;
124    
125    @Override
126    public void service(ServiceManager smanager) throws ServiceException
127    {
128        super.service(smanager);
129        _cTypeExtPt = (ContentTypeExtensionPoint) smanager.lookup(ContentTypeExtensionPoint.ROLE);
130        _siteManager = (SiteManager) smanager.lookup(SiteManager.ROLE);
131        _tagExtPt = (TagProviderExtensionPoint) smanager.lookup(TagProviderExtensionPoint.ROLE);
132        _resolver = (AmetysObjectResolver) smanager.lookup(AmetysObjectResolver.ROLE);
133        _dcProvider = (DublinCoreMetadataProvider) smanager.lookup(DublinCoreMetadataProvider.ROLE);
134        _contentTypesHelper = (ContentTypesHelper) smanager.lookup(ContentTypesHelper.ROLE);
135        _contentHelper = (ContentHelper) smanager.lookup(ContentHelper.ROLE);
136        _parser = (SAXParser) manager.lookup(SAXParser.ROLE);
137        _uriResolverEP = (URIResolverExtensionPoint) smanager.lookup(URIResolverExtensionPoint.ROLE);
138        _metadataManager = (MetadataManager) smanager.lookup(MetadataManager.ROLE);
139        _searcherFactory = (SearcherFactory) smanager.lookup(SearcherFactory.ROLE);
140    }
141
142    @Override
143    public void contextualize(Context context) throws ContextException
144    {
145        _context = (org.apache.cocoon.environment.Context) context.get(org.apache.cocoon.Constants.CONTEXT_ENVIRONMENT_CONTEXT);
146    }
147
148    @Override
149    public void generate() throws IOException, SAXException, ProcessingException
150    {
151        Request request = ObjectModelHelper.getRequest(objectModel);
152        
153        String currentSiteName = null;
154        String lang = null;
155        Page page = (Page) request.getAttribute(Page.class.getName());
156        if (page != null)
157        {
158            currentSiteName = page.getSiteName();
159            lang = page.getSitemapName();
160        }
161        else
162        {
163            currentSiteName = parameters.getParameter("siteName", null);
164            lang = parameters.getParameter("lang", null);
165        }
166
167        ZoneItem zoneItem = (ZoneItem) request.getAttribute(ZoneItem.class.getName());
168        if (zoneItem == null)
169        {
170            String zoneItemId = request.getParameter("zone-item-id");
171            zoneItem = _resolver.resolveById(zoneItemId);
172        }
173
174        int pageIndex = getPageIndex(request);
175        // TODO Rename to maxResults
176        int maxResults = parameters.getParameterAsInteger("offset", 10);
177        int start = (pageIndex - 1) * maxResults;
178
179        String[] sites = request.getParameterValues("sites");
180        List<String> siteNames = new ArrayList<>();
181        if (sites != null && sites.length > 0 && !(sites.length == 1 && sites[0].equals("")))
182        {
183            for (String site : sites)
184            {
185                siteNames.add(site);
186            }
187        }
188        else
189        {
190            siteNames.add(currentSiteName);
191        }
192        
193        contentHandler.startDocument();
194
195        AttributesImpl attrs = new AttributesImpl();
196        attrs.addCDATAAttribute("site", currentSiteName);
197        attrs.addCDATAAttribute("lang", lang);
198
199        XMLUtils.startElement(contentHandler, "search", attrs);
200
201        saxServiceIdentifiers();
202        saxAdditionalInfos();
203
204        // The search url
205        XMLUtils.createElement(contentHandler, "url", page != null ? lang + "/" + page.getPathInSitemap() + ".html" : lang + "/_plugins/" + currentSiteName + "/" + lang + "/service/search.html");
206
207        // Display the form and results on same page?
208        String searchMode = getSearchMode();
209        XMLUtils.createElement(contentHandler, "search-mode", searchMode);
210
211        try
212        {
213            SearchResults<AmetysObject> searchResults = null;
214            boolean submit = request.getParameter("submit-form") != null;
215            if (submit && isInputValid())
216            {
217                searchResults = search(request, siteNames, lang, pageIndex, start, maxResults);
218            }
219            else
220            {
221                searchResults = search(request, siteNames, lang, pageIndex, start, maxResults, false);
222            }
223            
224            saxFormParameters(request, searchResults, start, maxResults, currentSiteName, lang);
225        }
226        catch (IllegalArgumentException e)
227        {
228            getLogger().error("The search field is invalid", e);
229            XMLUtils.createElement(contentHandler, "illegal-textfield");
230            saxPagination(0, start, maxResults);
231        }
232        catch (Exception e)
233        {
234            getLogger().error("Unable to search", e);
235            saxPagination(0, start, maxResults);
236        }
237        
238        XMLUtils.endElement(contentHandler, "search");
239        contentHandler.endDocument();
240    }
241
242    /**
243     * Search
244     * @param request the request
245     * @param siteNames The name of the sites to search in 
246     * @param language The language code to search
247     * @param pageIndex the page index
248     * @param start The offset for search results
249     * @param maxResults The maximum number of results
250     * @return The search results
251     * @throws Exception If an error occurred during search
252     */
253    protected SearchResults<AmetysObject> search(Request request, Collection<String> siteNames, String language, int pageIndex, int start, int maxResults) throws Exception
254    {
255        return search(request, siteNames, language, pageIndex, start, maxResults, true);
256    }
257    
258    /**
259     * Search
260     * @param request the request
261     * @param siteNames The name of the sites to search in 
262     * @param language The language code to search
263     * @param pageIndex the page index
264     * @param start The offset for search results
265     * @param maxResults The maximum number of results
266     * @param saxResults false to not sax results
267     * @return The search results
268     * @throws Exception If an error occurred during search
269     */
270    protected SearchResults<AmetysObject> search(Request request, Collection<String> siteNames, String language, int pageIndex, int start, int maxResults, boolean saxResults) throws Exception
271    {
272        // Retrieve current workspace
273        String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request);
274        
275        SearchResults<AmetysObject> results = null;
276        
277        try
278        {
279            // Query
280            Query queryObject = getQuery(request, siteNames, language);
281            
282            // Filter queries
283            Collection<Query> filterQueries = getFilterQueries(request, siteNames, language);
284            // Document types query
285            Collection<String> documentTypes = getDocumentTypes(request);
286            Query documentTypesQuery = getDocumentTypesQuery(documentTypes);
287            
288            // Get first sort field
289            Sort sort = getSortField(request);
290            XMLUtils.createElement(contentHandler, sort == null || sort.getField() == null ? "sort-by-score" : "sort-by-" + sort.getField());
291            
292            Searcher searcher = _searcherFactory.create()
293                    .withQuery(queryObject)
294                    .withFilterQueries(filterQueries)
295                    .addFilterQuery(documentTypesQuery)
296                    .withLimits(0, Integer.MAX_VALUE)
297                    .withSort(getPrimarySortFields(request))
298                    .addSort(sort)
299                    .setCheckRights(true);
300            
301            _additionalSearchProcessing(searcher);
302            
303            results = searcher.searchWithFacets();
304            
305            if (saxResults)
306            {
307                // SAX results
308                AttributesImpl atts = new AttributesImpl();
309                atts.addCDATAAttribute("total", String.valueOf(results.getResults().getSize()));
310                XMLUtils.startElement(contentHandler, "hits", atts);
311                saxHits(results, start, maxResults);
312                XMLUtils.endElement(contentHandler, "hits");
313        
314                // SAX pagination
315                saxPagination(results.getResults().getSize(), start, maxResults);
316            }
317        }
318        finally
319        {
320            // Restore context
321            RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp);
322        }
323        
324        return results;
325    }
326    
327    /**
328     * Allow to perform other searcher configuration
329     * @param searcher The searcher that will be used for the search operation
330     */
331    protected void _additionalSearchProcessing(Searcher searcher)
332    {
333        // Nothing by default
334    }
335    
336    /**
337     * Build the search query to have results matching at least one of the given document types
338     * @param documentTypes The document types
339     * @return The document type query
340     */
341    protected Query getDocumentTypesQuery(Collection<String> documentTypes)
342    {
343        List<Query> queries = documentTypes.stream()
344                .map(t -> "\"" + ClientUtils.escapeQueryChars(t) + "\"")
345                .map(DocumentTypeQuery::new)
346                .collect(Collectors.toList());
347        return new OrQuery(queries);
348    }
349    
350    /**
351     * Get the document types.
352     * @param request The request.
353     * @return the document types to search.
354     */
355    protected abstract Collection<String> getDocumentTypes(Request request);
356    
357    /**
358     * Get the sort field
359     * @param request The request
360     * @return The sort field or null to sort by score
361     */
362    protected abstract Sort getSortField(Request request);
363    
364    /**
365     * Get the primary sort fields
366     * @param request The request
367     * @return the list additional sort fields or empty list.
368    */
369    protected abstract List<Sort> getPrimarySortFields(Request request);
370    
371    /**
372     * SAX the search parameters from the request parameters
373     * @param request The request
374     * @param searchResults The search results
375     * @param start The start index
376     * @param offset The number of results
377     * @param siteName The current site name
378     * @param lang The current language
379     * @throws SAXException If an error occurs while SAXing
380     */
381    protected void saxFormParameters (Request request, SearchResults<AmetysObject> searchResults, int start, int offset, String siteName, String lang) throws SAXException
382    {
383        Map<String, Criterion> criteria = getCriteria(request, siteName, lang);
384
385        XMLUtils.startElement(contentHandler, "form");
386
387        XMLUtils.startElement(contentHandler, "fields");
388        saxFormFields(request, searchResults, siteName, lang);
389        saxCriteria(criteria, request, searchResults, siteName, lang, true);
390        XMLUtils.endElement(contentHandler, "fields");
391
392        boolean submit = request.getParameter("submit-form") != null;
393        if (submit)
394        {
395            if (isInputValid())
396            {
397                XMLUtils.startElement(contentHandler, "values");
398
399                XMLUtils.createElement(contentHandler, "start", String.valueOf(start));
400                XMLUtils.createElement(contentHandler, "offset", String.valueOf(offset));
401
402                saxFormValues(request, searchResults, start, offset);
403
404                XMLUtils.endElement(contentHandler, "values");
405            }
406        }
407        XMLUtils.endElement(contentHandler, "form");
408    }
409
410    /**
411     * SAX the form search criteria
412     * @param request The request
413     * @param searchResults The search results
414     * @param siteName The current site name
415     * @param lang The current language
416     * @throws SAXException if an error occurs while SAXing
417     */
418    protected abstract void saxFormFields(Request request, SearchResults<AmetysObject> searchResults, String siteName, String lang) throws SAXException;
419
420    /**
421     * SAX criteria.
422     * @param criteria the criteria list.
423     * @param request the request
424     * @param searchResults The search results
425     * @param siteName the name of the site
426     * @param lang the language
427     * @param withHitCount true to sax the hit count as well
428     * @throws SAXException If an error occurred while saxing the criteria
429     */
430    protected void saxCriteria(Map<String, Criterion> criteria, Request request, SearchResults<AmetysObject> searchResults, String siteName, String lang, boolean withHitCount) throws SAXException
431    {
432        try
433        {
434            for (String criterionName : criteria.keySet())
435            {
436                Criterion criterion = criteria.get(criterionName);
437
438                saxCriterion(criterion, searchResults, siteName, lang);
439            }
440        }
441        catch (IOException e)
442        {
443            getLogger().error("Error generating the criteria.", e);
444            throw new SAXException("Error generating the criteria.", e);
445        }
446    }
447
448    /**
449     * SAX a single criterion.
450     * @param criterion the criterion.
451     * @param searchResults The search results
452     * @param siteName the site name.
453     * @param lang the language.
454     * @throws SAXException If a sax error occurred
455     * @throws IOException If another error occurred
456     */
457    protected void saxCriterion(Criterion criterion, SearchResults<AmetysObject> searchResults, String siteName, String lang) throws SAXException, IOException
458    {
459        AttributesImpl criterionAttrs = new AttributesImpl();
460        
461        // TODO Facet count.
462//        if (criteriaValuesBitSet != null)
463//        {
464//            int criterionCount = criteriaValuesBitSet.cardinality();
465//            criterionAttrs.addCDATAAttribute("count", Integer.toString(criterionCount));
466//        }
467        
468        XMLUtils.startElement(contentHandler, criterion.getXmlGroupName(), criterionAttrs);
469        
470        for (String value : criterion.getValues().keySet())
471        {
472            I18nizableText label = criterion.getValues().get(value);
473            
474            // TODO Replace with facets
475//            int count = getCount(criteriaValuesBitSet, criterion, value, siteName, lang);
476            int count = 0;
477            
478            AttributesImpl valueAttrs = new AttributesImpl();
479            valueAttrs.addCDATAAttribute("value", value);
480            valueAttrs.addCDATAAttribute("count", Integer.toString(count));
481            
482            XMLUtils.startElement(contentHandler, criterion.getXmlElementName(), valueAttrs);
483            label.toSAX(contentHandler);
484            XMLUtils.endElement(contentHandler, criterion.getXmlElementName());
485        }
486
487        XMLUtils.endElement(contentHandler, criterion.getXmlGroupName());
488    }
489
490    /**
491     * SAX the form search criteria values
492     * @param request The request
493     * @param searchResults The search results
494     * @param start The start index
495     * @param offset The number of results
496     * @throws SAXException if an error occurs while SAXing
497     */
498    protected abstract void saxFormValues (Request request, SearchResults<AmetysObject> searchResults, int start, int offset) throws SAXException;
499
500    /**
501     * Get the query from request parameters
502     * @param request The request
503     * @param siteNames The site names.
504     * @param language The language
505     * @return The query object.
506     * @throws IllegalArgumentException If the search field is invalid.
507     */
508    protected abstract Query getQuery(Request request, Collection<String> siteNames, String language) throws IllegalArgumentException;
509    
510    /**
511     * Get the filter queries from the request parameters.
512     * @param request The request.
513     * @param siteNames The site names.
514     * @param language The language.
515     * @return A collection of filter queries.
516     * @throws IllegalArgumentException If a search field is invalid.
517     */
518    protected abstract Collection<Query> getFilterQueries(Request request, Collection<String> siteNames, String language) throws IllegalArgumentException;
519
520    /**
521     * SAX the result hits
522     * @param results The search results.
523     * @param start The start index
524     * @param maxResults The number of results to generate.
525     * @throws SAXException If an error occurs while SAXing
526     * @throws IOException If there is a low-level IO error
527     */
528    protected abstract void saxHits(SearchResults<AmetysObject> results, int start, int maxResults) throws SAXException, IOException;
529
530    /**
531     * Get the searchable fields
532     * @return The fields
533     */
534    protected abstract Collection<String> getFields();
535
536    /**
537     * Get the criteria with their value list.
538     * @param request the request.
539     * @param siteName the site name.
540     * @param lang the language
541     * @return the criteria list.
542     */
543    protected abstract Map<String, Criterion> getCriteria(Request request, String siteName, String lang);
544
545    /**
546     * Get the criteria user values.
547     * @param criteria the criteria list.
548     * @param request the request object.
549     * @return the criteria user values.
550     */
551    protected Map<String, String> getCriteriaValues(Map<String, Criterion> criteria, Request request)
552    {
553        Map<String, String> values = new HashMap<>();
554
555        for (String name : criteria.keySet())
556        {
557            Criterion criterion = criteria.get(name);
558            String value = request.getParameter(criterion.getFieldName());
559
560            if (StringUtils.isNotEmpty(value))
561            {
562                values.put(criterion.getLuceneTermName(), value);
563            }
564        }
565
566        return values;
567    }
568    
569    /**
570     * SAX a hit of type page.
571     * @param score The score of the page
572     * @param maxScore The maximum score of the search results
573     * @param page The page
574     * @throws SAXException If an error occurs while SAXing
575     */
576    protected void saxPageHit(float score, float maxScore, Page page) throws SAXException
577    {
578        int percent = Math.min(Math.round(score * 100f / maxScore), 100);
579        
580        XMLUtils.startElement(contentHandler, "hit");
581        XMLUtils.createElement(contentHandler, "score", Float.toString(score));
582        XMLUtils.createElement(contentHandler, "percent", Integer.toString(percent));
583        XMLUtils.createElement(contentHandler, "title", page.getTitle());
584        
585        _saxPageContents(page);
586        
587        XMLUtils.createElement(contentHandler, "type", "page");
588        XMLUtils.createElement(contentHandler, "uri", page.getSitemap().getName() + "/" + page.getPathInSitemap());
589        
590        _saxLastModifiedDate(page);
591        _saxLastValidationDate(page);
592        
593        String siteName = page.getSiteName();
594        if (siteName != null)
595        {
596            Site site = _siteManager.getSite(siteName);
597            XMLUtils.createElement(contentHandler, "siteName", siteName);
598            XMLUtils.createElement(contentHandler, "siteTitle", site.getTitle());
599            String url = site.getUrl();
600            if (url != null)
601            {
602                XMLUtils.createElement(contentHandler, "siteUrl", url);
603            }
604        }
605        
606        saxAdditionalInfosOnPageHit(page);
607        
608        XMLUtils.endElement(contentHandler, "hit");
609    }
610    
611    private void _saxPageContents(Page page) throws SAXException
612    {
613        if (page.getType() == PageType.CONTAINER)
614        {
615            for (Zone zone : page.getZones())
616            {
617                for (ZoneItem zoneItem : zone.getZoneItems())
618                {
619                    if (zoneItem.getType() == ZoneType.CONTENT)
620                    {
621                        // Content
622                        Content content = zoneItem.getContent();
623                        saxContent(content.getId(), "index");
624                    }
625                }
626            }
627        }
628    }
629    
630    private void _saxLastModifiedDate(Page page) throws SAXException
631    {
632        Date lastModified = null;
633
634        if (page.getType() == PageType.CONTAINER)
635        {
636            for (Zone zone : page.getZones())
637            {
638                for (ZoneItem zoneItem : zone.getZoneItems())
639                {
640                    switch (zoneItem.getType())
641                    {
642                        case SERVICE:
643                            // A service has no last modification date
644                            break;
645                        case CONTENT:
646                            Date contentLastModified = zoneItem.getContent().getLastModified();
647
648                            if (contentLastModified != null && (lastModified == null || contentLastModified.after(lastModified)))
649                            {
650                                // Keep the latest modification date
651                                lastModified = contentLastModified;
652                            }
653                            break;
654                        default:
655                            break;
656                    }
657                }
658            }
659        }
660        if (lastModified != null)
661        {
662            XMLUtils.createElement(contentHandler, "lastModified", ParameterHelper.valueToString(lastModified));
663        }
664    }
665    
666    private void _saxLastValidationDate(Page page) throws SAXException
667    {
668        Date lastValidated = null;
669        
670        if (page.getType() == PageType.CONTAINER)
671        {
672            for (Zone zone : page.getZones())
673            {
674                for (ZoneItem zoneItem : zone.getZoneItems())
675                {
676                    switch (zoneItem.getType())
677                    {
678                        case SERVICE:
679                            // A service has no last validation date
680                            break;
681                        case CONTENT:
682                            Date contentLastValidation = zoneItem.getContent().getLastValidationDate();
683                            
684                            if (contentLastValidation != null && (lastValidated == null || contentLastValidation.after(lastValidated)))
685                            {
686                                // Keep the latest validation date
687                                lastValidated = contentLastValidation;
688                            }
689                            break;
690                        default:
691                            break;
692                    }
693                }
694            }
695        }
696        if (lastValidated != null)
697        {
698            XMLUtils.createElement(contentHandler, "lastValidation", ParameterHelper.valueToString(lastValidated));
699        }
700    }
701    
702    /**
703     * SAX additional information on page hit
704     * @param page the page
705     * @throws SAXException if something goes wrong when saxing the information
706     */
707    protected void saxAdditionalInfosOnPageHit(Page page) throws SAXException
708    {
709        // Nothing to do here.
710    }
711    
712    /**
713     * SAX a hit of type "resource".
714     * @param score The score of the page
715     * @param maxScore The maximum score of the search results
716     * @param resource The resource
717     * @throws SAXException If an error occurs while SAXing
718     */
719    protected void saxResourceHit(float score, float maxScore, Resource resource) throws SAXException
720    {
721        int percent = Math.min(Math.round(score * 100f / maxScore), 100);
722        
723        String filename = resource.getName();
724        
725        XMLUtils.startElement(contentHandler, "hit");
726        XMLUtils.createElement(contentHandler, "score", Float.toString(score));
727        XMLUtils.createElement(contentHandler, "percent", Integer.toString(percent));
728        XMLUtils.createElement(contentHandler, "filename", filename);
729        XMLUtils.createElement(contentHandler, "title", StringUtils.substringBeforeLast(resource.getName(), "."));
730        
731        String dcDescription = resource.getDCDescription();
732        String excerpt = _getResourceExcerpt(resource);
733        if (StringUtils.isNotBlank(dcDescription))
734        {
735            XMLUtils.createElement(contentHandler, "excerpt", dcDescription);
736        }
737        else if (StringUtils.isNotBlank(excerpt))
738        {
739            XMLUtils.createElement(contentHandler, "excerpt", excerpt + "...");
740        }
741        
742        XMLUtils.createElement(contentHandler, "type", "resource");
743        
744        Page page = _getResourcePage(resource);
745        if (page != null)
746        {
747            String pageUri = page.getSitemapName() + "/" + page.getPathInSitemap();
748            String encodedPath = _encodePath(_encodePath(resource.getResourcePath().substring(1)));
749            XMLUtils.createElement(contentHandler, "uri", pageUri + "/_attachments/" + encodedPath + "?download=true");
750        }
751        XMLUtils.createElement(contentHandler, "mime-types", resource.getMimeType());
752        _saxSize(resource.getLength());
753        _saxIcon(filename);
754        
755        Date lastModified = resource.getLastModified();
756        if (lastModified != null)
757        {
758            XMLUtils.createElement(contentHandler, "lastModified", ParameterHelper.valueToString(lastModified));
759        }
760        if (page != null)
761        {
762            Site site = page.getSite();
763            XMLUtils.createElement(contentHandler, "siteName", site.getName());
764            XMLUtils.createElement(contentHandler, "siteTitle", site.getTitle());
765            XMLUtils.createElement(contentHandler, "siteUrl", site.getUrl());
766        }
767        
768        XMLUtils.endElement(contentHandler, "hit");
769    }
770    
771    private String _getResourceExcerpt(Resource resource)
772    {
773        try (InputStream is = resource.getInputStream())
774        {
775            TikaProvider tikaProvider = (TikaProvider) manager.lookup(TikaProvider.ROLE);
776            Tika tika = tikaProvider.getTika();
777            String value = tika.parseToString(is);
778            if (StringUtils.isNotBlank(value))
779            {
780                int summaryEndIndex = value.lastIndexOf(' ', 200);
781                if (summaryEndIndex == -1)
782                {
783                    summaryEndIndex = value.length();
784                }
785                return value.substring(0, summaryEndIndex) + (summaryEndIndex != value.length() ? "…" : "");
786            }
787        }
788        catch (Exception e)
789        {
790            getLogger().error("Unable to index resource at " + resource.getPath(), e);
791        }
792        return null;
793    }
794    
795    private Page _getResourcePage(Resource resource)
796    {
797        if (resource != null)
798        {
799            AmetysObject parent = resource.getParent();
800            while (parent != null)
801            {
802                if (parent instanceof Page)
803                {
804                    // We have gone up to the page
805                    return (Page) parent;
806                }
807                parent = parent.getParent();
808            }
809        }
810        
811        return null;
812    }
813    
814    private String _encodePath(String path)
815    {
816        StringBuilder sb = new StringBuilder();
817        
818        String[] parts = StringUtils.split(path, '/');
819        for (int i = 0; i < parts.length; i++)
820        {
821            if (i > 0 || path.charAt(0) == '/')
822            {
823                sb.append("/");
824            }
825            try
826            {
827                sb.append(URLEncoder.encode(parts[i], "UTF-8"));
828            }
829            catch (UnsupportedEncodingException e)
830            {
831                throw new IllegalStateException("Cannot encode path", e);
832            }
833        }
834        return sb.toString();
835    }
836    
837    /**
838     * SAX elements for pagination
839     * @param totalHits The total number of result
840     * @param start The start index of search
841     * @param offset The max number of results per page
842     * @throws SAXException SAXException If an error occurs while SAXing
843     */
844    protected void saxPagination(long totalHits, int start, int offset) throws SAXException
845    {
846        int nbPages = (int) Math.ceil((double) totalHits / (double) offset);
847        
848        AttributesImpl atts = new AttributesImpl();
849        atts.addCDATAAttribute("total", String.valueOf(nbPages)); // Number of pages
850        atts.addCDATAAttribute("start", String.valueOf(start)); // Index of the first hit
851        atts.addCDATAAttribute("end", start + offset > totalHits ? String.valueOf(totalHits) : String.valueOf(start + offset)); // Index of the last hit
852        
853        XMLUtils.startElement(contentHandler, "pagination", atts);
854        
855        for (int i = 0; i < nbPages; i++)
856        {
857            AttributesImpl attr = new AttributesImpl();
858            attr.addAttribute("", "index", "index", "CDATA", String.valueOf(i + 1));
859            attr.addAttribute("", "start", "start", "CDATA", String.valueOf(i * offset));
860            XMLUtils.createElement(contentHandler, "page", attr);
861        }
862        
863        XMLUtils.endElement(contentHandler, "pagination");
864    }
865    
866    /**
867     * Get the page index
868     * @param request The request
869     * @return The page index
870     */
871    protected int getPageIndex(Request request)
872    {
873        Enumeration paramNames = request.getParameterNames();
874        while (paramNames.hasMoreElements())
875        {
876            String param = (String) paramNames.nextElement();
877            if (param.startsWith("page-"))
878            {
879                return Integer.parseInt(param.substring("page-".length()));
880            }
881        }
882        return 1;
883    }
884    
885    private void _saxIcon(String filename) throws SAXException
886    {
887        int index = filename.lastIndexOf('.');
888        String extension = filename.substring(index + 1);
889
890        XMLUtils.createElement(contentHandler, "icon", "plugins/explorer/icon-medium/" + extension + ".png");
891    }
892    
893    private void _saxSize(long size) throws SAXException
894    {
895        XMLUtils.startElement(contentHandler, "size");
896        if (size < 1024)
897        {
898            // Bytes
899            List<String> params = new ArrayList<>();
900            params.add(String.valueOf(size));
901            I18nizableText i18nSize = new I18nizableText("plugin.web", "ATTACHMENTS_FILE_SIZE_BYTES", params);
902            i18nSize.toSAX(contentHandler);
903        }
904        else if (size < 1024 * 1024)
905        {
906            // Kb
907            int sizeInKb = Math.round(size * 10 / 1024 / 10);
908            List<String> params = new ArrayList<>();
909            params.add(String.valueOf(sizeInKb));
910            I18nizableText i18nSize = new I18nizableText("plugin.web", "ATTACHMENTS_FILE_SIZE_KB", params);
911            i18nSize.toSAX(contentHandler);
912
913        }
914        else
915        {
916            // Mb
917            int sizeInMb = Math.round(size * 10 / (1024 * 1024) / 10);
918            List<String> params = new ArrayList<>();
919            params.add(String.valueOf(sizeInMb));
920            I18nizableText i18nSize = new I18nizableText("plugin.web", "ATTACHMENTS_FILE_SIZE_MB", params);
921            i18nSize.toSAX(contentHandler);
922        }
923        XMLUtils.endElement(contentHandler, "size");
924    }
925    
926    /**
927     * Generate the service identifiers: service group ID, ZoneItem ID, ...
928     * @throws SAXException if an error occurs SAXing data.
929     * @throws IOException if an error occurs SAXing data.
930     * @throws ProcessingException if a processing error occurs.
931     */
932    protected void saxServiceIdentifiers() throws SAXException, IOException, ProcessingException
933    {
934        Request request = ObjectModelHelper.getRequest(objectModel);
935        ZoneItem zoneItem = (ZoneItem) request.getAttribute(ZoneItem.class.getName());
936        String serviceGroupId = parameters.getParameter("service-group-id", "");
937        
938        // The service group ID.
939        if (StringUtils.isNotEmpty(serviceGroupId))
940        {
941            XMLUtils.createElement(contentHandler, "group-id", serviceGroupId);
942        }
943        
944        // Generate the ZoneItem ID if it exists.
945        if (zoneItem != null)
946        {
947            AttributesImpl atts = new AttributesImpl();
948            atts.addCDATAAttribute("id", zoneItem.getId());
949            XMLUtils.createElement(contentHandler, "zone-item", atts);
950        }
951    }
952    
953    /**
954     * Generate any additional information.
955     * @throws SAXException if an error occurs SAXing data.
956     * @throws IOException if an error occurs SAXing data.
957     * @throws ProcessingException if a processing error occurs.
958     */
959    protected void saxAdditionalInfos() throws SAXException, IOException, ProcessingException
960    {
961        // Nothing to do here.
962    }
963    
964    /**
965     * Get the search mode.
966     * @return the search mode as a string.
967     */
968    protected String getSearchMode()
969    {
970        return parameters.getParameter("search-mode", "criteria-and-results");
971    }
972    
973    /**
974     * Check if the input is valid.
975     * @return true if the input is valid, false otherwise.
976     */
977    protected boolean isInputValid()
978    {
979        boolean valid = true;
980        
981        Request request = ObjectModelHelper.getRequest(objectModel);
982        ZoneItem zoneItem = (ZoneItem) request.getAttribute(ZoneItem.class.getName());
983        String serviceGroupId = parameters.getParameter("service-group-id", "");
984        String requestParamGroupId = StringUtils.defaultString(request.getParameter("submit-form"));
985        String requestParamZoneItemId = StringUtils.defaultString(request.getParameter("zone-item-id"));
986        
987        // If the generator is not used as part of a service (zoneItem is null), consider the input valid.
988        if (zoneItem != null)
989        {
990            // If the generator is used as part of a service and both "group ID"
991            // and "zone item ID" are missing from the input, consider the input valid.
992            if (StringUtils.isNotEmpty(requestParamGroupId) || StringUtils.isNotEmpty(requestParamZoneItemId))
993            {
994                if (StringUtils.isEmpty(serviceGroupId))
995                {
996                    // No specified group ID: check the provided ZoneItem ID.
997                    valid = requestParamZoneItemId.equals(zoneItem.getId());
998                }
999                else
1000                {
1001                    // Check the group ID against the one sent by the form.
1002                    valid = requestParamGroupId.equals(serviceGroupId);
1003                }
1004            }
1005        }
1006        return valid;
1007    }
1008    
1009    /**
1010     * SAX the metadata set of a content if exists
1011     * @param contentId the id of the content 
1012     * @param metadataSetName The name of metadata set to sax
1013     * @throws SAXException if an exception occurs while saxing
1014     */
1015    protected void saxContent(String contentId, String metadataSetName) throws SAXException
1016    {   
1017        try
1018        {
1019            Content content = _resolver.resolveById(contentId);
1020            String[] contentTypes = content.getTypes();
1021            
1022            // content-name
1023            XMLUtils.createElement(contentHandler, "content-name", content.getName());
1024            
1025            // content-types
1026            XMLUtils.startElement(contentHandler, "content-types");
1027            Arrays.asList(contentTypes).forEach(LambdaUtils.wrapConsumer(cType -> XMLUtils.createElement(contentHandler, "content-type", cType)));
1028            XMLUtils.endElement(contentHandler, "content-types");
1029            
1030            MetadataSet metadataSet = _contentTypesHelper.getMetadataSetForView(metadataSetName, contentTypes, content.getMixinTypes());
1031            if (metadataSet != null)
1032            {
1033                Map<String, MetadataDefinition> metadataDefinitions = _contentTypesHelper.getMetadataDefinitions(metadataSet, content);
1034                Set<String> richTextMetadataPaths = metadataDefinitions.entrySet().parallelStream()
1035                                                                                  .filter(Objects::nonNull)
1036                                                                                  .filter(e -> e.getValue() != null && e.getValue().getType() == MetadataType.RICH_TEXT)
1037                                                                                  .map(Map.Entry::getKey)
1038                                                                                  .collect(Collectors.toSet());
1039                Map<String, Object> richTextValues = _contentHelper.getMetadataValues(content, richTextMetadataPaths);
1040                for (String key : richTextValues.keySet())
1041                {
1042                    @SuppressWarnings("unchecked")
1043                    List<RichText> richTextValueArray = (List<RichText>) richTextValues.get(key);
1044                    for (RichText richTextValue : richTextValueArray)
1045                    {
1046                        try (InputStream is = richTextValue.getInputStream())
1047                        {
1048                            RichTextHandler txtHandler = new RichTextHandler(200);
1049                            _parser.parse(new InputSource(is), txtHandler);
1050                            String textValue = txtHandler.getValue();
1051                            if (textValue != null)
1052                            {
1053                                XMLUtils.createElement(contentHandler, "excerpt", textValue);
1054                            }
1055                        }
1056                        catch (Exception e)
1057                        {
1058                            getLogger().error("Cannot convert a richtextvalue at path '" + key + "' of content '" + contentId + "'", e);
1059                        }
1060                    }
1061                }
1062                
1063                AttributesImpl attrs = new AttributesImpl();
1064                attrs.addCDATAAttribute("id", content.getId());
1065                attrs.addCDATAAttribute("name", content.getName());
1066                attrs.addCDATAAttribute("language", content.getLanguage());
1067                XMLUtils.startElement(contentHandler, "content", attrs);
1068                
1069                _metadataManager.saxMetadata(contentHandler, content, metadataSet);
1070
1071                XMLUtils.endElement(contentHandler, "content");
1072            }
1073        }
1074        catch (AmetysRepositoryException | IOException e)
1075        {
1076            getLogger().error("Cannot sax information about the content " + contentId, e);
1077        }
1078    }
1079
1080    /**
1081     * Criterion class.
1082     */
1083    protected class Criterion
1084    {
1085        /** Form field name. */
1086        protected String _fieldName;
1087
1088        /** Lucene field name. */
1089        protected String _luceneTermName;
1090
1091        /** XML element name. */
1092        protected String _xmlElementName;
1093
1094        /** XML group name. */
1095        protected String _xmlGroupName;
1096
1097        /** Value list. */
1098        protected Map<String, I18nizableText> _values;
1099
1100        /**
1101         * Build a new form field.
1102         */
1103        public Criterion()
1104        {
1105            this("", "", new HashMap<String, I18nizableText>());
1106        }
1107
1108        /**
1109         * Build a new form field.
1110         * @param fieldName the field's name
1111         * @param xmlGroupName the name of the xml group
1112         * @param values the values' list
1113         */
1114        public Criterion(String fieldName, String xmlGroupName, Map<String, I18nizableText> values)
1115        {
1116            this(fieldName, fieldName, xmlGroupName, values);
1117        }
1118
1119        /**
1120         * Build a new form field.
1121         * @param fieldName the field's name
1122         * @param luceneTermName the name of the lucene term 
1123         * @param xmlGroupName the name of the xml group
1124         * @param values the values' list
1125         */
1126        public Criterion(String fieldName, String luceneTermName, String xmlGroupName, Map<String, I18nizableText> values)
1127        {
1128            this(fieldName, luceneTermName, fieldName, xmlGroupName, values);
1129        }
1130
1131        /**
1132         * Build a new form field.
1133         * @param fieldName the name of the field
1134         * @param luceneTermName the name of the lucene term
1135         * @param xmlElementName the name of the xml element
1136         * @param xmlGroupName the name of the xml group
1137         * @param values the values' list
1138         */
1139        public Criterion(String fieldName, String luceneTermName, String xmlElementName, String xmlGroupName, Map<String, I18nizableText> values)
1140        {
1141            this._fieldName = fieldName;
1142            this._luceneTermName = luceneTermName;
1143            this._xmlElementName = xmlElementName;
1144            this._xmlGroupName = xmlGroupName;
1145            this._values = values;
1146        }
1147
1148        /**
1149         * Get the fieldName.
1150         * @return the fieldName
1151         */
1152        public String getFieldName()
1153        {
1154            return _fieldName;
1155        }
1156
1157        /**
1158         * Set the fieldName.
1159         * @param fieldName the fieldName to set
1160         */
1161        public void setFieldName(String fieldName)
1162        {
1163            this._fieldName = fieldName;
1164        }
1165
1166        /**
1167         * Get the luceneTermName.
1168         * @return the luceneTermName
1169         */
1170        public String getLuceneTermName()
1171        {
1172            return _luceneTermName;
1173        }
1174
1175        /**
1176         * Set the luceneTermName.
1177         * @param luceneTermName the luceneTermName to set
1178         */
1179        public void setLuceneTermName(String luceneTermName)
1180        {
1181            this._luceneTermName = luceneTermName;
1182        }
1183
1184        /**
1185         * Get the xmlElementName.
1186         * @return the xmlElementName
1187         */
1188        public String getXmlElementName()
1189        {
1190            return _xmlElementName;
1191        }
1192
1193        /**
1194         * Set the xmlElementName.
1195         * @param xmlElementName the xmlElementName to set
1196         */
1197        public void setXmlElementName(String xmlElementName)
1198        {
1199            this._xmlElementName = xmlElementName;
1200        }
1201
1202        /**
1203         * Get the xmlGroupName.
1204         * @return the xmlGroupName
1205         */
1206        public String getXmlGroupName()
1207        {
1208            return _xmlGroupName;
1209        }
1210
1211        /**
1212         * Set the xmlGroupName.
1213         * @param xmlGroupName the xmlGroupName to set
1214         */
1215        public void setXmlGroupName(String xmlGroupName)
1216        {
1217            this._xmlGroupName = xmlGroupName;
1218        }
1219
1220        /**
1221         * Get the values.
1222         * @return the values
1223         */
1224        public Map<String, I18nizableText> getValues()
1225        {
1226            return _values;
1227        }
1228
1229        /**
1230         * Set the values.
1231         * @param values the values to set
1232         */
1233        public void setValues(Map<String, I18nizableText> values)
1234        {
1235            this._values = values;
1236        }
1237    }
1238}