Source code

001/*
002 *  Copyright 2019 Anyware Services
003 *
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *
008 *      http://www.apache.org/licenses/LICENSE-2.0
009 *
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package org.ametys.core.util;
017
018import java.io.IOException;
019import java.net.HttpURLConnection;
020import java.net.MalformedURLException;
021import java.net.SocketTimeoutException;
022import java.net.URI;
023import java.net.URISyntaxException;
024import java.net.URL;
025import java.net.UnknownHostException;
026import java.util.ArrayList;
027import java.util.Collections;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.Map.Entry;
032import java.util.Optional;
033import java.util.regex.Pattern;
034
035import javax.net.ssl.SSLHandshakeException;
036
037import org.apache.avalon.framework.component.Component;
038import org.apache.commons.lang3.RegExUtils;
039import org.apache.commons.lang3.StringUtils;
040import org.apache.hc.client5.http.classic.HttpClient;
041import org.apache.hc.client5.http.config.ConnectionConfig;
042import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
043import org.apache.hc.client5.http.impl.classic.HttpClients;
044import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
045import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
046import org.apache.hc.core5.util.Timeout;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050import org.ametys.core.ui.Callable;
051
052/**
053 * Utility class for HTTP urls.
054 */
055public class HttpUtils implements Component
056{
057    /** The Avalon role */
058    public static final String ROLE = HttpUtils.class.getName();
059    
060    /** Regexp for HTTP url */
061    public static final Pattern HTTP_URL_VALIDATOR = Pattern.compile("^(https?:\\/\\/.+)?$");
062    
063    /** The status of HTTP check */
064    public static enum HttpCheck
065    {
066        /** All right */
067        SUCCESS,
068        /** Server error. */
069        SERVER_ERROR,
070        /** URL not found.*/
071        NOT_FOUND,
072        /** Unauthorized. */
073        UNAUTHORIZED,
074        /** Timeout (too long) */
075        TIMEOUT,
076        /** A redirect occurs */
077        REDIRECT,
078        /** Security level error */
079        SECURITY_LEVEL_ERROR,
080        /** No HTTP url */
081        NOT_HTTP
082        
083    }
084    
085    /**
086     * HTTP Status-Code 307: Temporay Redirect
087     */
088    public static final int HTTP_REDIRECT_TEMP = 307;
089    
090    /**
091     * HTTP Status-Code 308: Temporay Redirect
092     */
093    public static final int HTTP_REDIRECT_PERM = 308;
094    
095    
096    private static Logger __logger = LoggerFactory.getLogger(HttpUtils.class);
097    
098    /**
099     * Prepare the connection to the remote url
100     * @param url The url
101     * @param userAgent The user agent. Can be null.
102     * @param method The method for teh URL request. Can be null.
103     * @param timeout The connection timeout in milliseconds. Set to -1 to not set a timeout.
104     * @param readTimeOut The read timeout in milliseconds. Set to -1 to not set a timeout.
105     * @param followRedirects Sets to true to follow HTTP redirects
106     * @return The URL connection
107     * @throws MalformedURLException If the given url is a malformed URL
108     * @throws IOException if an I/O exception occurs
109     */
110    public static HttpURLConnection prepareConnection(String url, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects) throws MalformedURLException, IOException
111    {
112        return prepareConnection(url, userAgent, method, timeout, readTimeOut, followRedirects, Collections.EMPTY_MAP);
113    }
114    
115    /**
116     * Prepare the connection to the remote url
117     * @param httpUrl The HTTP url
118     * @param userAgent The user agent. Can be null.
119     * @param method The method for teh URL request. Can be null.
120     * @param timeout The connection timeout in milliseconds. Set to -1 to not set a timeout.
121     * @param readTimeOut The read timeout in milliseconds. Set to -1 to not set a timeout.
122     * @param followRedirects Sets to true to follow HTTP redirects
123     * @param requestHeaders The request headers.
124     * @return The URL connection
125     * @throws MalformedURLException If the given url is a malformed URL
126     * @throws IOException if an I/O exception occurs
127     */
128    public static HttpURLConnection prepareConnection(String httpUrl, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects, Map<String, String> requestHeaders) throws MalformedURLException, IOException
129    {
130        try
131        {
132            return prepareConnection(new URI(httpUrl).toURL(), userAgent, method, timeout, readTimeOut, followRedirects, requestHeaders);
133        }
134        catch (URISyntaxException e)
135        {
136            throw new IOException(e);
137        }
138    }
139    
140    /**
141     * Prepare the connection to the remote url
142     * @param url The url
143     * @param userAgent The user agent. Can be null.
144     * @param method The method for teh URL request. Can be null.
145     * @param timeout The connection timeout in milliseconds. Set to -1 to not set a timeout.
146     * @param readTimeOut The read timeout in milliseconds. Set to -1 to not set a timeout.
147     * @param followRedirects Sets to true to follow HTTP redirects
148     * @param requestHeaders The request headers.
149     * @return The URL connection
150     * @throws MalformedURLException If the given url is a malformed URL
151     * @throws IOException if an I/O exception occurs
152     */
153    public static HttpURLConnection prepareConnection(URL url, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects, Map<String, String> requestHeaders) throws MalformedURLException, IOException
154    {
155        HttpURLConnection httpUrlConnection = (HttpURLConnection) url.openConnection();
156        
157        if (userAgent != null)
158        {
159            httpUrlConnection.setRequestProperty("User-Agent", userAgent);
160        }
161        if (method != null)
162        {
163            httpUrlConnection.setRequestMethod(method);
164        }
165        if (timeout != -1)
166        {
167            httpUrlConnection.setConnectTimeout(timeout);
168        }
169        
170        if (readTimeOut != -1)
171        {
172            httpUrlConnection.setReadTimeout(readTimeOut);
173        }
174        
175        if (followRedirects)
176        {
177            httpUrlConnection.setInstanceFollowRedirects(true);
178        }
179        
180        for (Entry<String, String> headers : requestHeaders.entrySet())
181        {
182            httpUrlConnection.setRequestProperty(headers.getKey(), headers.getValue());
183        }
184        
185        return httpUrlConnection;
186    }
187    
188    /**
189     * Check the HTTP url
190     * @param httpUrl The HTTP url to test
191     * @param userAgent The user agent. Can be null.
192     * @param method The method for teh URL request. Can be null.
193     * @param timeout The connection timeout in milliseconds. Set to -1 to not set a timeout.
194     * @param readTimeOut The read timeout in milliseconds. Set to -1 to not set a timeout.
195     * @param followRedirects Sets to true to follow HTTP redirects
196     * @return The URL connection
197     */
198    public static HttpCheckReport checkHttpUrl(String httpUrl, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects)
199    {
200        return checkHttpUrl(httpUrl, userAgent, method, timeout, readTimeOut, followRedirects, Collections.EMPTY_MAP);
201    }
202    
203    /**
204     * Check the HTTP url
205     * @param httpUrl The url to test
206     * @param userAgent The user agent. Can be null.
207     * @param method The method for teh URL request. Can be null.
208     * @param timeout The connection timeout in milliseconds. Set to -1 to not set a timeout.
209     * @param readTimeOut The read timeout in milliseconds. Set to -1 to not set a timeout.
210     * @param followRedirects Sets to true to follow HTTP redirects
211     * @param requestHeaders The request headers.
212     * @return The URL connection
213     */
214    public static HttpCheckReport checkHttpUrl(String httpUrl, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects, Map<String, String> requestHeaders)
215    {
216        if (!HTTP_URL_VALIDATOR.matcher(StringUtils.defaultIfEmpty(httpUrl, "")).matches())
217        {
218            __logger.debug("Url '{}' is not a valid HTTP url", httpUrl);
219            return new HttpCheckReport(HttpCheck.NOT_HTTP);
220        }
221        
222        try
223        {
224            return checkHttpUrl(new URI(httpUrl).toURL(), userAgent, method, timeout, readTimeOut, followRedirects, requestHeaders);
225        }
226        catch (MalformedURLException | URISyntaxException e)
227        {
228            __logger.debug("Unable to parse '{}' as a HTTP url", httpUrl, e);
229            return new HttpCheckReport(HttpCheck.NOT_HTTP);
230        }
231    }
232    
233    /**
234     * Check the HTTP url
235     * @param url The url to test
236     * @param userAgent The user agent. Can be null.
237     * @param method The method for teh URL request. Can be null.
238     * @param timeout The connection timeout in milliseconds. Set to -1 to not set a timeout.
239     * @param readTimeOut The read timeout in milliseconds. Set to -1 to not set a timeout.
240     * @param followRedirects Sets to true to follow HTTP redirects
241     * @param requestHeaders The request headers.
242     * @return The URL connection
243     */
244    public static HttpCheckReport checkHttpUrl(URL url, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects, Map<String, String> requestHeaders)
245    {
246        return _checkHttpUrl(url, userAgent, method, timeout, readTimeOut, followRedirects, requestHeaders, new ArrayList<>(), 5);
247    }
248    
249    private static HttpCheckReport _checkHttpUrl(URL url, String userAgent, String method, int timeout, int readTimeOut, boolean followRedirects, Map<String, String> requestHeaders, List<String> visitedUrl, int maxRedirects)
250    {
251        try
252        {
253            HttpURLConnection connection = prepareConnection(url, userAgent, method, timeout, readTimeOut, followRedirects, requestHeaders);
254                     
255            if (connection.getResponseCode() == HttpURLConnection.HTTP_OK)
256            {
257                __logger.debug("Check of URL '{}' successed", url);
258                return new HttpCheckReport(HttpCheck.SUCCESS);
259            }
260            else if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP /* 302 */
261                    || connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM /* 301 */
262                    || connection.getResponseCode() == HttpURLConnection.HTTP_SEE_OTHER /* 303 */
263                    || connection.getResponseCode() == HTTP_REDIRECT_TEMP /* 307 */
264                    || connection.getResponseCode() == HTTP_REDIRECT_PERM /* 308 */)
265            {
266                if (followRedirects)
267                {
268                    // Redirection between HTTP and HTTPS URLs is not followed (do it manually)
269                    if (maxRedirects < 0 || visitedUrl.contains(url.toExternalForm()))
270                    {
271                        // The maximum number of redirects has been reached or the url was already been visited when following redirects
272                        throw new IOException("Exceed max number of redirects");
273                    }
274                    
275                    visitedUrl.add(url.toExternalForm());
276                    
277                    URL base = connection.getURL();
278                    String location = connection.getHeaderField("Location");
279                    URL redirect = base.toURI().resolve(location).toURL(); // Deal with relative URLs
280                    
281                    return _checkHttpUrl(redirect, userAgent, method, timeout, readTimeOut, true, requestHeaders, visitedUrl, maxRedirects - 1);
282                }
283                else
284                {
285                    return new HttpCheckReport(HttpCheck.REDIRECT);
286                }
287            }
288            else
289            {
290                __logger.debug("Check of URL '{}' returns the status code {}", url, connection.getResponseCode());
291                
292                int responseCode = connection.getResponseCode();
293                
294                switch (responseCode)
295                {
296                    case HttpURLConnection.HTTP_NOT_FOUND:
297                        return new HttpCheckReport(HttpCheck.NOT_FOUND);
298                    case HttpURLConnection.HTTP_FORBIDDEN:
299                    case HttpURLConnection.HTTP_UNAUTHORIZED:
300                        return new HttpCheckReport(HttpCheck.UNAUTHORIZED);
301                    case HttpURLConnection.HTTP_INTERNAL_ERROR:
302                    default:
303                        return new HttpCheckReport(HttpCheck.SERVER_ERROR, Optional.ofNullable(connection.getResponseMessage()));
304                }
305            }
306        }
307        catch (SSLHandshakeException e)
308        {
309            __logger.debug("Certificate error for URL '{}'", url, e);
310            return new HttpCheckReport(HttpCheck.SECURITY_LEVEL_ERROR, Optional.ofNullable(e.getMessage()));
311        }
312        catch (SocketTimeoutException e)
313        {
314            __logger.debug("Aborting test for URL '{}' because too long", url, e);
315            return new HttpCheckReport(HttpCheck.TIMEOUT);
316        }
317        catch (UnknownHostException e)
318        {
319            __logger.debug("Unknown host for URL '{}'", url, e);
320            return new HttpCheckReport(HttpCheck.NOT_FOUND);
321        }
322        catch (IOException | URISyntaxException e)
323        {
324            __logger.debug("Cannot test URL '{}'", url, e);
325            return new HttpCheckReport(HttpCheck.SERVER_ERROR, Optional.ofNullable(e.getMessage()));
326        }
327    }
328    
329    /**
330     * Method to check a HTTP url from client side.
331     * The HTTP redirects will be followed.
332     * @param httpUrl the http url to check
333     * @return the result of the check
334     */
335    @Callable (rights = Callable.SKIP_BUILTIN_CHECK)
336    public Map<String, Object> checkHttpUrl(String httpUrl)
337    {
338        Map<String, Object> result = new HashMap<>();
339        
340        HttpCheckReport report = checkHttpUrl(httpUrl, null, null, 2000, 2000, true);
341        result.put("success", HttpCheck.SUCCESS.equals(report.status()));
342        if (report.message().isPresent())
343        {
344            result.put("message", report.message().get());
345        }
346        result.put("checkResult", report.status().name());
347        
348        return result;
349    }
350    
351    /**
352     * Represent the result of a check and a potential message
353     * @param status the result status
354     * @param message the message if needed
355     */
356    public record HttpCheckReport(HttpCheck status, Optional<String> message)
357    {
358        /**
359         * Create a report with the provided status and no message.
360         * @param result the result status
361         */
362        public HttpCheckReport(HttpCheck result)
363        {
364            this(result, Optional.empty());
365        }
366    }
367    
368    /**
369     * Create ands return a configured, ready to use, {@link HttpClient}.<br>
370     * This method is well suited to create a client dedicated to a single route.
371     * @param maxConnections the maximum simultaneous connections. If zero or negative, defaults to HttpClient's default values.
372     * @param timeout the socket and connect timeout, in seconds. If zero or negative, defaults to HttpClient's default values.
373     * @return a configured {@link HttpClient}
374     */
375    public static CloseableHttpClient createHttpClient(int maxConnections, int timeout)
376    {
377        ConnectionConfig connectionConfig = ConnectionConfig.custom()
378                                                            .setConnectTimeout(timeout > 0 ? Timeout.ofSeconds(timeout) : null)
379                                                            .setSocketTimeout(timeout > 0 ? Timeout.ofSeconds(timeout) : null)
380                                                            .build();
381        
382        PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
383                                                                                                        .setDefaultConnectionConfig(connectionConfig)
384                                                                                                        .setMaxConnTotal(maxConnections > 0 ? maxConnections : PoolingHttpClientConnectionManager.DEFAULT_MAX_TOTAL_CONNECTIONS)
385                                                                                                        .setMaxConnPerRoute(maxConnections > 0 ? maxConnections : PoolingHttpClientConnectionManager.DEFAULT_MAX_CONNECTIONS_PER_ROUTE)
386                                                                                                        .build();
387        
388        return HttpClients.custom()
389                          .setConnectionManager(connectionManager)
390                          .useSystemProperties()
391                          .build();
392    }
393    
394    /**
395     * Strips the end of an uri to remove "*.html" ends and "/" ending characters
396     * @param uri The uri to edit
397     * @return The edited uri
398     */
399    public static String sanitize(String uri)
400    {
401        return StringUtils.stripEnd(RegExUtils.removePattern(uri, "[^/]*.html$"), "/");
402    }
403}