001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.plugins.syndication; 017 018import java.io.IOException; 019import java.io.InputStream; 020import java.io.Reader; 021import java.net.HttpURLConnection; 022import java.net.MalformedURLException; 023import java.util.Calendar; 024import java.util.Collection; 025import java.util.Date; 026import java.util.GregorianCalendar; 027import java.util.concurrent.CountDownLatch; 028import java.util.concurrent.ExecutionException; 029import java.util.concurrent.TimeUnit; 030 031import org.apache.avalon.framework.activity.Initializable; 032import org.apache.avalon.framework.component.Component; 033import org.apache.avalon.framework.context.ContextException; 034import org.apache.avalon.framework.logger.AbstractLogEnabled; 035import org.apache.avalon.framework.logger.Logger; 036import org.apache.avalon.framework.service.ServiceException; 037import org.apache.http.client.config.RequestConfig; 038import org.apache.http.client.methods.CloseableHttpResponse; 039import org.apache.http.client.methods.HttpGet; 040import org.apache.http.impl.client.CloseableHttpClient; 041import org.apache.http.impl.client.HttpClientBuilder; 042import org.apache.http.impl.client.LaxRedirectStrategy; 043 044import com.google.common.cache.CacheBuilder; 045import com.google.common.cache.CacheLoader; 046import com.google.common.cache.LoadingCache; 047import com.rometools.rome.feed.synd.SyndFeed; 048import com.rometools.rome.io.FeedException; 049import com.rometools.rome.io.SyndFeedInput; 050import com.rometools.rome.io.XmlReader; 051 052/** 053 * Feed cache, supports preloading multiple feeds in multiple concurrent threads. 054 */ 055public class FeedCache extends AbstractLogEnabled implements Component, Initializable 056{ 057 /** The component role. */ 058 public static final String ROLE = FeedCache.class.getName(); 059 060 /** The user information cache. */ 061 protected LoadingCache<String, FeedResult> _cache; 062 063 @Override 064 public void initialize() throws Exception 065 { 066 FeedCacheLoader loader = new FeedCacheLoader(); 067 068 CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder() 069 .expireAfterWrite(24, TimeUnit.HOURS); 070 071 _cache = cacheBuilder.build(loader); 072 } 073 074 /** 075 * Pre-load a collection of feeds, one by thread, to avoid getting timeouts sequentially. 076 * @param feeds the feeds to preload. 077 */ 078 public void preload(Collection<String> feeds) 079 { 080 if (getLogger().isDebugEnabled()) 081 { 082 getLogger().debug("Preloading " + feeds.size() + " feeds..."); 083 } 084 085 long start = System.currentTimeMillis(); 086 087 // Synchronization helper. 088 CountDownLatch doneSignal = new CountDownLatch(feeds.size()); 089 090 for (String feed : feeds) 091 { 092 // Test if the feed is already in the cache. 093 FeedResult result = _cache.getIfPresent(feed); 094 095 if (result == null) 096 { 097 // The feed is not in the cache: prepare and launch a thread to retrieve it. 098 FeedLoadWorker worker = new FeedLoadWorker(feed, doneSignal); 099 100 try 101 { 102 worker.initialize(getLogger()); 103 } 104 catch (Exception e) 105 { 106 // Ignore 107 } 108 109 new Thread(worker).start(); 110 } 111 else 112 { 113 // The feed is already in the cache: count down. 114 doneSignal.countDown(); 115 } 116 } 117 118 try 119 { 120 // Wait for all the URLs to be loaded (i.e. all the threads to end). 121 doneSignal.await(); 122 123 if (getLogger().isDebugEnabled()) 124 { 125 long end = System.currentTimeMillis(); 126 getLogger().debug("Feed preloading ended in " + (end - start) + " millis."); 127 } 128 } 129 catch (InterruptedException e) 130 { 131 // Ignore, let the threads finish or die. 132 } 133 } 134 135 /** 136 * Get a feed not from the cache. 137 * @param feedUrl the feed. 138 * @return the feed result. 139 * @throws IOException if an error occurs while loading the feed 140 */ 141 public FeedResult getFeedNoCache(String feedUrl) throws IOException 142 { 143 return loadFeed(feedUrl); 144 } 145 146 /** 147 * Get a feed. 148 * @param feedUrl the feed. 149 * @param lifeTime the amount of date or time to be added to the field 150 * @return the feed response. 151 */ 152 public FeedResult getFeed(String feedUrl, int lifeTime) 153 { 154 try 155 { 156 FeedResult feedResult = _cache.get(feedUrl); 157 Date dateFeed = feedResult.getCreationDate(); 158 159 GregorianCalendar calendar = new GregorianCalendar(); 160 calendar.setTime(dateFeed); 161 162 calendar.add(Calendar.MINUTE, lifeTime); 163 164 Date dateToday = new Date(); 165 if (dateToday.after(calendar.getTime())) 166 { 167 _cache.invalidate(feedUrl); 168 return _cache.get(feedUrl); 169 } 170 else 171 { 172 return feedResult; 173 } 174 } 175 catch (ExecutionException e) 176 { 177 getLogger().error(e.getLocalizedMessage()); 178 } 179 180 return null; 181 } 182 183 /** 184 * Build a HttpClient object 185 * @return The HttpClient object 186 */ 187 protected CloseableHttpClient _getHttpClient() 188 { 189 RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(2000).setSocketTimeout(2000).build(); 190 return HttpClientBuilder.create() 191 .useSystemProperties() 192 .setDefaultRequestConfig(requestConfig) 193 .setRedirectStrategy(new LaxRedirectStrategy()) // follow 301 and 302 redirections 194 .build(); 195 } 196 197 /** 198 * Retrieve a feed's content to store it into the cache. 199 * @param feedUrl the feed to load. 200 * @return the feed content. 201 * @throws IOException if an error occurs with the {@link HttpURLConnection} 202 */ 203 protected FeedResult loadFeed(String feedUrl) throws IOException 204 { 205 FeedResult result = new FeedResult(); 206 207 Date todayDate = new Date(); 208 result.setCreationDate(todayDate); 209 210 SyndFeedInput input = new SyndFeedInput(); 211 SyndFeed synFeed = null; 212 213 try (CloseableHttpClient httpClient = _getHttpClient()) 214 { 215 HttpGet feedSource = new HttpGet(feedUrl); 216 217 CloseableHttpResponse response = httpClient.execute(feedSource); 218 219 int statusCode = response.getStatusLine().getStatusCode(); 220 if (statusCode != 200) 221 { 222 result.setStatus(FeedResult.STATUS_ERROR); 223 result.setMessageError("Unable to join the RSS feed : " + feedUrl + ". HTTP response code is " + statusCode + "."); 224 getLogger().error("Unable to join the RSS feed : " + feedUrl + ". HTTP response code is " + statusCode + "."); 225 return result; 226 } 227 228 try (InputStream is = response.getEntity().getContent(); Reader reader = new XmlReader(is)) 229 { 230 synFeed = input.build(reader); 231 } 232 233 result.setStatus(FeedResult.STATUS_OK); 234 result.setResponse(synFeed); 235 } 236 catch (FeedException e) 237 { 238 result.setStatus(FeedResult.STATUS_ERROR); 239 result.setMessageError(e.getLocalizedMessage()); 240 getLogger().error("Unable to read the RSS feed to the url : " + feedUrl); 241 } 242 catch (MalformedURLException e) 243 { 244 result.setStatus(FeedResult.STATUS_ERROR); 245 result.setMessageError(e.getLocalizedMessage()); 246 getLogger().error("Unable to read the RSS feed to the url : " + feedUrl + ". Malformed url"); 247 } 248 catch (IOException e) 249 { 250 result.setStatus(FeedResult.STATUS_ERROR); 251 result.setMessageError(e.getLocalizedMessage()); 252 getLogger().error("Unable to read the RSS feed to the url : " + feedUrl); 253 } 254 catch (Exception e) 255 { 256 result.setStatus(FeedResult.STATUS_ERROR); 257 result.setMessageError(e.getLocalizedMessage()); 258 getLogger().error("Unable to read the RSS feed to the url : " + feedUrl); 259 } 260 261 return result; 262 } 263 264 /** 265 * The feed cache loader, delegates the loading to the component's loadFeed method (to be easily overridable). 266 */ 267 protected class FeedCacheLoader extends CacheLoader<String, FeedResult> 268 { 269 @Override 270 public FeedResult load(String feedUrl) throws Exception 271 { 272 return loadFeed(feedUrl); 273 } 274 } 275 276 /** 277 * Runnable loading an URL into the cache. 278 */ 279 protected class FeedLoadWorker implements Runnable 280 { 281 282 /** The logger. */ 283 protected Logger _logger; 284 285 /** Is the engine initialized ? */ 286 protected boolean _initialized; 287 288 private final CountDownLatch _endSignal; 289 290 private final String _feedUrl; 291 292 /** 293 * Build a worker loading a specific feed. 294 * @param feedUrl the feed to load. 295 * @param endSignal the signal when done. 296 */ 297 FeedLoadWorker(String feedUrl, CountDownLatch endSignal) 298 { 299 this._feedUrl = feedUrl; 300 this._endSignal = endSignal; 301 } 302 303 /** 304 * Initialize the feed loader. 305 * @param logger the logger. 306 * @throws ContextException if the context is badly formed 307 * @throws ServiceException if somthing goes wrong with the service 308 */ 309 public void initialize(Logger logger) throws ContextException, ServiceException 310 { 311 _logger = logger; 312 _initialized = true; 313 } 314 315 public void run() 316 { 317 try 318 { 319 if (_logger.isDebugEnabled()) 320 { 321 _logger.debug("Preparing to load the URL '" + _feedUrl + "'"); 322 } 323 324 long start = System.currentTimeMillis(); 325 326 _checkInitialization(); 327 328 // Load the feed and signal that it's done. 329 _cache.get(_feedUrl); 330 331 if (_logger.isDebugEnabled()) 332 { 333 long end = System.currentTimeMillis(); 334 _logger.debug("URL '" + _feedUrl + "' was loaded successfully in " + (end - start) + " millis."); 335 } 336 } 337 catch (Exception e) 338 { 339 _logger.error("An error occurred loading the URL '" + _feedUrl + "'", e); 340 } 341 finally 342 { 343 // Signal that the worker has finished. 344 _endSignal.countDown(); 345 346 // Dispose of the resources. 347 _dispose(); 348 } 349 } 350 351 /** 352 * Check the initialization and throw an exception if not initialized. 353 */ 354 protected void _checkInitialization() 355 { 356 if (!_initialized) 357 { 358 String message = "Le composant de synchronisation doit être initialisé avant d'être lancé."; //TODO 359 _logger.error(message); 360 throw new IllegalStateException(message); 361 } 362 } 363 364 /** 365 * Dispose of the resources and looked-up components. 366 */ 367 protected void _dispose() 368 { 369 _initialized = false; 370 } 371 372 } 373 374}