001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.plugins.syndication; 017 018import java.io.IOException; 019import java.io.InputStream; 020import java.io.Reader; 021import java.net.HttpURLConnection; 022import java.util.Calendar; 023import java.util.Collection; 024import java.util.Date; 025import java.util.GregorianCalendar; 026import java.util.concurrent.CountDownLatch; 027import java.util.concurrent.ExecutionException; 028import java.util.concurrent.TimeUnit; 029 030import org.apache.avalon.framework.activity.Initializable; 031import org.apache.avalon.framework.component.Component; 032import org.apache.avalon.framework.context.ContextException; 033import org.apache.avalon.framework.logger.AbstractLogEnabled; 034import org.apache.avalon.framework.logger.Logger; 035import org.apache.avalon.framework.service.ServiceException; 036import org.apache.http.client.config.RequestConfig; 037import org.apache.http.client.methods.CloseableHttpResponse; 038import org.apache.http.client.methods.HttpGet; 039import org.apache.http.impl.client.CloseableHttpClient; 040import org.apache.http.impl.client.HttpClientBuilder; 041import org.apache.http.impl.client.LaxRedirectStrategy; 042 043import com.google.common.cache.CacheBuilder; 044import com.google.common.cache.CacheLoader; 045import com.google.common.cache.LoadingCache; 046import com.rometools.rome.feed.synd.SyndFeed; 047import com.rometools.rome.io.SyndFeedInput; 048import com.rometools.rome.io.XmlReader; 049 050/** 051 * Feed cache, supports preloading multiple feeds in multiple concurrent threads. 052 */ 053public class FeedCache extends AbstractLogEnabled implements Component, Initializable 054{ 055 /** The component role. */ 056 public static final String ROLE = FeedCache.class.getName(); 057 058 /** The user information cache. */ 059 protected LoadingCache<String, FeedResult> _cache; 060 061 @Override 062 public void initialize() throws Exception 063 { 064 FeedCacheLoader loader = new FeedCacheLoader(); 065 066 CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder() 067 .expireAfterWrite(24, TimeUnit.HOURS); 068 069 _cache = cacheBuilder.build(loader); 070 } 071 072 /** 073 * Pre-load a collection of feeds, one by thread, to avoid getting timeouts sequentially. 074 * @param feeds the feeds to preload. 075 */ 076 public void preload(Collection<String> feeds) 077 { 078 if (getLogger().isDebugEnabled()) 079 { 080 getLogger().debug("Preloading " + feeds.size() + " feeds..."); 081 } 082 083 long start = System.currentTimeMillis(); 084 085 // Synchronization helper. 086 CountDownLatch doneSignal = new CountDownLatch(feeds.size()); 087 088 for (String feed : feeds) 089 { 090 // Test if the feed is already in the cache. 091 FeedResult result = _cache.getIfPresent(feed); 092 093 if (result == null) 094 { 095 // The feed is not in the cache: prepare and launch a thread to retrieve it. 096 FeedLoadWorker worker = new FeedLoadWorker(feed, doneSignal); 097 098 try 099 { 100 worker.initialize(getLogger()); 101 } 102 catch (Exception e) 103 { 104 // Ignore 105 } 106 107 new Thread(worker).start(); 108 } 109 else 110 { 111 // The feed is already in the cache: count down. 112 doneSignal.countDown(); 113 } 114 } 115 116 try 117 { 118 // Wait for all the URLs to be loaded (i.e. all the threads to end). 119 doneSignal.await(); 120 121 if (getLogger().isDebugEnabled()) 122 { 123 long end = System.currentTimeMillis(); 124 getLogger().debug("Feed preloading ended in " + (end - start) + " millis."); 125 } 126 } 127 catch (InterruptedException e) 128 { 129 // Ignore, let the threads finish or die. 130 } 131 } 132 133 /** 134 * Get a feed not from the cache. 135 * @param feedUrl the feed. 136 * @return the feed result. 137 * @throws IOException if an error occurs while loading the feed 138 */ 139 public FeedResult getFeedNoCache(String feedUrl) throws IOException 140 { 141 return loadFeed(feedUrl); 142 } 143 144 /** 145 * Get a feed. 146 * @param feedUrl the feed. 147 * @param lifeTime the amount of date or time to be added to the field 148 * @return the feed response. 149 */ 150 public FeedResult getFeed(String feedUrl, int lifeTime) 151 { 152 try 153 { 154 FeedResult feedResult = _cache.get(feedUrl); 155 Date dateFeed = feedResult.getCreationDate(); 156 157 GregorianCalendar calendar = new GregorianCalendar(); 158 calendar.setTime(dateFeed); 159 160 calendar.add(Calendar.MINUTE, lifeTime); 161 162 Date dateToday = new Date(); 163 if (dateToday.after(calendar.getTime())) 164 { 165 _cache.invalidate(feedUrl); 166 return _cache.get(feedUrl); 167 } 168 else 169 { 170 return feedResult; 171 } 172 } 173 catch (ExecutionException e) 174 { 175 getLogger().error(e.getLocalizedMessage()); 176 } 177 178 return null; 179 } 180 181 /** 182 * Build a HttpClient object 183 * @return The HttpClient object 184 */ 185 protected CloseableHttpClient _getHttpClient() 186 { 187 RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(2000).setSocketTimeout(2000).build(); 188 return HttpClientBuilder.create() 189 .useSystemProperties() 190 .setDefaultRequestConfig(requestConfig) 191 .setRedirectStrategy(new LaxRedirectStrategy()) // follow 301 and 302 redirections 192 .build(); 193 } 194 195 /** 196 * Retrieve a feed's content to store it into the cache. 197 * @param feedUrl the feed to load. 198 * @return the feed content. 199 * @throws IOException if an error occurs with the {@link HttpURLConnection} 200 */ 201 protected FeedResult loadFeed(String feedUrl) throws IOException 202 { 203 FeedResult result = new FeedResult(); 204 205 Date todayDate = new Date(); 206 result.setCreationDate(todayDate); 207 208 SyndFeedInput input = new SyndFeedInput(); 209 SyndFeed synFeed = null; 210 211 try (CloseableHttpClient httpClient = _getHttpClient()) 212 { 213 HttpGet feedSource = new HttpGet(feedUrl); 214 215 CloseableHttpResponse response = httpClient.execute(feedSource); 216 217 int statusCode = response.getStatusLine().getStatusCode(); 218 if (statusCode != 200) 219 { 220 result.setStatus(FeedResult.STATUS_ERROR); 221 result.setMessageError("Unable to join the RSS feed : " + feedUrl + ". HTTP response code is " + statusCode + "."); 222 getLogger().error("Unable to join the RSS feed : " + feedUrl + ". HTTP response code is " + statusCode + "."); 223 return result; 224 } 225 226 try (InputStream is = response.getEntity().getContent(); Reader reader = new XmlReader(is)) 227 { 228 synFeed = input.build(reader); 229 } 230 231 result.setStatus(FeedResult.STATUS_OK); 232 result.setResponse(synFeed); 233 } 234 catch (Exception e) 235 { 236 result.setStatus(FeedResult.STATUS_ERROR); 237 result.setMessageError(e.getLocalizedMessage()); 238 getLogger().error("Unable to read the RSS feed to the url : " + feedUrl, e); 239 } 240 241 return result; 242 } 243 244 /** 245 * The feed cache loader, delegates the loading to the component's loadFeed method (to be easily overridable). 246 */ 247 protected class FeedCacheLoader extends CacheLoader<String, FeedResult> 248 { 249 @Override 250 public FeedResult load(String feedUrl) throws Exception 251 { 252 return loadFeed(feedUrl); 253 } 254 } 255 256 /** 257 * Runnable loading an URL into the cache. 258 */ 259 protected class FeedLoadWorker implements Runnable 260 { 261 262 /** The logger. */ 263 protected Logger _logger; 264 265 /** Is the engine initialized ? */ 266 protected boolean _initialized; 267 268 private final CountDownLatch _endSignal; 269 270 private final String _feedUrl; 271 272 /** 273 * Build a worker loading a specific feed. 274 * @param feedUrl the feed to load. 275 * @param endSignal the signal when done. 276 */ 277 FeedLoadWorker(String feedUrl, CountDownLatch endSignal) 278 { 279 this._feedUrl = feedUrl; 280 this._endSignal = endSignal; 281 } 282 283 /** 284 * Initialize the feed loader. 285 * @param logger the logger. 286 * @throws ContextException if the context is badly formed 287 * @throws ServiceException if somthing goes wrong with the service 288 */ 289 public void initialize(Logger logger) throws ContextException, ServiceException 290 { 291 _logger = logger; 292 _initialized = true; 293 } 294 295 public void run() 296 { 297 try 298 { 299 if (_logger.isDebugEnabled()) 300 { 301 _logger.debug("Preparing to load the URL '" + _feedUrl + "'"); 302 } 303 304 long start = System.currentTimeMillis(); 305 306 _checkInitialization(); 307 308 // Load the feed and signal that it's done. 309 _cache.get(_feedUrl); 310 311 if (_logger.isDebugEnabled()) 312 { 313 long end = System.currentTimeMillis(); 314 _logger.debug("URL '" + _feedUrl + "' was loaded successfully in " + (end - start) + " millis."); 315 } 316 } 317 catch (Exception e) 318 { 319 _logger.error("An error occurred loading the URL '" + _feedUrl + "'", e); 320 } 321 finally 322 { 323 // Signal that the worker has finished. 324 _endSignal.countDown(); 325 326 // Dispose of the resources. 327 _dispose(); 328 } 329 } 330 331 /** 332 * Check the initialization and throw an exception if not initialized. 333 */ 334 protected void _checkInitialization() 335 { 336 if (!_initialized) 337 { 338 String message = "Le composant de synchronisation doit être initialisé avant d'être lancé."; //TODO 339 _logger.error(message); 340 throw new IllegalStateException(message); 341 } 342 } 343 344 /** 345 * Dispose of the resources and looked-up components. 346 */ 347 protected void _dispose() 348 { 349 _initialized = false; 350 } 351 352 } 353 354}