001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.plugins.syndication; 017 018import java.io.IOException; 019import java.io.InputStream; 020import java.io.Reader; 021import java.net.HttpURLConnection; 022import java.util.Calendar; 023import java.util.Collection; 024import java.util.Date; 025import java.util.GregorianCalendar; 026import java.util.concurrent.CountDownLatch; 027import java.util.concurrent.ExecutionException; 028import java.util.concurrent.TimeUnit; 029 030import org.apache.avalon.framework.activity.Initializable; 031import org.apache.avalon.framework.component.Component; 032import org.apache.avalon.framework.context.ContextException; 033import org.apache.avalon.framework.logger.AbstractLogEnabled; 034import org.apache.avalon.framework.logger.Logger; 035import org.apache.avalon.framework.service.ServiceException; 036import org.apache.http.client.config.RequestConfig; 037import org.apache.http.client.methods.CloseableHttpResponse; 038import org.apache.http.client.methods.HttpGet; 039import org.apache.http.impl.client.CloseableHttpClient; 040import org.apache.http.impl.client.HttpClientBuilder; 041import org.apache.http.impl.client.LaxRedirectStrategy; 042 043import org.ametys.runtime.config.Config; 044 045import com.google.common.cache.CacheBuilder; 046import com.google.common.cache.CacheLoader; 047import com.google.common.cache.LoadingCache; 048import com.rometools.rome.feed.synd.SyndFeed; 049import com.rometools.rome.io.SyndFeedInput; 050import com.rometools.rome.io.XmlReader; 051 052/** 053 * Feed cache, supports preloading multiple feeds in multiple concurrent threads. 054 */ 055public class FeedCache extends AbstractLogEnabled implements Component, Initializable 056{ 057 /** The component role. */ 058 public static final String ROLE = FeedCache.class.getName(); 059 060 /** The user information cache. */ 061 protected LoadingCache<String, FeedResult> _cache; 062 063 @Override 064 public void initialize() throws Exception 065 { 066 FeedCacheLoader loader = new FeedCacheLoader(); 067 068 CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder() 069 .expireAfterWrite(24, TimeUnit.HOURS); 070 071 _cache = cacheBuilder.build(loader); 072 } 073 074 /** 075 * Pre-load a collection of feeds, one by thread, to avoid getting timeouts sequentially. 076 * @param feeds the feeds to preload. 077 */ 078 public void preload(Collection<String> feeds) 079 { 080 if (getLogger().isDebugEnabled()) 081 { 082 getLogger().debug("Preloading " + feeds.size() + " feeds..."); 083 } 084 085 long start = System.currentTimeMillis(); 086 087 // Synchronization helper. 088 CountDownLatch doneSignal = new CountDownLatch(feeds.size()); 089 090 for (String feed : feeds) 091 { 092 // Test if the feed is already in the cache. 093 FeedResult result = _cache.getIfPresent(feed); 094 095 if (result == null) 096 { 097 // The feed is not in the cache: prepare and launch a thread to retrieve it. 098 FeedLoadWorker worker = new FeedLoadWorker(feed, doneSignal); 099 100 try 101 { 102 worker.initialize(getLogger()); 103 } 104 catch (Exception e) 105 { 106 // Ignore 107 } 108 109 new Thread(worker).start(); 110 } 111 else 112 { 113 // The feed is already in the cache: count down. 114 doneSignal.countDown(); 115 } 116 } 117 118 try 119 { 120 // Wait for all the URLs to be loaded (i.e. all the threads to end). 121 doneSignal.await(); 122 123 if (getLogger().isDebugEnabled()) 124 { 125 long end = System.currentTimeMillis(); 126 getLogger().debug("Feed preloading ended in " + (end - start) + " millis."); 127 } 128 } 129 catch (InterruptedException e) 130 { 131 // Ignore, let the threads finish or die. 132 } 133 } 134 135 /** 136 * Get a feed not from the cache. 137 * @param feedUrl the feed. 138 * @return the feed result. 139 * @throws IOException if an error occurs while loading the feed 140 */ 141 public FeedResult getFeedNoCache(String feedUrl) throws IOException 142 { 143 return loadFeed(feedUrl); 144 } 145 146 /** 147 * Get a feed. 148 * @param feedUrl the feed. 149 * @param lifeTime the amount of date or time to be added to the field 150 * @return the feed response. 151 */ 152 public FeedResult getFeed(String feedUrl, int lifeTime) 153 { 154 try 155 { 156 FeedResult feedResult = _cache.get(feedUrl); 157 Date dateFeed = feedResult.getCreationDate(); 158 159 GregorianCalendar calendar = new GregorianCalendar(); 160 calendar.setTime(dateFeed); 161 162 calendar.add(Calendar.MINUTE, lifeTime); 163 164 Date dateToday = new Date(); 165 if (dateToday.after(calendar.getTime())) 166 { 167 _cache.invalidate(feedUrl); 168 return _cache.get(feedUrl); 169 } 170 else 171 { 172 return feedResult; 173 } 174 } 175 catch (ExecutionException e) 176 { 177 getLogger().error(e.getLocalizedMessage()); 178 } 179 180 return null; 181 } 182 183 /** 184 * Build a HttpClient object 185 * @return The HttpClient object 186 */ 187 protected CloseableHttpClient _getHttpClient() 188 { 189 int timeoutValue = Config.getInstance().getValue("syndication.timeout", false, 2000L).intValue(); 190 RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(timeoutValue).setSocketTimeout(timeoutValue).build(); 191 return HttpClientBuilder.create() 192 .useSystemProperties() 193 .setDefaultRequestConfig(requestConfig) 194 .setRedirectStrategy(new LaxRedirectStrategy()) // follow 301 and 302 redirections 195 .build(); 196 } 197 198 /** 199 * Retrieve a feed's content to store it into the cache. 200 * @param feedUrl the feed to load. 201 * @return the feed content. 202 * @throws IOException if an error occurs with the {@link HttpURLConnection} 203 */ 204 protected FeedResult loadFeed(String feedUrl) throws IOException 205 { 206 FeedResult result = new FeedResult(); 207 208 Date todayDate = new Date(); 209 result.setCreationDate(todayDate); 210 211 SyndFeedInput input = new SyndFeedInput(); 212 SyndFeed synFeed = null; 213 214 try (CloseableHttpClient httpClient = _getHttpClient()) 215 { 216 HttpGet feedSource = new HttpGet(feedUrl); 217 218 CloseableHttpResponse response = httpClient.execute(feedSource); 219 220 int statusCode = response.getStatusLine().getStatusCode(); 221 if (statusCode != 200) 222 { 223 result.setStatus(FeedResult.STATUS_ERROR); 224 result.setMessageError("Unable to join the RSS feed : " + feedUrl + ". HTTP response code is " + statusCode + "."); 225 getLogger().error("Unable to join the RSS feed : " + feedUrl + ". HTTP response code is " + statusCode + "."); 226 return result; 227 } 228 229 try (InputStream is = response.getEntity().getContent(); Reader reader = new XmlReader(is)) 230 { 231 synFeed = input.build(reader); 232 } 233 234 result.setStatus(FeedResult.STATUS_OK); 235 result.setResponse(synFeed); 236 } 237 catch (Exception e) 238 { 239 result.setStatus(FeedResult.STATUS_ERROR); 240 result.setMessageError(e.getLocalizedMessage()); 241 getLogger().error("Unable to read the RSS feed to the url : " + feedUrl, e); 242 } 243 244 return result; 245 } 246 247 /** 248 * The feed cache loader, delegates the loading to the component's loadFeed method (to be easily overridable). 249 */ 250 protected class FeedCacheLoader extends CacheLoader<String, FeedResult> 251 { 252 @Override 253 public FeedResult load(String feedUrl) throws Exception 254 { 255 return loadFeed(feedUrl); 256 } 257 } 258 259 /** 260 * Runnable loading an URL into the cache. 261 */ 262 protected class FeedLoadWorker implements Runnable 263 { 264 265 /** The logger. */ 266 protected Logger _logger; 267 268 /** Is the engine initialized ? */ 269 protected boolean _initialized; 270 271 private final CountDownLatch _endSignal; 272 273 private final String _feedUrl; 274 275 /** 276 * Build a worker loading a specific feed. 277 * @param feedUrl the feed to load. 278 * @param endSignal the signal when done. 279 */ 280 FeedLoadWorker(String feedUrl, CountDownLatch endSignal) 281 { 282 this._feedUrl = feedUrl; 283 this._endSignal = endSignal; 284 } 285 286 /** 287 * Initialize the feed loader. 288 * @param logger the logger. 289 * @throws ContextException if the context is badly formed 290 * @throws ServiceException if somthing goes wrong with the service 291 */ 292 public void initialize(Logger logger) throws ContextException, ServiceException 293 { 294 _logger = logger; 295 _initialized = true; 296 } 297 298 public void run() 299 { 300 try 301 { 302 if (_logger.isDebugEnabled()) 303 { 304 _logger.debug("Preparing to load the URL '" + _feedUrl + "'"); 305 } 306 307 long start = System.currentTimeMillis(); 308 309 _checkInitialization(); 310 311 // Load the feed and signal that it's done. 312 _cache.get(_feedUrl); 313 314 if (_logger.isDebugEnabled()) 315 { 316 long end = System.currentTimeMillis(); 317 _logger.debug("URL '" + _feedUrl + "' was loaded successfully in " + (end - start) + " millis."); 318 } 319 } 320 catch (Exception e) 321 { 322 _logger.error("An error occurred loading the URL '" + _feedUrl + "'", e); 323 } 324 finally 325 { 326 // Signal that the worker has finished. 327 _endSignal.countDown(); 328 329 // Dispose of the resources. 330 _dispose(); 331 } 332 } 333 334 /** 335 * Check the initialization and throw an exception if not initialized. 336 */ 337 protected void _checkInitialization() 338 { 339 if (!_initialized) 340 { 341 String message = "Le composant de synchronisation doit être initialisé avant d'être lancé."; //TODO 342 _logger.error(message); 343 throw new IllegalStateException(message); 344 } 345 } 346 347 /** 348 * Dispose of the resources and looked-up components. 349 */ 350 protected void _dispose() 351 { 352 _initialized = false; 353 } 354 355 } 356 357}