001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.nio.ByteBuffer; 020import java.nio.CharBuffer; 021import java.nio.charset.CharsetDecoder; 022import java.nio.charset.CodingErrorAction; 023import java.nio.charset.StandardCharsets; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Collection; 028import java.util.Comparator; 029import java.util.HashMap; 030import java.util.HashSet; 031import java.util.List; 032import java.util.Map; 033import java.util.Objects; 034import java.util.Set; 035import java.util.function.Function; 036import java.util.stream.Collectors; 037import java.util.stream.StreamSupport; 038 039import javax.jcr.RepositoryException; 040 041import org.apache.avalon.framework.activity.Initializable; 042import org.apache.avalon.framework.component.Component; 043import org.apache.avalon.framework.context.Context; 044import org.apache.avalon.framework.context.ContextException; 045import org.apache.avalon.framework.context.Contextualizable; 046import org.apache.avalon.framework.service.ServiceException; 047import org.apache.avalon.framework.service.ServiceManager; 048import org.apache.avalon.framework.service.Serviceable; 049import org.apache.cocoon.components.ContextHelper; 050import org.apache.cocoon.environment.Request; 051import org.apache.commons.collections4.IterableUtils; 052import org.apache.commons.lang3.ObjectUtils; 053import org.apache.commons.lang3.StringUtils; 054import org.apache.solr.client.solrj.SolrClient; 055import org.apache.solr.client.solrj.SolrQuery; 056import org.apache.solr.client.solrj.SolrResponse; 057import org.apache.solr.client.solrj.SolrServerException; 058import org.apache.solr.client.solrj.request.CoreAdminRequest; 059import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 060import org.apache.solr.client.solrj.request.schema.SchemaRequest; 061import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 062import org.apache.solr.client.solrj.response.CoreAdminResponse; 063import org.apache.solr.client.solrj.response.QueryResponse; 064import org.apache.solr.client.solrj.response.SolrResponseBase; 065import org.apache.solr.client.solrj.response.UpdateResponse; 066import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 067import org.apache.solr.client.solrj.response.schema.SchemaResponse; 068import org.apache.solr.client.solrj.util.ClientUtils; 069import org.apache.solr.common.SolrInputDocument; 070import org.apache.solr.common.util.NamedList; 071import org.slf4j.Logger; 072 073import org.ametys.cms.indexing.IndexingException; 074import org.ametys.cms.indexing.solr.ReloadAclCacheRequest; 075import org.ametys.cms.indexing.solr.UpdateAclCacheRequest; 076import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest; 077import org.ametys.cms.repository.Content; 078import org.ametys.cms.repository.ContentQueryHelper; 079import org.ametys.cms.repository.WorkflowAwareContent; 080import org.ametys.cms.rights.solrchecking.ReadAccessHelper; 081import org.ametys.cms.search.query.ContentAttachmentQuery; 082import org.ametys.cms.search.query.DocumentTypeQuery; 083import org.ametys.cms.search.query.OrQuery; 084import org.ametys.cms.search.query.Query; 085import org.ametys.cms.search.query.ResourceLocationQuery; 086import org.ametys.cms.search.solr.NoAutoCommitUpdateClient; 087import org.ametys.cms.search.solr.SolrClientProvider; 088import org.ametys.cms.search.solr.schema.SchemaDefinition; 089import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 090import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 091import org.ametys.cms.search.solr.schema.SchemaFields; 092import org.ametys.cms.search.solr.schema.SchemaHelper; 093import org.ametys.core.group.GroupIdentity; 094import org.ametys.core.right.AllowedUsers; 095import org.ametys.core.schedule.progression.ContainerProgressionTracker; 096import org.ametys.core.schedule.progression.ProgressionTrackerFactory; 097import org.ametys.core.schedule.progression.SimpleProgressionTracker; 098import org.ametys.core.user.UserIdentity; 099import org.ametys.core.util.DateUtils; 100import org.ametys.plugins.explorer.resources.Resource; 101import org.ametys.plugins.explorer.resources.ResourceCollection; 102import org.ametys.plugins.repository.AmetysObject; 103import org.ametys.plugins.repository.AmetysObjectIterable; 104import org.ametys.plugins.repository.AmetysObjectResolver; 105import org.ametys.plugins.repository.RepositoryConstants; 106import org.ametys.plugins.repository.TraversableAmetysObject; 107import org.ametys.plugins.repository.UnknownAmetysObjectException; 108import org.ametys.plugins.repository.collection.AmetysObjectCollection; 109import org.ametys.plugins.repository.provider.AbstractRepository; 110import org.ametys.plugins.repository.provider.JackrabbitRepository; 111import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector; 112import org.ametys.plugins.repository.provider.WorkspaceSelector; 113import org.ametys.runtime.config.Config; 114import org.ametys.runtime.i18n.I18nizableText; 115import org.ametys.runtime.plugin.component.AbstractLogEnabled; 116 117/** 118 * Solr indexer. 119 */ 120public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 121{ 122 /** The component role. */ 123 public static final String ROLE = SolrIndexer.class.getName(); 124 125 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 126 127 private static final String _CONFIGSET_NAME_PREFIX = "configset-"; 128 129 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_"); 130 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general"); 131 132 private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766; 133 134 /** The ametys object resolver. */ 135 protected AmetysObjectResolver _resolver; 136 /** The schema definition provider extension point. */ 137 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 138 /** The schema helper. */ 139 protected SchemaHelper _schemaHelper; 140 /** Solr Ametys contents indexer */ 141 protected SolrContentIndexer _solrContentIndexer; 142 /** Solr workflow indexer. */ 143 protected SolrWorkflowIndexer _solrWorkflowIndexer; 144 /** Solr resource indexer. */ 145 protected SolrResourceIndexer _solrResourceIndexer; 146 147 /** The Solr client provider */ 148 protected SolrClientProvider _solrClientProvider; 149 150 /** The solr core prefix. */ 151 protected String _solrCorePrefix; 152 /** The Ametys internal URL used by Solr to query Ametys */ 153 protected String _ametysInternalUrl; 154 155 /** The workspace selector. */ 156 protected WorkspaceSelector _workspaceSelector; 157 /** The JCR repository */ 158 protected JackrabbitRepository _repository; 159 /** The helper for read access */ 160 protected ReadAccessHelper _readAccessHelper; 161 162 /** The avalon context */ 163 protected Context _context; 164 165 /** 166 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 167 * @return The date format for indexing dates 168 * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead 169 */ 170 @Deprecated 171 public static SimpleDateFormat dateFormat() 172 { 173 if (__DATE_FORMAT.get() == null) 174 { 175 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 176 } 177 return __DATE_FORMAT.get(); 178 } 179 180 /** 181 * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr. 182 * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept. 183 * @param value The string value to index 184 * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log. 185 * @param documentId The id of the document being indexed. Can be null if you do not want to log. 186 * @param fieldName The name of the field being indexed. Can be null if you do not want to log. 187 * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes) 188 */ 189 public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName) 190 { 191 if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT) 192 { 193 // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length()) 194 // As a result, value.getBytes().length <= limit 195 return value; 196 } 197 198 // There is a doubt, the string may need to be truncated (or not) 199 byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); 200 int bytesLength = valueBytes.length; 201 if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT) 202 { 203 return value; 204 } 205 206 if (ObjectUtils.allNotNull(logger, documentId, fieldName)) 207 { 208 logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT); 209 } 210 211 // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974) 212 CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT); 213 CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() 214 .onMalformedInput(CodingErrorAction.IGNORE); 215 decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true); 216 decoder.flush(charBuffer); 217 return new String(charBuffer.array(), 0, charBuffer.position()); 218 } 219 220 @Override 221 public void service(ServiceManager serviceManager) throws ServiceException 222 { 223 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 224 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 225 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 226 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 227 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 228 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 229 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 230 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 231 _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE); 232 _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE); 233 } 234 235 @Override 236 public void initialize() throws Exception 237 { 238 Config config = Config.getInstance(); 239 _solrCorePrefix = config.getValue("cms.solr.core.prefix"); 240 241 _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url"); 242 if (StringUtils.isBlank(_ametysInternalUrl)) 243 { 244 // fallback to CMS URL as internal URL is an optional parameter 245 _ametysInternalUrl = config.getValue("cms.url"); 246 } 247 } 248 249 @Override 250 public void contextualize(Context context) throws ContextException 251 { 252 _context = context; 253 } 254 255 /** 256 * Gets the 'autocommit' Solr client 257 * @param workspaceName The name of the workspace 258 * @return the Solr client 259 */ 260 protected SolrClient _getAutoCommitSolrClient(String workspaceName) 261 { 262 return _solrClientProvider.getUpdateClient(workspaceName, true); 263 } 264 265 /** 266 * Gets the 'no autocommit' Solr client 267 * @param workspaceName The name of the workspace 268 * @return the Solr client 269 */ 270 protected SolrClient _getNoAutoCommitSolrClient(String workspaceName) 271 { 272 return _solrClientProvider.getUpdateClient(workspaceName, false); 273 } 274 275 // for admin operations 276 private SolrClient _defaultSolrClient() 277 { 278 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 279 } 280 281 /** 282 * Get the names of the Solr cores. 283 * @return The names of the Solr cores. 284 * @throws IOException If an I/O error occurs. 285 * @throws SolrServerException If a Solr error occurs. 286 */ 287 @SuppressWarnings("unchecked") 288 public Set<String> getCoreNames() throws IOException, SolrServerException 289 { 290 Set<String> coreNames = new HashSet<>(); 291 292 getLogger().debug("Getting core list."); 293 294 SolrQuery query = new SolrQuery(); 295 query.setRequestHandler("/admin/cores"); 296 query.setParam("action", "STATUS"); 297 298 QueryResponse response = _defaultSolrClient().query(query); 299 300 NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status"); 301 for (Map.Entry<String, NamedList<?>> core : status) 302 { 303 String fullName = (String) core.getValue().get("name"); 304 if (fullName.startsWith(_solrCorePrefix)) 305 { 306 coreNames.add(fullName.substring(_solrCorePrefix.length())); 307 } 308 } 309 310 return coreNames; 311 } 312 313 /** 314 * Get the names of the Solr cores. 315 * @return The names of the Solr cores. 316 * @throws IOException If an I/O error occurs. 317 * @throws SolrServerException If a Solr error occurs. 318 */ 319 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 320 { 321 Set<String> coreNames = new HashSet<>(); 322 323 getLogger().debug("Getting core list."); 324 325 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 326 327 NamedList<NamedList<Object>> status = response.getCoreStatus(); 328 for (Map.Entry<String, NamedList<Object>> core : status) 329 { 330 String fullName = (String) core.getValue().get("name"); 331 if (fullName.startsWith(_solrCorePrefix)) 332 { 333 coreNames.add(fullName.substring(_solrCorePrefix.length())); 334 } 335 } 336 337 return coreNames; 338 } 339 340 /** 341 * Create a Solr core. 342 * @param name The name of the core to create. 343 * @throws IOException If an I/O error occurs. 344 * @throws SolrServerException If a Solr error occurs. 345 */ 346 public void createCore(String name) throws IOException, SolrServerException 347 { 348 String fullName = _solrCorePrefix + name; 349 String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix); 350 _createConfigset(configsetName); 351 352 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 353 354 SolrQuery query = new SolrQuery(); 355 query.setRequestHandler("/admin/cores"); 356 query.setParam("action", "CREATE"); 357 query.setParam("name", fullName); 358 query.setParam("configSet", configsetName); 359 query.setParam("property.ametys.url", _ametysInternalUrl); 360 361 QueryResponse response = _defaultSolrClient().query(query); 362 NamedList<?> results = response.getResponse(); 363 364 NamedList<?> error = (NamedList<?>) results.get("error"); 365 if (error != null) 366 { 367 throw new IOException("Error creating the core: " + error.get("msg")); 368 } 369 } 370 371 /** 372 * Updates the ametys.url property of the Solr cores. 373 */ 374 public void updateAmetysUrlCoreProperty() 375 { 376 Set<String> coreNames; 377 try 378 { 379 coreNames = getCoreNames(); 380 } 381 catch (SolrServerException | IOException e) 382 { 383 getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e); 384 return; 385 } 386 387 for (String coreName : coreNames) 388 { 389 String collection = _solrClientProvider.getCollectionName(coreName); 390 SolrResponseBase response; 391 try 392 { 393 response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection); 394 } 395 catch (SolrServerException | IOException e) 396 { 397 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e); 398 continue; 399 } 400 401 NamedList<Object> responseParams = response.getResponse(); 402 if ("ok".equals(responseParams.get("result"))) 403 { 404 Boolean valueChanged = responseParams.getBooleanArg("valueChanged"); 405 if (valueChanged) 406 { 407 getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName); 408 } 409 else 410 { 411 getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName); 412 } 413 } 414 else 415 { 416 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName); 417 } 418 } 419 } 420 421 private void _createConfigset(String name) throws IOException, SolrServerException 422 { 423 getLogger().info("Creating (if necessary) configset '{}'", name); 424 425 // This request handler will check if configset exists. If not it will be created. 426 SolrQuery query = new SolrQuery(); 427 query.setRequestHandler("/admin/cores"); 428 query.setParam("action", "createConfigset"); 429 query.setParam("name", name); 430 431 QueryResponse response = _defaultSolrClient().query(query); 432 NamedList<?> results = response.getResponse(); 433 434 NamedList<?> error = (NamedList<?>) results.get("error"); 435 if (error != null) 436 { 437 throw new IOException("Error creating the core: " + error.get("msg")); 438 } 439 } 440 441 /** 442 * Delete a Solr core. 443 * @param name The name of the core to delete. 444 * @throws IOException If an I/O error occurs. 445 * @throws SolrServerException If a Solr error occurs. 446 */ 447 public void deleteCore(String name) throws IOException, SolrServerException 448 { 449 String fullName = _solrCorePrefix + name; 450 451 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 452 453 SolrQuery query = new SolrQuery(); 454 query.setRequestHandler("/admin/cores"); 455 query.setParam("action", "UNLOAD"); 456 query.setParam("core", fullName); 457 query.setParam("deleteInstanceDir", "true"); 458 459 QueryResponse response = _defaultSolrClient().query(query); 460 NamedList<?> results = response.getResponse(); 461 462 NamedList<?> error = (NamedList<?>) results.get("error"); 463 if (error != null) 464 { 465 throw new IOException("Error deleting core" + name + ": " + error.get("msg")); 466 } 467 } 468 469 /** 470 * Send the schema. 471 * @throws IOException If a communication error occurs. 472 * @throws SolrServerException If a solr error occurs. 473 */ 474 public void sendSchema() throws IOException, SolrServerException 475 { 476 getLogger().info("Computing and sending the schema to the solr server."); 477 478 String workspaceName = _workspaceSelector.getWorkspace(); 479 String collection = _solrClientProvider.getCollectionName(workspaceName); 480 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 481 482// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 483 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 484 485 // TODO Clear the schema except fields marked ametysReadOnly="true". 486 487 // Clear the current schema. 488 clearSchema(solrClient, collection); 489 490 SchemaRequest schemaRequest = new SchemaRequest(); 491 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 492 493 // The cleared schema contains only the basic fields which can't be deleted. 494 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 495 SchemaFields schemaFields = new SchemaFields(clearedSchema); 496 497 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 498 499 // Add the static schema types and fields. 500 List<SchemaRequest.Update> updates = new ArrayList<>(); 501 502 // Set "add field" definitions from the static schema to the update list. 503 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 504 505 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 506 507 // Set "add field" definitions from the static schema to the update list. 508 addCustomUpdates(updates, schemaFields); 509 510 updates.sort(new SchemaRequestComparator()); 511 512 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 513 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 514 515 getLogger().debug("Send schema response: {}", updateResponse.toString()); 516 Object errors = updateResponse.getResponse().get("errors"); 517 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 518 { 519 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 520 throw new SolrServerException(msg); 521 } 522 523 getLogger().info("Schema sent to the solr server."); 524 525 reloadCores(); 526 } 527 528 /** 529 * Compute the list of {@link Update} directives from the static schema. 530 * @param updates The list of {@link Update} directives to fill. 531 * @param staticSchema The static schema representation. 532 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 533 */ 534 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 535 { 536 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 537 for (FieldTypeDefinition fieldType : fieldTypes) 538 { 539 String name = (String) fieldType.getAttributes().get("name"); 540 if (!schemaFields.hasFieldType(name)) 541 { 542 updates.add(new SchemaRequest.AddFieldType(fieldType)); 543 schemaFields.addFieldType(name); 544 } 545 } 546 for (Map<String, Object> field : staticSchema.getFields()) 547 { 548 String name = (String) field.get("name"); 549 if (!schemaFields.hasField(name)) 550 { 551 updates.add(new SchemaRequest.AddField(field)); 552 schemaFields.addField(name); 553 } 554 } 555 for (Map<String, Object> field : staticSchema.getDynamicFields()) 556 { 557 String name = (String) field.get("name"); 558 if (!schemaFields.hasDynamicField(name)) 559 { 560 updates.add(new SchemaRequest.AddDynamicField(field)); 561 schemaFields.addDynamicField(name); 562 } 563 } 564 for (Map<String, Object> field : staticSchema.getCopyFields()) 565 { 566 String source = (String) field.get("source"); 567 String dest = (String) field.get("dest"); 568 if (!schemaFields.hasCopyField(source, dest)) 569 { 570 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 571 schemaFields.addCopyField(source, dest); 572 } 573 } 574 } 575 576 /** 577 * Compute the list of custom {@link Update} directives. 578 * @param updates The list of {@link Update} directives to fill. 579 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 580 */ 581 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 582 { 583 // Add all our property-managed fields. 584 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 585 { 586 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 587 588 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 589 { 590 if (!definition.exists(schemaFields)) 591 { 592 SchemaRequest.Update update = definition.getSchemaUpdate(); 593 if (update != null) 594 { 595 updates.add(update); 596 } 597 } 598 } 599 } 600 } 601 602 /** 603 * Delete all the fields of the existing schema in the given collection. 604 * @param solrClient The Solr client 605 * @param collection The collection. 606 * @throws IOException If a communication error occurs. 607 * @throws SolrServerException If a solr error occurs. 608 */ 609 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 610 { 611 try 612 { 613 getLogger().info("Clearing the existing schema on the solr server."); 614 615 SchemaRequest schemaRequest = new SchemaRequest(); 616 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 617 618 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 619 620 List<SchemaRequest.Update> deletions = new ArrayList<>(); 621 622 // First the copy fields, then dynamic and simple fields, and field types in the end. 623 for (Map<String, Object> field : schema.getCopyFields()) 624 { 625 String source = (String) field.get("source"); 626 String dest = (String) field.get("dest"); 627 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 628 } 629 for (Map<String, Object> field : schema.getDynamicFields()) 630 { 631 String name = (String) field.get("name"); 632 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 633 } 634 for (Map<String, Object> field : schema.getFields()) 635 { 636 String name = (String) field.get("name"); 637 if (!_READ_ONLY_FIELDS.contains(name)) 638 { 639 deletions.add(new SchemaRequest.DeleteField(name)); 640 } 641 } 642 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 643 { 644 String name = (String) fieldType.getAttributes().get("name"); 645 if (!_READ_ONLY_FIELDTYPES.contains(name)) 646 { 647 deletions.add(new SchemaRequest.DeleteFieldType(name)); 648 } 649 } 650 651 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 652 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 653 654 Object errors = updateResponse.getResponse().get("errors"); 655 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 656 { 657 String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString(); 658 throw new SolrServerException(msg); 659 } 660 else 661 { 662 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 663 } 664 665 getLogger().info("Solr schema cleared."); 666 } 667 catch (SolrServerException | IOException e) 668 { 669 getLogger().error("Error clearing schema in collection " + collection, e); 670 throw e; 671 } 672 } 673 674 /** 675 * Reload the solr cores. 676 * @throws IOException If a communication error occurs. 677 * @throws SolrServerException If a solr error occurs. 678 */ 679 protected void reloadCores() throws IOException, SolrServerException 680 { 681 getLogger().info("Reloading solr cores."); 682 683 for (String coreName : getCoreNames()) 684 { 685 String fullName = _solrCorePrefix + coreName; 686 687 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 688 689 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 690 } 691 692 getLogger().info("All cores reloaded."); 693 } 694 695 /** 696 * Reloads the ACL Solr cache for all users 697 * @throws IOException If an I/O error occurs. 698 * @throws SolrServerException If a Solr error occurs. 699 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 700 */ 701 public void reloadAclCache() throws IOException, SolrServerException, RepositoryException 702 { 703 String[] workspaceNames = _repository.getWorkspaces(); 704 for (String workspaceName : workspaceNames) 705 { 706 reloadAclCache(workspaceName); 707 } 708 } 709 710 /** 711 * Reloads the ACL Solr cache for all users 712 * @param workspaceName The workspace name 713 * @throws IOException If an I/O error occurs. 714 * @throws SolrServerException If a Solr error occurs. 715 */ 716 public void reloadAclCache(String workspaceName) throws IOException, SolrServerException 717 { 718 reloadAclCache(workspaceName, false); 719 } 720 721 /** 722 * Reloads the ACL Solr cache for all users 723 * @param workspaceName The workspace name 724 * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache) 725 * @throws IOException If an I/O error occurs. 726 * @throws SolrServerException If a Solr error occurs. 727 */ 728 public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException 729 { 730 getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 731 732 String collection = _solrClientProvider.getCollectionName(workspaceName); 733 SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection); 734 NamedList<Object> responseObj = responseBase.getResponse(); 735 736 if ("ok".equals(responseObj.get("result"))) 737 { 738 getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 739 } 740 else 741 { 742 Object error = responseObj.get("error"); 743 getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error); 744 } 745 } 746 747 /** 748 * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces. 749 * @param objects the {@link AmetysObject}s to update. 750 * @throws IOException If an I/O error occurs. 751 * @throws SolrServerException If a Solr error occurs. 752 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 753 */ 754 public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException 755 { 756 String[] workspaceNames = _repository.getWorkspaces(); 757 for (String workspaceName : workspaceNames) 758 { 759 updateAclCache(objects, workspaceName); 760 } 761 } 762 763 /** 764 * Updates the ACL Solr cache for some {@link AmetysObject}s. 765 * @param objects the {@link AmetysObject}s to update. 766 * @param workspaceName The workspace name 767 * @throws IOException If an I/O error occurs. 768 * @throws SolrServerException If a Solr error occurs. 769 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 770 */ 771 public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException 772 { 773 Map<String, Map<String, Object>> solrParams = new HashMap<>(); 774 775 for (AmetysObject object : objects) 776 { 777 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object); 778 779 solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(), 780 "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(), 781 "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 782 "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 783 "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()), 784 "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()))); 785 } 786 787 String collection = _solrClientProvider.getCollectionName(workspaceName); 788 SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection); 789 NamedList<Object> responseObj = response.getResponse(); 790 791 if ("ok".equals(responseObj.get("result"))) 792 { 793 if (getLogger().isInfoEnabled()) 794 { 795 getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet()); 796 } 797 } 798 else 799 { 800 @SuppressWarnings("unchecked") 801 List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects"); 802 getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects); 803 } 804 } 805 806 /** 807 * Index all the contents in a given workspace. 808 * @param workspaceName the workspace where to index 809 * @param indexAttachments to index content attachments 810 * @param solrClient The solr client to use 811 * @return The indexation result as a Map. 812 * @throws Exception if an error occurs while indexing. 813 */ 814 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 815 { 816 return indexAllContents(workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index all contents", getLogger())); 817 } 818 819 /** 820 * Index all the contents in a given workspace. 821 * @param workspaceName the workspace where to index 822 * @param indexAttachments to index content attachments 823 * @param solrClient The solr client to use 824 * @param progressionTracker The progression of the indexation 825 * @return The indexation result as a Map. 826 * @throws Exception if an error occurs while indexing. 827 */ 828 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 829 { 830 Request request = ContextHelper.getRequest(_context); 831 832 // Retrieve the current workspace. 833 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 834 835 try 836 { 837 // Force the workspace. 838 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 839 840 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 841 842 long start = System.currentTimeMillis(); 843 844 // Delete all contents 845 unindexAllContents(workspaceName, indexAttachments, solrClient); 846 847 String query = ContentQueryHelper.getContentXPathQuery(null); 848 AmetysObjectIterable<Content> contents = _resolver.query(query); 849 850 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient, progressionTracker); 851 852 long end = System.currentTimeMillis(); 853 854 if (!result.hasErrors()) 855 { 856 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 857 } 858 else 859 { 860 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 861 } 862 863 Map<String, Object> results = new HashMap<>(); 864 results.put("successCount", result.getSuccessCount()); 865 if (result.hasErrors()) 866 { 867 results.put("errorCount", result.getErrorCount()); 868 } 869 870 return results; 871 } 872 catch (Exception e) 873 { 874 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 875 getLogger().error(error, e); 876 throw new IndexingException(error, e); 877 } 878 finally 879 { 880 // Restore context 881 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 882 } 883 } 884 /** 885 * Unindex all content documents. 886 * @param workspaceName The workspace name 887 * @param unindexAttachments also unindex content attachments 888 * @param solrClient The solr client to use 889 * @throws Exception if an error occurs while unindexing. 890 */ 891 protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 892 { 893 String collection = _solrClientProvider.getCollectionName(workspaceName); 894 895 Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT); 896 Query query; 897 if (unindexAttachments) 898 { 899 Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE); 900 Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE); 901 query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents); 902 } 903 else 904 { 905 query = contents; 906 } 907 solrClient.deleteByQuery(collection, query.build()); 908 } 909 910 /** 911 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 912 * @param collectionId The id of collection 913 * @param indexAttachments to index content attachments 914 * @throws Exception if an error occurs while indexing. 915 */ 916 public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception 917 { 918 String[] workspaceNames = _repository.getWorkspaces(); 919 for (String workspaceName : workspaceNames) 920 { 921 indexSubcontents(collectionId, workspaceName, indexAttachments); 922 } 923 } 924 925 /** 926 * Index the child contents of a {@link AmetysObjectCollection} 927 * @param collectionId The id of collection 928 * @param workspaceName the workspace where to index 929 * @param indexAttachments to index content attachments 930 * @throws Exception if an error occurs while unindexing. 931 */ 932 public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception 933 { 934 Request request = ContextHelper.getRequest(_context); 935 936 // Retrieve the current workspace. 937 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 938 939 try 940 { 941 // Force the workspace. 942 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 943 944 if (_resolver.hasAmetysObjectForId(collectionId)) 945 { 946 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 947 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 948 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 949 950 for (AmetysObject child : children) 951 { 952 if (child instanceof Content) 953 { 954 Content content = (Content) child; 955 956 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 957 } 958 } 959 } 960 } 961 finally 962 { 963 // Restore context 964 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 965 } 966 } 967 968 /** 969 * Add or update a content into Solr index on all workspaces and commit 970 * @param contentId The id of the content to index 971 * @param indexAttachments to index content attachments 972 * @throws Exception if an error occurs while indexing. 973 */ 974 public void indexContent(String contentId, boolean indexAttachments) throws Exception 975 { 976 String[] workspaceNames = _repository.getWorkspaces(); 977 for (String workspaceName : workspaceNames) 978 { 979 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 980 indexContent(contentId, workspaceName, indexAttachments, solrClient); 981 } 982 } 983 984 /** 985 * Add or update a content into Solr index 986 * @param contentId The id of the content to index 987 * @param workspaceName the workspace where to index 988 * @param indexAttachments to index content attachments 989 * @throws Exception if an error occurs while indexing. 990 */ 991 public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception 992 { 993 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 994 indexContent(contentId, workspaceName, indexAttachments, solrClient); 995 } 996 997 /** 998 * Add or update a content into Solr index 999 * @param contentId The id of the content to index 1000 * @param workspaceName the workspace where to index 1001 * @param indexAttachments to index content attachments 1002 * @param solrClient The solr client to use 1003 * @throws Exception if an error occurs while indexing. 1004 */ 1005 public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1006 { 1007 Request request = ContextHelper.getRequest(_context); 1008 1009 // Retrieve the current workspace. 1010 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1011 1012 try 1013 { 1014 // Force the workspace. 1015 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1016 1017 if (_resolver.hasAmetysObjectForId(contentId)) 1018 { 1019 Content content = _resolver.resolveById(contentId); 1020 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1021 } 1022 } 1023 finally 1024 { 1025 // Restore context 1026 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1027 } 1028 } 1029 1030 private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException 1031 { 1032 try 1033 { 1034 long time_0 = System.currentTimeMillis(); 1035 1036 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 1037 1038 deleteRepeaterDocs(content.getId(), workspaceName, solrClient); 1039 doIndexContent(content, workspaceName, solrClient); 1040 doIndexContentWorkflow(content, workspaceName, solrClient); 1041 if (indexAttachments) 1042 { 1043 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1044 } 1045 1046 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 1047 } 1048 catch (Exception e) 1049 { 1050 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 1051 getLogger().error(error, e); 1052 throw new IndexingException(error, e); 1053 } 1054 } 1055 1056 /** 1057 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1058 * @param contents the collection of contents to index. 1059 * @return the indexation result. 1060 * @throws Exception if an error occurs while indexing. 1061 */ 1062 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 1063 { 1064 return indexContents(contents, ProgressionTrackerFactory.createContainerProgressionTracker("Index contents", getLogger())); 1065 } 1066 1067 1068 /** 1069 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1070 * @param contents the collection of contents to index. 1071 * @param progressionTracker The progression of the indexation 1072 * @return the indexation result. 1073 * @throws Exception if an error occurs while indexing. 1074 */ 1075 public IndexationResult indexContents(Iterable<Content> contents, ContainerProgressionTracker progressionTracker) throws Exception 1076 { 1077 1078 IndexationResult result = new IndexationResult(); 1079 String[] workspaceNames = _repository.getWorkspaces(); 1080 1081 for (String workspaceName : workspaceNames) 1082 { 1083 progressionTracker.addSimpleStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_WORKSPACE_STEP_LABEL", List.of(workspaceName))); 1084 } 1085 1086 for (String workspaceName : workspaceNames) 1087 { 1088 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1089 IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient, progressionTracker.getStep(workspaceName)); 1090 1091 result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount()); 1092 } 1093 1094 return result; 1095 } 1096 1097 /** 1098 * Send a collection of contents for indexation in the solr server. 1099 * @param contents the collection of contents to index. 1100 * @param workspaceName the workspace where to index 1101 * @param indexAttachments to index content attachments 1102 * @param solrClient The solr client to use 1103 * @return the indexation result. 1104 * @throws Exception if an error occurs while indexing. 1105 */ 1106 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1107 { 1108 return indexContents(contents, workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index contents for workspace " + workspaceName, getLogger())); 1109 } 1110 1111 /** 1112 * Send a collection of contents for indexation in the solr server. 1113 * @param contents the collection of contents to index. 1114 * @param workspaceName the workspace where to index 1115 * @param indexAttachments to index content attachments 1116 * @param solrClient The solr client to use 1117 * @param progressionTracker The progression of the indexation 1118 * @return the indexation result. 1119 * @throws Exception if an error occurs while indexing. 1120 */ 1121 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1122 { 1123 Request request = ContextHelper.getRequest(_context); 1124 1125 // Retrieve the current workspace. 1126 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1127 1128 try 1129 { 1130 // Force the workspace. 1131 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1132 1133 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 1134 1135 long start = System.currentTimeMillis(); 1136 1137 List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false) 1138 .map(Content::getId) 1139 .map(this::_resolveSilently) 1140 .filter(Objects::nonNull) 1141 .collect(Collectors.toList()); 1142 1143 IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient, progressionTracker); 1144 1145 long end = System.currentTimeMillis(); 1146 1147 if (!result.hasErrors()) 1148 { 1149 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1150 } 1151 else 1152 { 1153 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1154 } 1155 1156 return result; 1157 } 1158 catch (Exception e) 1159 { 1160 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 1161 getLogger().error(error, e); 1162 throw new IndexingException(error, e); 1163 } 1164 finally 1165 { 1166 // Restore context 1167 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1168 } 1169 } 1170 1171 private Content _resolveSilently(String contentId) 1172 { 1173 try 1174 { 1175 return _resolver.resolveById(contentId); 1176 } 1177 catch (UnknownAmetysObjectException e) 1178 { 1179 return null; 1180 } 1181 } 1182 1183 /** 1184 * Send some contents for indexation in the solr server. 1185 * @param contents the contents to index. 1186 * @param workspaceName The workspace name 1187 * @param indexAttachments to index content attachments 1188 * @param solrClient The solr client to use 1189 * @param progressionTracker The progression of the indexation 1190 * @return the indexation result. 1191 * @throws Exception if an error occurs committing the results. 1192 */ 1193 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1194 { 1195 int successCount = 0; 1196 int errorCount = 0; 1197 int numberOfContents = IterableUtils.size(contents); 1198 1199 progressionTracker.setSize(numberOfContents); 1200 1201 for (Content content : contents) 1202 { 1203 try 1204 { 1205 doIndexContent(content, workspaceName, solrClient); 1206 doIndexContentWorkflow(content, workspaceName, solrClient); 1207 if (indexAttachments) 1208 { 1209 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1210 } 1211 successCount++; 1212 } 1213 catch (Exception e) 1214 { 1215 getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e); 1216 errorCount++; 1217 } 1218 1219 progressionTracker.increment(); 1220 } 1221 1222 return new IndexationResult(successCount, errorCount); 1223 } 1224 1225 /** 1226 * Update the value of a specific system property in a content document. 1227 * @param content The content to update. 1228 * @param propertyId The system property ID. 1229 * @param workspaceName The workspace name 1230 * @throws Exception if an error occurs while indexing. 1231 */ 1232 public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception 1233 { 1234 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1235 updateSystemProperty(content, propertyId, workspaceName, solrClient); 1236 } 1237 1238 /** 1239 * Update the value of a specific system property in a content document. 1240 * @param content The content to update. 1241 * @param propertyId The system property ID. 1242 * @param workspaceName The workspace name 1243 * @param solrClient The solr client to use 1244 * @throws Exception if an error occurs while indexing. 1245 */ 1246 public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1247 { 1248 getLogger().debug("Updating the system property '{}' for content {} into Solr.", propertyId, content); 1249 1250 SolrInputDocument document = new SolrInputDocument(); 1251 boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 1252 1253 if (!hasUpdate) 1254 { 1255 getLogger().debug("Did not index '{}' system property for content {} in Solr because no update to apply.", propertyId, content); 1256 return; 1257 } 1258 1259 int status = _pushSolrDocument(document, workspaceName, solrClient); 1260 if (status != 0) 1261 { 1262 throw new IOException("Indexing of system property '" + propertyId + "': got status code '" + status + "'."); 1263 } 1264 1265 getLogger().debug("Succesfully indexed '{}' system property for content {} in Solr.", propertyId, content); 1266 } 1267 1268 /** 1269 * Update the value of a specific property in a content document. 1270 * @param content The content to update. 1271 * @param propertyId The system property ID. 1272 * @param workspaceName The workspace name 1273 * @throws Exception if an error occurs while indexing. 1274 */ 1275 public void updateProperty(Content content, String propertyId, String workspaceName) throws Exception 1276 { 1277 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1278 updateProperty(content, propertyId, workspaceName, solrClient); 1279 } 1280 1281 /** 1282 * Update the value of a specific property in a content document. 1283 * @param content The content to update. 1284 * @param propertyId The system property ID. 1285 * @param workspaceName The workspace name 1286 * @param solrClient The solr client to use 1287 * @throws Exception if an error occurs while indexing. 1288 */ 1289 public void updateProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1290 { 1291 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 1292 1293 SolrInputDocument document = new SolrInputDocument(); 1294 boolean hasUpdate = _solrContentIndexer.indexPartialProperty(content, propertyId, document); 1295 1296 if (!hasUpdate) 1297 { 1298 getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content); 1299 return; 1300 } 1301 1302 int status = _pushSolrDocument(document, workspaceName, solrClient); 1303 if (status != 0) 1304 { 1305 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 1306 } 1307 1308 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 1309 } 1310 1311 private int _pushSolrDocument(SolrInputDocument document, String workspaceName, SolrClient solrClient) throws Exception 1312 { 1313 String collection = _solrClientProvider.getCollectionName(workspaceName); 1314 UpdateResponse solrResponse = solrClient.add(collection, document); 1315 return solrResponse.getStatus(); 1316 } 1317 1318 /** 1319 * Remove a content from Solr index for all workspaces and commit 1320 * @param contentId The id of content to unindex 1321 * @param unindexAttachments also unindex content attachments 1322 * @throws Exception if an error occurs while indexing. 1323 */ 1324 public void unindexContent(String contentId, boolean unindexAttachments) throws Exception 1325 { 1326 String[] workspaceNames = _repository.getWorkspaces(); 1327 for (String workspaceName : workspaceNames) 1328 { 1329 unindexContent(contentId, workspaceName, unindexAttachments); 1330 } 1331 } 1332 1333 /** 1334 * Remove a content from Solr index 1335 * @param contentId The id of content to unindex 1336 * @param workspaceName The workspace where to work in 1337 * @param unindexAttachments also unindex content attachments 1338 * @throws Exception if an error occurs while indexing. 1339 */ 1340 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception 1341 { 1342 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1343 unindexContent(contentId, workspaceName, unindexAttachments, solrClient); 1344 } 1345 1346 /** 1347 * Remove a content from Solr index 1348 * @param contentId The id of content to unindex 1349 * @param workspaceName The workspace where to work in 1350 * @param unindexAttachments also unindex content attachments 1351 * @param solrClient The solr client to use 1352 * @throws Exception if an error occurs while indexing. 1353 */ 1354 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 1355 { 1356 Request request = ContextHelper.getRequest(_context); 1357 1358 // Retrieve the current workspace. 1359 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1360 1361 try 1362 { 1363 // Force the workspace. 1364 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1365 1366 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 1367 1368 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1369 doUnindexDocument(contentId, workspaceName, solrClient); 1370 if (unindexAttachments) 1371 { 1372 doUnindexContentAttachments(contentId, workspaceName, solrClient); 1373 } 1374 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1375 1376 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 1377 } 1378 catch (Exception e) 1379 { 1380 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 1381 getLogger().error(error, e); 1382 throw new IndexingException(error, e); 1383 } 1384 finally 1385 { 1386 // Restore workspace 1387 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1388 } 1389 } 1390 1391 /** 1392 * Remove a content from Solr index for all workspaces and commit 1393 * @param contentIds The id of content to unindex 1394 * @throws Exception if an error occurs while indexing. 1395 */ 1396 public void unindexContents(Collection<String> contentIds) throws Exception 1397 { 1398 String[] workspaceNames = _repository.getWorkspaces(); 1399 for (String workspaceName : workspaceNames) 1400 { 1401 unindexContents(contentIds, workspaceName); 1402 } 1403 } 1404 1405 /** 1406 * Remove a content from Solr index 1407 * @param contentIds The id of content to unindex 1408 * @param workspaceName The workspace where to work in 1409 * @throws Exception if an error occurs while indexing. 1410 */ 1411 public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception 1412 { 1413 Request request = ContextHelper.getRequest(_context); 1414 1415 // Retrieve the current workspace. 1416 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1417 1418 try 1419 { 1420 // Force the workspace. 1421 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1422 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1423 1424 getLogger().debug("Unindexing several contents from Solr."); 1425 1426 for (String contentId : contentIds) 1427 { 1428 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1429 doUnindexDocument(contentId, workspaceName, solrClient); 1430 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1431 } 1432 1433 getLogger().debug("Succesfully unindexed content from Solr."); 1434 } 1435 catch (Exception e) 1436 { 1437 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1438 getLogger().error(error, e); 1439 throw new IndexingException(error, e); 1440 } 1441 finally 1442 { 1443 // Restore workspace 1444 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1445 } 1446 } 1447 1448 /** 1449 * Add or update a content into Solr index 1450 * @param content The content to index 1451 * @param workspaceName The workspace where to index 1452 * @param solrClient The solr client to use 1453 * @throws Exception if an error occurs while indexing. 1454 */ 1455 protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception 1456 { 1457 long time_0 = System.currentTimeMillis(); 1458 1459 SolrInputDocument document = new SolrInputDocument(); 1460 List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document); 1461 1462 long time_1 = System.currentTimeMillis(); 1463 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1464 1465 indexAclInitValues(content, document); 1466 1467 long time_2 = System.currentTimeMillis(); 1468 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1469 1470 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1471 documents.add(document); 1472 1473 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents); 1474 int status = solrResponse.getStatus(); 1475 1476 long time_3 = System.currentTimeMillis(); 1477 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1478 1479 if (status != 0) 1480 { 1481 throw new IOException("Content indexation: got status code '" + status + "'."); 1482 } 1483 } 1484 1485 /** 1486 * Indexes read-ACl initial values for object 1487 * @param ametysObject The object 1488 * @param document The Solr document 1489 */ 1490 public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document) 1491 { 1492 // Indexation of AmetysObject property 1493 document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true); 1494 1495 // Indexation of initValues for AllowedUsers 1496 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject); 1497 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed()); 1498 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed()); 1499 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString); 1500 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString); 1501 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString); 1502 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString); 1503 } 1504 1505 private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier) 1506 { 1507 if (values == null) 1508 { 1509 return; 1510 } 1511 1512 for (T value : values) 1513 { 1514 document.addField(fieldName, stringifier.apply(value)); 1515 } 1516 } 1517 1518 /** 1519 * Index the whole workflow of a content. 1520 * @param content The content. 1521 * @param workspaceName The workspace name 1522 * @param solrClient The solr client to use 1523 * @throws Exception if an error occurs while indexing. 1524 */ 1525 protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception 1526 { 1527 if (content instanceof WorkflowAwareContent) 1528 { 1529 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient); 1530 } 1531 } 1532 1533 /** 1534 * Index content attachments as new entries in the idnex 1535 * @param collection the collection of attachments 1536 * @param content the content whose attachments will be indexed 1537 * @throws Exception if something goes wrong when indexing the attachments of the content 1538 */ 1539 public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception 1540 { 1541 Request request = ContextHelper.getRequest(_context); 1542 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1543 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1544 indexContentAttachments(collection, content, solrClient); 1545 } 1546 1547 /** 1548 * Index content attachments as new entries in the idnex 1549 * @param collection the collection of attachments 1550 * @param content the content whose attachments will be indexed 1551 * @param solrClient The solr client to use 1552 * @throws Exception if something goes wrong when indexing the attachments of the content 1553 */ 1554 public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception 1555 { 1556 if (collection == null) 1557 { 1558 return; 1559 } 1560 1561 try (AmetysObjectIterable<AmetysObject> children = collection.getChildren()) 1562 { 1563 for (AmetysObject object : children) 1564 { 1565 if (object instanceof ResourceCollection) 1566 { 1567 indexContentAttachments((ResourceCollection) object, content, solrClient); 1568 } 1569 else if (object instanceof Resource) 1570 { 1571 Resource resource = (Resource) object; 1572 indexContentAttachment(resource, content, solrClient); 1573 } 1574 } 1575 } 1576 } 1577 1578 /** 1579 * Index a content attachment 1580 * @param resource the content attachment as a {@link Resource} 1581 * @param content the content whose attachment is going to be indexed 1582 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1583 */ 1584 public void indexContentAttachment(Resource resource, Content content) throws Exception 1585 { 1586 Request request = ContextHelper.getRequest(_context); 1587 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1588 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1589 indexContentAttachment(resource, content, solrClient); 1590 } 1591 1592 /** 1593 * Index a content attachment 1594 * @param resource the content attachment as a {@link Resource} 1595 * @param content the content whose attachment is going to be indexed 1596 * @param solrClient The solr client to use 1597 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1598 */ 1599 public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception 1600 { 1601 SolrInputDocument document = new SolrInputDocument(); 1602 1603 // Prepare resource doc 1604 _indexContentAttachment(resource, document, content); 1605 1606 // Indexation of the document 1607 _indexResourceDocument(resource, document, solrClient); 1608 } 1609 1610 private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception 1611 { 1612 String language = content.getLanguage(); 1613 1614 _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language); 1615 1616 // Need the id of the content for unindexing attachment during the unindexing of the content 1617 document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId()); 1618 } 1619 1620 /** 1621 * Index a populated solr input document of type Resource. 1622 * @param resource the resource from which the input document is created 1623 * @param document the input document 1624 * @param solrClient The solr client to use 1625 * @throws SolrServerException if there is an error on the server 1626 * @throws IOException if there is a communication error with the server 1627 */ 1628 protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException 1629 { 1630 String resourceId = resource.getId(); 1631 1632 // Retrieve appropriate collection name 1633 Request request = ContextHelper.getRequest(_context); 1634 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1635 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 1636 1637 // Add document 1638 UpdateResponse solrResponse = solrClient.add(collectionName, document); 1639 int status = solrResponse.getStatus(); 1640 1641 if (status != 0) 1642 { 1643 throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId); 1644 } 1645 1646 getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId); 1647 } 1648 1649 /** 1650 * Delete repeater documents of a specified content. 1651 * @param workspaceName The workspace name 1652 * @param contentId the content ID. 1653 * @param solrClient The solr client to use 1654 * @throws Exception if an error occurs while indexing. 1655 */ 1656 protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception 1657 { 1658 long time_0 = System.currentTimeMillis(); 1659 1660 // _documentType:repeater AND id:content\://xxx/* 1661 StringBuilder query = new StringBuilder(); 1662 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1663 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1664 1665 solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1666 1667 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1668 } 1669 1670 /** 1671 * Index all the resources in a given workspace. 1672 * @param workspaceName The workspace where to index 1673 * @param solrClient The solr client to use 1674 * @param progressionTracker The progression of the indexation 1675 * @return The indexation result as a Map. 1676 * @throws Exception if an error occurs while indexing. 1677 */ 1678 public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1679 { 1680 Request request = ContextHelper.getRequest(_context); 1681 1682 // Retrieve the current workspace. 1683 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1684 1685 try 1686 { 1687 // Force the workspace. 1688 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1689 1690 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1691 1692 long start = System.currentTimeMillis(); 1693 1694 // Delete all resources 1695 _unindexAllResources(workspaceName, solrClient); 1696 1697 Map<String, Object> results = new HashMap<>(); 1698 1699 try 1700 { 1701 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1702 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1703 1704 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient, progressionTracker); 1705 1706 long end = System.currentTimeMillis(); 1707 1708 if (!result.hasErrors()) 1709 { 1710 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1711 } 1712 else 1713 { 1714 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1715 } 1716 1717 results.put("successCount", result.getSuccessCount()); 1718 if (result.hasErrors()) 1719 { 1720 results.put("errorCount", result.getErrorCount()); 1721 } 1722 } 1723 catch (UnknownAmetysObjectException e) 1724 { 1725 getLogger().info("There is no root for resources in current workspace."); 1726 progressionTracker.setSize(0); 1727 } 1728 1729 return results; 1730 } 1731 catch (Exception e) 1732 { 1733 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1734 getLogger().error(error, e); 1735 throw new IndexingException(error, e); 1736 } 1737 finally 1738 { 1739 // Restore context 1740 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1741 } 1742 } 1743 1744 /** 1745 * Send a collection of contents for indexation in the solr server. 1746 * @param resources the collection of contents to index. 1747 * @param documentType The document type of the resource 1748 * @param workspaceName The workspace where to index 1749 * @return the indexation result. 1750 * @throws Exception if an error occurs while indexing. 1751 */ 1752 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception 1753 { 1754 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1755 return indexResources(resources, documentType, null, workspaceName, solrClient); 1756 } 1757 1758 /** 1759 * Send a collection of contents for indexation in the solr server. 1760 * @param resources the collection of contents to index. 1761 * @param documentType The document type of the resource 1762 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1763 * @param workspaceName The workspace where to index 1764 * @param solrClient The solr client to use 1765 * @return the indexation result. 1766 * @throws Exception if an error occurs while indexing. 1767 */ 1768 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1769 { 1770 return indexResources(resources, documentType, resourceRoot, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resources", getLogger())); 1771 } 1772 1773 /** 1774 * Send a collection of contents for indexation in the solr server. 1775 * @param resources the collection of contents to index. 1776 * @param documentType The document type of the resource 1777 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1778 * @param workspaceName The workspace where to index 1779 * @param solrClient The solr client to use 1780 * @param progressionTracker The progression of the indexation 1781 * @return the indexation result. 1782 * @throws Exception if an error occurs while indexing. 1783 */ 1784 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1785 { 1786 Request request = ContextHelper.getRequest(_context); 1787 1788 // Retrieve the current workspace. 1789 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1790 1791 try 1792 { 1793 // Force the workspace. 1794 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1795 1796 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1797 1798 long start = System.currentTimeMillis(); 1799 1800 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient, progressionTracker); 1801 1802 long end = System.currentTimeMillis(); 1803 1804 if (!result.hasErrors()) 1805 { 1806 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1807 } 1808 else 1809 { 1810 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1811 } 1812 1813 return result; 1814 } 1815 catch (Exception e) 1816 { 1817 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1818 getLogger().error(error, e); 1819 throw new IndexingException(error, e); 1820 } 1821 finally 1822 { 1823 // Restore context 1824 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1825 } 1826 } 1827 1828 /** 1829 * Send some resources for indexation in the solr server. 1830 * @param resources the resources to index. 1831 * @param documentType The document type of the resource 1832 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1833 * @param workspaceName The workspace where to index 1834 * @param solrClient The solr client to use 1835 * @param progressionTracker The progression of the indexation 1836 * @return the indexation result. 1837 * @throws Exception if an error occurs committing the results. 1838 */ 1839 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1840 { 1841 int successCount = 0; 1842 int errorCount = 0; 1843 int numberOfResources = IterableUtils.size(resources); 1844 1845 progressionTracker.setSize(numberOfResources); 1846 1847 for (AmetysObject resource : resources) 1848 { 1849 try 1850 { 1851 doIndexExplorerItem(resource, documentType, resourceRoot, solrClient); 1852 successCount++; 1853 } 1854 catch (Exception e) 1855 { 1856 getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e); 1857 errorCount++; 1858 } 1859 1860 progressionTracker.increment(); 1861 } 1862 1863 return new IndexationResult(successCount, errorCount); 1864 } 1865 1866 /** 1867 * Add or update a resource into Solr index 1868 * @param resource The resource to index 1869 * @param documentType The document type of the resource 1870 * @param workspaceName The workspace where to index 1871 * @throws Exception if an error occurs while indexing. 1872 */ 1873 public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception 1874 { 1875 Request request = ContextHelper.getRequest(_context); 1876 1877 // Retrieve the current workspace. 1878 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1879 1880 try 1881 { 1882 // Force the workspace. 1883 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1884 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1885 1886 getLogger().debug("Indexing resource {} into Solr.", resource.getId()); 1887 1888 doIndexResource(resource, documentType, null, solrClient); 1889 1890 getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId()); 1891 } 1892 catch (Exception e) 1893 { 1894 String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName); 1895 getLogger().error(error, e); 1896 throw new IndexingException(error, e); 1897 } 1898 finally 1899 { 1900 // Restore context 1901 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1902 } 1903 } 1904 1905 private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1906 { 1907 getLogger().debug("Unindexing all resources from Solr."); 1908 1909 String collection = _solrClientProvider.getCollectionName(workspaceName); 1910 solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1911 1912 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1913 } 1914 1915 /** 1916 * Delete all resource documents at a given path for all workspaces and commit 1917 * @param rootId The resource root ID, must not be null. 1918 * @param path The resource path relative to the given root, must start with a slash. 1919 * @throws Exception If an error occurs while unindexing. 1920 */ 1921 public void unindexResourcesByPath(String rootId, String path) throws Exception 1922 { 1923 String[] workspaceNames = _repository.getWorkspaces(); 1924 for (String workspaceName : workspaceNames) 1925 { 1926 unindexResourcesByPath(rootId, path, workspaceName); 1927 } 1928 } 1929 1930 /** 1931 * Delete all resource documents at a given path. 1932 * @param rootId The resource root ID, must not be null. 1933 * @param path The resource path relative to the given root, must start with a slash. 1934 * @param workspaceName The workspace where to work in 1935 * @throws Exception If an error occurs while unindexing. 1936 */ 1937 public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception 1938 { 1939 Request request = ContextHelper.getRequest(_context); 1940 1941 // Retrieve the current workspace. 1942 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1943 1944 try 1945 { 1946 // Force the workspace. 1947 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1948 1949 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1950 1951 Query query = new ResourceLocationQuery(rootId, path); 1952 1953 String collection = _solrClientProvider.getCollectionName(workspaceName); 1954 _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1955 1956 getLogger().debug("Succesfully deleted resource document from Solr."); 1957 } 1958 catch (Exception e) 1959 { 1960 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1961 getLogger().error(error, e); 1962 throw new IndexingException(error, e); 1963 } 1964 finally 1965 { 1966 // Restore workspace 1967 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1968 } 1969 } 1970 1971 /** 1972 * Remove a resource from Solr index for all workspaces and commit 1973 * @param resourceId The id of resource to unindex 1974 * @throws Exception if an error occurs while indexing. 1975 */ 1976 public void unindexResource(String resourceId) throws Exception 1977 { 1978 String[] workspaceNames = _repository.getWorkspaces(); 1979 for (String workspaceName : workspaceNames) 1980 { 1981 unindexResource(resourceId, workspaceName); 1982 } 1983 } 1984 1985 /** 1986 * Remove a resource from the Solr index. 1987 * @param resourceId The id of resource to unindex 1988 * @param workspaceName The workspace where to work in 1989 * @throws Exception if an error occurs while unindexing. 1990 */ 1991 public void unindexResource(String resourceId, String workspaceName) throws Exception 1992 { 1993 Request request = ContextHelper.getRequest(_context); 1994 1995 // Retrieve the current workspace. 1996 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1997 1998 try 1999 { 2000 // Force the workspace. 2001 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 2002 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 2003 2004 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 2005 2006 doUnindexDocument(resourceId, workspaceName, solrClient); 2007 2008 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 2009 } 2010 catch (Exception e) 2011 { 2012 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 2013 getLogger().error(error, e); 2014 throw new IndexingException(error, e); 2015 } 2016 finally 2017 { 2018 // Restore workspace 2019 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 2020 } 2021 } 2022 2023 /** 2024 * Add or update a resource into Solr index 2025 * @param node The resource to index 2026 * @param documentType The document type of the resource 2027 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 2028 * @param solrClient The solr client to use 2029 * @throws Exception if an error occurs while indexing. 2030 */ 2031 protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2032 { 2033 if (node instanceof ResourceCollection) 2034 { 2035 try (AmetysObjectIterable<AmetysObject> children = ((ResourceCollection) node).getChildren()) 2036 { 2037 for (AmetysObject child : children) 2038 { 2039 doIndexExplorerItem(child, documentType, resourceRoot, solrClient); 2040 } 2041 } 2042 } 2043 else if (node instanceof Resource) 2044 { 2045 doIndexResource((Resource) node, documentType, resourceRoot, solrClient); 2046 } 2047 } 2048 2049 /** 2050 * Add or update a resource into Solr index 2051 * @param resource The resource to index 2052 * @param documentType The document type of the resource 2053 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 2054 * @param solrClient The solr client to use 2055 * @throws Exception if an error occurs while indexing. 2056 */ 2057 protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2058 { 2059 SolrInputDocument document = new SolrInputDocument(); 2060 2061 _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot); 2062 2063 String workspaceName = _workspaceSelector.getWorkspace(); 2064 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document); 2065 int status = solrResponse.getStatus(); 2066 2067 if (status != 0) 2068 { 2069 throw new IOException("Resource indexation: got status code '" + status + "'."); 2070 } 2071 } 2072 2073 /** 2074 * Process a Solr commit operation in all workspaces. 2075 * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient} 2076 * @throws SolrServerException if there is an error on the server 2077 * @throws IOException if there is a communication error with the server 2078 */ 2079 public void commit() throws SolrServerException, IOException 2080 { 2081 String[] workspaceNames; 2082 try 2083 { 2084 workspaceNames = _repository.getWorkspaces(); 2085 for (String workspaceName : workspaceNames) 2086 { 2087 SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName); 2088 commit(workspaceName, solrClient); 2089 } 2090 } 2091 catch (RepositoryException e) 2092 { 2093 throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e); 2094 } 2095 } 2096 2097 /** 2098 * Process a Solr commit operation in given workspace. 2099 * @param workspaceName The workspace's name 2100 * @param solrClient The solr client to use 2101 * @throws SolrServerException if there is an error on the server 2102 * @throws IOException if there is a communication error with the server 2103 */ 2104 public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2105 { 2106 long time_0 = System.currentTimeMillis(); 2107 2108 // Commit 2109 UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName)); 2110 int status = solrResponse.getStatus(); 2111 2112 if (status != 0) 2113 { 2114 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2115 } 2116 2117 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 2118 } 2119 2120 /** 2121 * Launch a solr index optimization. 2122 * @param workspaceName The workspace's name 2123 * @param solrClient The solr client to use 2124 * @throws SolrServerException if there is an error on the server 2125 * @throws IOException if there is a communication error with the server 2126 */ 2127 public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2128 { 2129 UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName)); 2130 int status = solrResponse.getStatus(); 2131 2132 if (status != 0) 2133 { 2134 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2135 } 2136 2137 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 2138 } 2139 2140 /** 2141 * Delete all documents from the solr index. 2142 * @param workspaceName The workspace name 2143 * @param solrClient The solr client to use 2144 * @throws Exception if an error occurs while unindexing. 2145 */ 2146 public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception 2147 { 2148 getLogger().debug("Deleting all documents from Solr."); 2149 2150 String collection = _solrClientProvider.getCollectionName(workspaceName); 2151 solrClient.deleteByQuery(collection, "*:*"); 2152 2153 getLogger().debug("Successfully deleted all documents from Solr."); 2154 } 2155 2156 /** 2157 * Delete a document from the Solr server. 2158 * @param id The id of the document to delete from Solr 2159 * @param workspaceName The workspace name 2160 * @param solrClient The solr client to use 2161 * @throws Exception if an error occurs while indexing. 2162 */ 2163 protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception 2164 { 2165 UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 2166 int status = solrResponse.getStatus(); 2167 2168 if (status != 0) 2169 { 2170 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 2171 } 2172 } 2173 2174 /** 2175 * Delete content attachments documents of a given content from the Solr server. 2176 * @param contentId The id of the content 2177 * @param workspaceName The workspace name 2178 * @param solrClient The solr client to use 2179 * @throws Exception if an error occurs while indexing. 2180 */ 2181 protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception 2182 { 2183 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 2184 2185 Query query = new ContentAttachmentQuery(contentId); 2186 UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build()); 2187 int status = solrResponse.getStatus(); 2188 2189 if (status != 0) 2190 { 2191 throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'."); 2192 } 2193 } 2194 2195// /** 2196// * Get the collection to use. 2197// * @return The name of the collection to index into. 2198// */ 2199// protected String getCollection() 2200// { 2201// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 2202// } 2203 2204 class IndexationResult 2205 { 2206 protected int _successCount; 2207 2208 protected int _errorCount; 2209 2210 /** 2211 * Constructor 2212 */ 2213 public IndexationResult() 2214 { 2215 this(0, 0); 2216 } 2217 2218 /** 2219 * Constructor 2220 * @param successCount The success count. 2221 * @param errorCount The error count. 2222 */ 2223 public IndexationResult(int successCount, int errorCount) 2224 { 2225 this._successCount = successCount; 2226 this._errorCount = errorCount; 2227 } 2228 2229 /** 2230 * Get the successCount. 2231 * @return the successCount 2232 */ 2233 public int getSuccessCount() 2234 { 2235 return _successCount; 2236 } 2237 2238 /** 2239 * Set the successCount. 2240 * @param successCount the successCount to set 2241 */ 2242 public void setSuccessCount(int successCount) 2243 { 2244 this._successCount = successCount; 2245 } 2246 2247 /** 2248 * Test if the indexation had errors. 2249 * @return true if the indexation had errors, false otherwise. 2250 */ 2251 public boolean hasErrors() 2252 { 2253 return _errorCount > 0; 2254 } 2255 2256 /** 2257 * Get the errorCount. 2258 * @return the errorCount 2259 */ 2260 public int getErrorCount() 2261 { 2262 return _errorCount; 2263 } 2264 2265 /** 2266 * Set the errorCount. 2267 * @param errorCount the errorCount to set 2268 */ 2269 public void setErrorCount(int errorCount) 2270 { 2271 this._errorCount = errorCount; 2272 } 2273 } 2274 2275 private static class SchemaRequestComparator implements Comparator<SchemaRequest.Update> 2276 { 2277 public int compare(SchemaRequest.Update u1, SchemaRequest.Update u2) 2278 { 2279 return Integer.compare(_getOrder(u1), _getOrder(u2)); 2280 } 2281 2282 private int _getOrder(SchemaRequest.Update update) 2283 { 2284 // Field types are needed first to define fields 2285 if (update instanceof SchemaRequest.AddFieldType) 2286 { 2287 return 1; 2288 } 2289 2290 // Fields or dynamic fields can be defined at the same time 2291 if (update instanceof SchemaRequest.AddField || update instanceof SchemaRequest.AddDynamicField) 2292 { 2293 return 2; 2294 } 2295 2296 // Copy fields can be based on fields or dynamic fields 2297 if (update instanceof SchemaRequest.AddCopyField) 2298 { 2299 return 3; 2300 } 2301 2302 return 0; 2303 } 2304 } 2305}