001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.nio.ByteBuffer; 020import java.nio.CharBuffer; 021import java.nio.charset.CharsetDecoder; 022import java.nio.charset.CodingErrorAction; 023import java.nio.charset.StandardCharsets; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Collection; 028import java.util.Comparator; 029import java.util.HashMap; 030import java.util.HashSet; 031import java.util.List; 032import java.util.Map; 033import java.util.Objects; 034import java.util.Set; 035import java.util.function.Function; 036import java.util.stream.Collectors; 037import java.util.stream.StreamSupport; 038 039import javax.jcr.RepositoryException; 040 041import org.apache.avalon.framework.activity.Initializable; 042import org.apache.avalon.framework.component.Component; 043import org.apache.avalon.framework.context.Context; 044import org.apache.avalon.framework.context.ContextException; 045import org.apache.avalon.framework.context.Contextualizable; 046import org.apache.avalon.framework.service.ServiceException; 047import org.apache.avalon.framework.service.ServiceManager; 048import org.apache.avalon.framework.service.Serviceable; 049import org.apache.cocoon.components.ContextHelper; 050import org.apache.cocoon.environment.Request; 051import org.apache.commons.collections4.IterableUtils; 052import org.apache.commons.lang3.ObjectUtils; 053import org.apache.commons.lang3.StringUtils; 054import org.apache.solr.client.solrj.SolrClient; 055import org.apache.solr.client.solrj.SolrQuery; 056import org.apache.solr.client.solrj.SolrResponse; 057import org.apache.solr.client.solrj.SolrServerException; 058import org.apache.solr.client.solrj.request.CoreAdminRequest; 059import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 060import org.apache.solr.client.solrj.request.schema.SchemaRequest; 061import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 062import org.apache.solr.client.solrj.response.CoreAdminResponse; 063import org.apache.solr.client.solrj.response.QueryResponse; 064import org.apache.solr.client.solrj.response.SolrResponseBase; 065import org.apache.solr.client.solrj.response.UpdateResponse; 066import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 067import org.apache.solr.client.solrj.response.schema.SchemaResponse; 068import org.apache.solr.client.solrj.util.ClientUtils; 069import org.apache.solr.common.SolrInputDocument; 070import org.apache.solr.common.util.NamedList; 071import org.slf4j.Logger; 072 073import org.ametys.cms.indexing.IndexingException; 074import org.ametys.cms.indexing.solr.ReloadAclCacheRequest; 075import org.ametys.cms.indexing.solr.UpdateAclCacheRequest; 076import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest; 077import org.ametys.cms.repository.Content; 078import org.ametys.cms.repository.ContentQueryHelper; 079import org.ametys.cms.repository.WorkflowAwareContent; 080import org.ametys.cms.rights.solrchecking.ReadAccessHelper; 081import org.ametys.cms.search.query.ContentAttachmentQuery; 082import org.ametys.cms.search.query.DocumentTypeQuery; 083import org.ametys.cms.search.query.OrQuery; 084import org.ametys.cms.search.query.Query; 085import org.ametys.cms.search.query.ResourceLocationQuery; 086import org.ametys.cms.search.solr.NoAutoCommitUpdateClient; 087import org.ametys.cms.search.solr.SolrClientProvider; 088import org.ametys.cms.search.solr.schema.SchemaDefinition; 089import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 090import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 091import org.ametys.cms.search.solr.schema.SchemaFields; 092import org.ametys.cms.search.solr.schema.SchemaHelper; 093import org.ametys.core.group.GroupIdentity; 094import org.ametys.core.right.AllowedUsers; 095import org.ametys.core.schedule.progression.ContainerProgressionTracker; 096import org.ametys.core.schedule.progression.ProgressionTrackerFactory; 097import org.ametys.core.schedule.progression.SimpleProgressionTracker; 098import org.ametys.core.user.UserIdentity; 099import org.ametys.core.util.DateUtils; 100import org.ametys.plugins.explorer.resources.Resource; 101import org.ametys.plugins.explorer.resources.ResourceCollection; 102import org.ametys.plugins.repository.AmetysObject; 103import org.ametys.plugins.repository.AmetysObjectIterable; 104import org.ametys.plugins.repository.AmetysObjectResolver; 105import org.ametys.plugins.repository.RepositoryConstants; 106import org.ametys.plugins.repository.TraversableAmetysObject; 107import org.ametys.plugins.repository.UnknownAmetysObjectException; 108import org.ametys.plugins.repository.collection.AmetysObjectCollection; 109import org.ametys.plugins.repository.provider.AbstractRepository; 110import org.ametys.plugins.repository.provider.JackrabbitRepository; 111import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector; 112import org.ametys.plugins.repository.provider.WorkspaceSelector; 113import org.ametys.runtime.config.Config; 114import org.ametys.runtime.i18n.I18nizableText; 115import org.ametys.runtime.plugin.component.AbstractLogEnabled; 116 117/** 118 * Solr indexer. 119 */ 120public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 121{ 122 /** The component role. */ 123 public static final String ROLE = SolrIndexer.class.getName(); 124 125 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 126 127 private static final String _CONFIGSET_NAME_PREFIX = "configset-"; 128 129 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_"); 130 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general"); 131 132 private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766; 133 134 /** The ametys object resolver. */ 135 protected AmetysObjectResolver _resolver; 136 /** The schema definition provider extension point. */ 137 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 138 /** The schema helper. */ 139 protected SchemaHelper _schemaHelper; 140 /** Solr Ametys contents indexer */ 141 protected SolrContentIndexer _solrContentIndexer; 142 /** Solr workflow indexer. */ 143 protected SolrWorkflowIndexer _solrWorkflowIndexer; 144 /** Solr resource indexer. */ 145 protected SolrResourceIndexer _solrResourceIndexer; 146 147 /** The Solr client provider */ 148 protected SolrClientProvider _solrClientProvider; 149 150 /** The solr core prefix. */ 151 protected String _solrCorePrefix; 152 /** The Ametys internal URL used by Solr to query Ametys */ 153 protected String _ametysInternalUrl; 154 155 /** The workspace selector. */ 156 protected WorkspaceSelector _workspaceSelector; 157 /** The JCR repository */ 158 protected JackrabbitRepository _repository; 159 /** The helper for read access */ 160 protected ReadAccessHelper _readAccessHelper; 161 162 /** The avalon context */ 163 protected Context _context; 164 165 /** 166 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 167 * @return The date format for indexing dates 168 * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead 169 */ 170 @Deprecated 171 public static SimpleDateFormat dateFormat() 172 { 173 if (__DATE_FORMAT.get() == null) 174 { 175 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 176 } 177 return __DATE_FORMAT.get(); 178 } 179 180 /** 181 * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr. 182 * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept. 183 * @param value The string value to index 184 * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log. 185 * @param documentId The id of the document being indexed. Can be null if you do not want to log. 186 * @param fieldName The name of the field being indexed. Can be null if you do not want to log. 187 * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes) 188 */ 189 public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName) 190 { 191 if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT) 192 { 193 // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length()) 194 // As a result, value.getBytes().length <= limit 195 return value; 196 } 197 198 // There is a doubt, the string may need to be truncated (or not) 199 byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); 200 int bytesLength = valueBytes.length; 201 if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT) 202 { 203 return value; 204 } 205 206 if (ObjectUtils.allNotNull(logger, documentId, fieldName)) 207 { 208 logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT); 209 } 210 211 // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974) 212 CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT); 213 CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() 214 .onMalformedInput(CodingErrorAction.IGNORE); 215 decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true); 216 decoder.flush(charBuffer); 217 return new String(charBuffer.array(), 0, charBuffer.position()); 218 } 219 220 @Override 221 public void service(ServiceManager serviceManager) throws ServiceException 222 { 223 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 224 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 225 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 226 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 227 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 228 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 229 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 230 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 231 _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE); 232 _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE); 233 } 234 235 @Override 236 public void initialize() throws Exception 237 { 238 Config config = Config.getInstance(); 239 _solrCorePrefix = config.getValue("cms.solr.core.prefix"); 240 241 _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url"); 242 if (StringUtils.isBlank(_ametysInternalUrl)) 243 { 244 // fallback to CMS URL as internal URL is an optional parameter 245 _ametysInternalUrl = config.getValue("cms.url"); 246 } 247 } 248 249 @Override 250 public void contextualize(Context context) throws ContextException 251 { 252 _context = context; 253 } 254 255 /** 256 * Gets the 'autocommit' Solr client 257 * @param workspaceName The name of the workspace 258 * @return the Solr client 259 */ 260 protected SolrClient _getAutoCommitSolrClient(String workspaceName) 261 { 262 return _solrClientProvider.getUpdateClient(workspaceName, true); 263 } 264 265 /** 266 * Gets the 'no autocommit' Solr client 267 * @param workspaceName The name of the workspace 268 * @return the Solr client 269 */ 270 protected SolrClient _getNoAutoCommitSolrClient(String workspaceName) 271 { 272 return _solrClientProvider.getUpdateClient(workspaceName, false); 273 } 274 275 // for admin operations 276 private SolrClient _defaultSolrClient() 277 { 278 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 279 } 280 281 /** 282 * Get the names of the Solr cores. 283 * @return The names of the Solr cores. 284 * @throws IOException If an I/O error occurs. 285 * @throws SolrServerException If a Solr error occurs. 286 */ 287 @SuppressWarnings("unchecked") 288 public Set<String> getCoreNames() throws IOException, SolrServerException 289 { 290 Set<String> coreNames = new HashSet<>(); 291 292 getLogger().debug("Getting core list."); 293 294 SolrQuery query = new SolrQuery(); 295 query.setRequestHandler("/admin/cores"); 296 query.setParam("action", "STATUS"); 297 298 QueryResponse response = _defaultSolrClient().query(query); 299 300 NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status"); 301 for (Map.Entry<String, NamedList<?>> core : status) 302 { 303 String fullName = (String) core.getValue().get("name"); 304 if (fullName.startsWith(_solrCorePrefix)) 305 { 306 coreNames.add(fullName.substring(_solrCorePrefix.length())); 307 } 308 } 309 310 return coreNames; 311 } 312 313 /** 314 * Get the names of the Solr cores. 315 * @return The names of the Solr cores. 316 * @throws IOException If an I/O error occurs. 317 * @throws SolrServerException If a Solr error occurs. 318 */ 319 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 320 { 321 Set<String> coreNames = new HashSet<>(); 322 323 getLogger().debug("Getting core list."); 324 325 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 326 327 NamedList<NamedList<Object>> status = response.getCoreStatus(); 328 for (Map.Entry<String, NamedList<Object>> core : status) 329 { 330 String fullName = (String) core.getValue().get("name"); 331 if (fullName.startsWith(_solrCorePrefix)) 332 { 333 coreNames.add(fullName.substring(_solrCorePrefix.length())); 334 } 335 } 336 337 return coreNames; 338 } 339 340 /** 341 * Create a Solr core. 342 * @param name The name of the core to create. 343 * @throws IOException If an I/O error occurs. 344 * @throws SolrServerException If a Solr error occurs. 345 */ 346 public void createCore(String name) throws IOException, SolrServerException 347 { 348 Set<String> cores = getCoreNames(); 349 if (!cores.contains(name)) 350 { 351 String fullName = _solrCorePrefix + name; 352 String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix); 353 _createConfigset(configsetName); 354 355 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 356 357 SolrQuery query = new SolrQuery(); 358 query.setRequestHandler("/admin/cores"); 359 query.setParam("action", "CREATE"); 360 query.setParam("name", fullName); 361 query.setParam("configSet", configsetName); 362 query.setParam("property.ametys.url", _ametysInternalUrl); 363 364 QueryResponse response = _defaultSolrClient().query(query); 365 NamedList<?> results = response.getResponse(); 366 367 NamedList<?> error = (NamedList<?>) results.get("error"); 368 if (error != null) 369 { 370 throw new IOException("Error creating the core: " + error.get("msg")); 371 } 372 } 373 else 374 { 375 if (getLogger().isDebugEnabled()) 376 { 377 getLogger().debug("Core '" + name + "' already exists, skipping it."); 378 } 379 } 380 } 381 382 /** 383 * Updates the ametys.url property of the Solr cores. 384 */ 385 public void updateAmetysUrlCoreProperty() 386 { 387 Set<String> coreNames; 388 try 389 { 390 coreNames = getCoreNames(); 391 } 392 catch (SolrServerException | IOException e) 393 { 394 getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e); 395 return; 396 } 397 398 for (String coreName : coreNames) 399 { 400 String collection = _solrClientProvider.getCollectionName(coreName); 401 SolrResponseBase response; 402 try 403 { 404 response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection); 405 } 406 catch (SolrServerException | IOException e) 407 { 408 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e); 409 continue; 410 } 411 412 NamedList<Object> responseParams = response.getResponse(); 413 if ("ok".equals(responseParams.get("result"))) 414 { 415 Boolean valueChanged = responseParams.getBooleanArg("valueChanged"); 416 if (valueChanged) 417 { 418 getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName); 419 } 420 else 421 { 422 getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName); 423 } 424 } 425 else 426 { 427 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName); 428 } 429 } 430 } 431 432 private void _createConfigset(String name) throws IOException, SolrServerException 433 { 434 getLogger().info("Creating (if necessary) configset '{}'", name); 435 436 // This request handler will check if configset exists. If not it will be created. 437 SolrQuery query = new SolrQuery(); 438 query.setRequestHandler("/admin/cores"); 439 query.setParam("action", "createConfigset"); 440 query.setParam("name", name); 441 442 QueryResponse response = _defaultSolrClient().query(query); 443 NamedList<?> results = response.getResponse(); 444 445 NamedList<?> error = (NamedList<?>) results.get("error"); 446 if (error != null) 447 { 448 throw new IOException("Error creating the core: " + error.get("msg")); 449 } 450 } 451 452 /** 453 * Delete a Solr core. 454 * @param name The name of the core to delete. 455 * @throws IOException If an I/O error occurs. 456 * @throws SolrServerException If a Solr error occurs. 457 */ 458 public void deleteCore(String name) throws IOException, SolrServerException 459 { 460 String fullName = _solrCorePrefix + name; 461 462 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 463 464 SolrQuery query = new SolrQuery(); 465 query.setRequestHandler("/admin/cores"); 466 query.setParam("action", "UNLOAD"); 467 query.setParam("core", fullName); 468 query.setParam("deleteInstanceDir", "true"); 469 470 QueryResponse response = _defaultSolrClient().query(query); 471 NamedList<?> results = response.getResponse(); 472 473 NamedList<?> error = (NamedList<?>) results.get("error"); 474 if (error != null) 475 { 476 throw new IOException("Error deleting core" + name + ": " + error.get("msg")); 477 } 478 } 479 480 /** 481 * Send the schema. 482 * @throws IOException If a communication error occurs. 483 * @throws SolrServerException If a solr error occurs. 484 */ 485 public void sendSchema() throws IOException, SolrServerException 486 { 487 getLogger().info("Computing and sending the schema to the solr server."); 488 489 String workspaceName = _workspaceSelector.getWorkspace(); 490 String collection = _solrClientProvider.getCollectionName(workspaceName); 491 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 492 493// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 494 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 495 496 // TODO Clear the schema except fields marked ametysReadOnly="true". 497 498 // Clear the current schema. 499 clearSchema(solrClient, collection); 500 501 SchemaRequest schemaRequest = new SchemaRequest(); 502 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 503 504 // The cleared schema contains only the basic fields which can't be deleted. 505 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 506 SchemaFields schemaFields = new SchemaFields(clearedSchema); 507 508 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 509 510 // Add the static schema types and fields. 511 List<SchemaRequest.Update> updates = new ArrayList<>(); 512 513 // Set "add field" definitions from the static schema to the update list. 514 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 515 516 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 517 518 // Set "add field" definitions from the static schema to the update list. 519 addCustomUpdates(updates, schemaFields); 520 521 updates.sort(new SchemaRequestComparator()); 522 523 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 524 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 525 526 getLogger().debug("Send schema response: {}", updateResponse.toString()); 527 Object errors = updateResponse.getResponse().get("errors"); 528 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 529 { 530 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 531 throw new SolrServerException(msg); 532 } 533 534 getLogger().info("Schema sent to the solr server."); 535 536 reloadCores(); 537 } 538 539 /** 540 * Compute the list of {@link Update} directives from the static schema. 541 * @param updates The list of {@link Update} directives to fill. 542 * @param staticSchema The static schema representation. 543 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 544 */ 545 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 546 { 547 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 548 for (FieldTypeDefinition fieldType : fieldTypes) 549 { 550 String name = (String) fieldType.getAttributes().get("name"); 551 if (!schemaFields.hasFieldType(name)) 552 { 553 updates.add(new SchemaRequest.AddFieldType(fieldType)); 554 schemaFields.addFieldType(name); 555 } 556 } 557 for (Map<String, Object> field : staticSchema.getFields()) 558 { 559 String name = (String) field.get("name"); 560 if (!schemaFields.hasField(name)) 561 { 562 updates.add(new SchemaRequest.AddField(field)); 563 schemaFields.addField(name); 564 } 565 } 566 for (Map<String, Object> field : staticSchema.getDynamicFields()) 567 { 568 String name = (String) field.get("name"); 569 if (!schemaFields.hasDynamicField(name)) 570 { 571 updates.add(new SchemaRequest.AddDynamicField(field)); 572 schemaFields.addDynamicField(name); 573 } 574 } 575 for (Map<String, Object> field : staticSchema.getCopyFields()) 576 { 577 String source = (String) field.get("source"); 578 String dest = (String) field.get("dest"); 579 if (!schemaFields.hasCopyField(source, dest)) 580 { 581 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 582 schemaFields.addCopyField(source, dest); 583 } 584 } 585 } 586 587 /** 588 * Compute the list of custom {@link Update} directives. 589 * @param updates The list of {@link Update} directives to fill. 590 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 591 */ 592 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 593 { 594 // Add all our property-managed fields. 595 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 596 { 597 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 598 599 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 600 { 601 if (!definition.exists(schemaFields)) 602 { 603 SchemaRequest.Update update = definition.getSchemaUpdate(); 604 if (update != null) 605 { 606 updates.add(update); 607 } 608 } 609 } 610 } 611 } 612 613 /** 614 * Delete all the fields of the existing schema in the given collection. 615 * @param solrClient The Solr client 616 * @param collection The collection. 617 * @throws IOException If a communication error occurs. 618 * @throws SolrServerException If a solr error occurs. 619 */ 620 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 621 { 622 try 623 { 624 getLogger().info("Clearing the existing schema on the solr server."); 625 626 SchemaRequest schemaRequest = new SchemaRequest(); 627 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 628 629 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 630 631 List<SchemaRequest.Update> deletions = new ArrayList<>(); 632 633 // First the copy fields, then dynamic and simple fields, and field types in the end. 634 for (Map<String, Object> field : schema.getCopyFields()) 635 { 636 String source = (String) field.get("source"); 637 String dest = (String) field.get("dest"); 638 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 639 } 640 for (Map<String, Object> field : schema.getDynamicFields()) 641 { 642 String name = (String) field.get("name"); 643 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 644 } 645 for (Map<String, Object> field : schema.getFields()) 646 { 647 String name = (String) field.get("name"); 648 if (!_READ_ONLY_FIELDS.contains(name)) 649 { 650 deletions.add(new SchemaRequest.DeleteField(name)); 651 } 652 } 653 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 654 { 655 String name = (String) fieldType.getAttributes().get("name"); 656 if (!_READ_ONLY_FIELDTYPES.contains(name)) 657 { 658 deletions.add(new SchemaRequest.DeleteFieldType(name)); 659 } 660 } 661 662 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 663 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 664 665 Object errors = updateResponse.getResponse().get("errors"); 666 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 667 { 668 String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString(); 669 throw new SolrServerException(msg); 670 } 671 else 672 { 673 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 674 } 675 676 getLogger().info("Solr schema cleared."); 677 } 678 catch (SolrServerException | IOException e) 679 { 680 getLogger().error("Error clearing schema in collection " + collection, e); 681 throw e; 682 } 683 } 684 685 /** 686 * Reload the solr cores. 687 * @throws IOException If a communication error occurs. 688 * @throws SolrServerException If a solr error occurs. 689 */ 690 protected void reloadCores() throws IOException, SolrServerException 691 { 692 getLogger().info("Reloading solr cores."); 693 694 for (String coreName : getCoreNames()) 695 { 696 String fullName = _solrCorePrefix + coreName; 697 698 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 699 700 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 701 } 702 703 getLogger().info("All cores reloaded."); 704 } 705 706 /** 707 * Reloads the ACL Solr cache for all users 708 * @throws IOException If an I/O error occurs. 709 * @throws SolrServerException If a Solr error occurs. 710 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 711 */ 712 public void reloadAclCache() throws IOException, SolrServerException, RepositoryException 713 { 714 String[] workspaceNames = _repository.getWorkspaces(); 715 for (String workspaceName : workspaceNames) 716 { 717 reloadAclCache(workspaceName); 718 } 719 } 720 721 /** 722 * Reloads the ACL Solr cache for all users 723 * @param workspaceName The workspace name 724 * @throws IOException If an I/O error occurs. 725 * @throws SolrServerException If a Solr error occurs. 726 */ 727 public void reloadAclCache(String workspaceName) throws IOException, SolrServerException 728 { 729 reloadAclCache(workspaceName, false); 730 } 731 732 /** 733 * Reloads the ACL Solr cache for all users 734 * @param workspaceName The workspace name 735 * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache) 736 * @throws IOException If an I/O error occurs. 737 * @throws SolrServerException If a Solr error occurs. 738 */ 739 public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException 740 { 741 getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 742 743 String collection = _solrClientProvider.getCollectionName(workspaceName); 744 SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection); 745 NamedList<Object> responseObj = responseBase.getResponse(); 746 747 if ("ok".equals(responseObj.get("result"))) 748 { 749 getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 750 } 751 else 752 { 753 Object error = responseObj.get("error"); 754 getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error); 755 } 756 } 757 758 /** 759 * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces. 760 * @param objects the {@link AmetysObject}s to update. 761 * @throws IOException If an I/O error occurs. 762 * @throws SolrServerException If a Solr error occurs. 763 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 764 */ 765 public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException 766 { 767 String[] workspaceNames = _repository.getWorkspaces(); 768 for (String workspaceName : workspaceNames) 769 { 770 updateAclCache(objects, workspaceName); 771 } 772 } 773 774 /** 775 * Updates the ACL Solr cache for some {@link AmetysObject}s. 776 * @param objects the {@link AmetysObject}s to update. 777 * @param workspaceName The workspace name 778 * @throws IOException If an I/O error occurs. 779 * @throws SolrServerException If a Solr error occurs. 780 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 781 */ 782 public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException 783 { 784 Map<String, Map<String, Object>> solrParams = new HashMap<>(); 785 786 for (AmetysObject object : objects) 787 { 788 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object); 789 790 solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(), 791 "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(), 792 "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 793 "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 794 "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()), 795 "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()))); 796 } 797 798 String collection = _solrClientProvider.getCollectionName(workspaceName); 799 SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection); 800 NamedList<Object> responseObj = response.getResponse(); 801 802 if ("ok".equals(responseObj.get("result"))) 803 { 804 if (getLogger().isInfoEnabled()) 805 { 806 getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet()); 807 } 808 } 809 else 810 { 811 @SuppressWarnings("unchecked") 812 List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects"); 813 getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects); 814 } 815 } 816 817 /** 818 * Index all the contents in a given workspace. 819 * @param workspaceName the workspace where to index 820 * @param indexAttachments to index content attachments 821 * @param solrClient The solr client to use 822 * @return The indexation result as a Map. 823 * @throws Exception if an error occurs while indexing. 824 */ 825 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 826 { 827 return indexAllContents(workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index all contents", getLogger())); 828 } 829 830 /** 831 * Index all the contents in a given workspace. 832 * @param workspaceName the workspace where to index 833 * @param indexAttachments to index content attachments 834 * @param solrClient The solr client to use 835 * @param progressionTracker The progression of the indexation 836 * @return The indexation result as a Map. 837 * @throws Exception if an error occurs while indexing. 838 */ 839 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 840 { 841 Request request = ContextHelper.getRequest(_context); 842 843 // Retrieve the current workspace. 844 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 845 846 try 847 { 848 // Force the workspace. 849 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 850 851 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 852 853 long start = System.currentTimeMillis(); 854 855 // Delete all contents 856 unindexAllContents(workspaceName, indexAttachments, solrClient); 857 858 String query = ContentQueryHelper.getContentXPathQuery(null); 859 AmetysObjectIterable<Content> contents = _resolver.query(query); 860 861 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient, progressionTracker); 862 863 long end = System.currentTimeMillis(); 864 865 if (!result.hasErrors()) 866 { 867 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 868 } 869 else 870 { 871 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 872 } 873 874 Map<String, Object> results = new HashMap<>(); 875 results.put("successCount", result.getSuccessCount()); 876 if (result.hasErrors()) 877 { 878 results.put("errorCount", result.getErrorCount()); 879 } 880 881 return results; 882 } 883 catch (Exception e) 884 { 885 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 886 getLogger().error(error, e); 887 throw new IndexingException(error, e); 888 } 889 finally 890 { 891 // Restore context 892 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 893 } 894 } 895 /** 896 * Unindex all content documents. 897 * @param workspaceName The workspace name 898 * @param unindexAttachments also unindex content attachments 899 * @param solrClient The solr client to use 900 * @throws Exception if an error occurs while unindexing. 901 */ 902 protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 903 { 904 String collection = _solrClientProvider.getCollectionName(workspaceName); 905 906 Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT); 907 Query query; 908 if (unindexAttachments) 909 { 910 Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE); 911 Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE); 912 query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents); 913 } 914 else 915 { 916 query = contents; 917 } 918 solrClient.deleteByQuery(collection, query.build()); 919 } 920 921 /** 922 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 923 * @param collectionId The id of collection 924 * @param indexAttachments to index content attachments 925 * @throws Exception if an error occurs while indexing. 926 */ 927 public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception 928 { 929 String[] workspaceNames = _repository.getWorkspaces(); 930 for (String workspaceName : workspaceNames) 931 { 932 indexSubcontents(collectionId, workspaceName, indexAttachments); 933 } 934 } 935 936 /** 937 * Index the child contents of a {@link AmetysObjectCollection} 938 * @param collectionId The id of collection 939 * @param workspaceName the workspace where to index 940 * @param indexAttachments to index content attachments 941 * @throws Exception if an error occurs while unindexing. 942 */ 943 public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception 944 { 945 Request request = ContextHelper.getRequest(_context); 946 947 // Retrieve the current workspace. 948 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 949 950 try 951 { 952 // Force the workspace. 953 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 954 955 if (_resolver.hasAmetysObjectForId(collectionId)) 956 { 957 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 958 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 959 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 960 961 for (AmetysObject child : children) 962 { 963 if (child instanceof Content) 964 { 965 Content content = (Content) child; 966 967 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 968 } 969 } 970 } 971 } 972 finally 973 { 974 // Restore context 975 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 976 } 977 } 978 979 /** 980 * Add or update a content into Solr index on all workspaces and commit 981 * @param contentId The id of the content to index 982 * @param indexAttachments to index content attachments 983 * @throws Exception if an error occurs while indexing. 984 */ 985 public void indexContent(String contentId, boolean indexAttachments) throws Exception 986 { 987 String[] workspaceNames = _repository.getWorkspaces(); 988 for (String workspaceName : workspaceNames) 989 { 990 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 991 indexContent(contentId, workspaceName, indexAttachments, solrClient); 992 } 993 } 994 995 /** 996 * Add or update a content into Solr index 997 * @param contentId The id of the content to index 998 * @param workspaceName the workspace where to index 999 * @param indexAttachments to index content attachments 1000 * @throws Exception if an error occurs while indexing. 1001 */ 1002 public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception 1003 { 1004 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1005 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1006 } 1007 1008 /** 1009 * Add or update a content into Solr index 1010 * @param contentId The id of the content to index 1011 * @param workspaceName the workspace where to index 1012 * @param indexAttachments to index content attachments 1013 * @param solrClient The solr client to use 1014 * @throws Exception if an error occurs while indexing. 1015 */ 1016 public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1017 { 1018 Request request = ContextHelper.getRequest(_context); 1019 1020 // Retrieve the current workspace. 1021 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1022 1023 try 1024 { 1025 // Force the workspace. 1026 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1027 1028 if (_resolver.hasAmetysObjectForId(contentId)) 1029 { 1030 Content content = _resolver.resolveById(contentId); 1031 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1032 } 1033 } 1034 finally 1035 { 1036 // Restore context 1037 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1038 } 1039 } 1040 1041 private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException 1042 { 1043 try 1044 { 1045 long time_0 = System.currentTimeMillis(); 1046 1047 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 1048 1049 deleteRepeaterDocs(content.getId(), workspaceName, solrClient); 1050 doIndexContent(content, workspaceName, solrClient); 1051 doIndexContentWorkflow(content, workspaceName, solrClient); 1052 if (indexAttachments) 1053 { 1054 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1055 } 1056 1057 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 1058 } 1059 catch (Exception e) 1060 { 1061 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 1062 getLogger().error(error, e); 1063 throw new IndexingException(error, e); 1064 } 1065 } 1066 1067 /** 1068 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1069 * @param contents the collection of contents to index. 1070 * @return the indexation result. 1071 * @throws Exception if an error occurs while indexing. 1072 */ 1073 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 1074 { 1075 return indexContents(contents, ProgressionTrackerFactory.createContainerProgressionTracker("Index contents", getLogger())); 1076 } 1077 1078 1079 /** 1080 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1081 * @param contents the collection of contents to index. 1082 * @param progressionTracker The progression of the indexation 1083 * @return the indexation result. 1084 * @throws Exception if an error occurs while indexing. 1085 */ 1086 public IndexationResult indexContents(Iterable<Content> contents, ContainerProgressionTracker progressionTracker) throws Exception 1087 { 1088 1089 IndexationResult result = new IndexationResult(); 1090 String[] workspaceNames = _repository.getWorkspaces(); 1091 1092 for (String workspaceName : workspaceNames) 1093 { 1094 progressionTracker.addSimpleStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_WORKSPACE_STEP_LABEL", List.of(workspaceName))); 1095 } 1096 1097 for (String workspaceName : workspaceNames) 1098 { 1099 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1100 IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient, progressionTracker.getStep(workspaceName)); 1101 1102 result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount()); 1103 } 1104 1105 return result; 1106 } 1107 1108 /** 1109 * Send a collection of contents for indexation in the solr server. 1110 * @param contents the collection of contents to index. 1111 * @param workspaceName the workspace where to index 1112 * @param indexAttachments to index content attachments 1113 * @param solrClient The solr client to use 1114 * @return the indexation result. 1115 * @throws Exception if an error occurs while indexing. 1116 */ 1117 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1118 { 1119 return indexContents(contents, workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index contents for workspace " + workspaceName, getLogger())); 1120 } 1121 1122 /** 1123 * Send a collection of contents for indexation in the solr server. 1124 * @param contents the collection of contents to index. 1125 * @param workspaceName the workspace where to index 1126 * @param indexAttachments to index content attachments 1127 * @param solrClient The solr client to use 1128 * @param progressionTracker The progression of the indexation 1129 * @return the indexation result. 1130 * @throws Exception if an error occurs while indexing. 1131 */ 1132 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1133 { 1134 Request request = ContextHelper.getRequest(_context); 1135 1136 // Retrieve the current workspace. 1137 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1138 1139 try 1140 { 1141 // Force the workspace. 1142 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1143 1144 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 1145 1146 long start = System.currentTimeMillis(); 1147 1148 List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false) 1149 .map(Content::getId) 1150 .map(this::_resolveSilently) 1151 .filter(Objects::nonNull) 1152 .collect(Collectors.toList()); 1153 1154 IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient, progressionTracker); 1155 1156 long end = System.currentTimeMillis(); 1157 1158 if (!result.hasErrors()) 1159 { 1160 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1161 } 1162 else 1163 { 1164 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1165 } 1166 1167 return result; 1168 } 1169 catch (Exception e) 1170 { 1171 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 1172 getLogger().error(error, e); 1173 throw new IndexingException(error, e); 1174 } 1175 finally 1176 { 1177 // Restore context 1178 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1179 } 1180 } 1181 1182 private Content _resolveSilently(String contentId) 1183 { 1184 try 1185 { 1186 return _resolver.resolveById(contentId); 1187 } 1188 catch (UnknownAmetysObjectException e) 1189 { 1190 return null; 1191 } 1192 } 1193 1194 /** 1195 * Send some contents for indexation in the solr server. 1196 * @param contents the contents to index. 1197 * @param workspaceName The workspace name 1198 * @param indexAttachments to index content attachments 1199 * @param solrClient The solr client to use 1200 * @param progressionTracker The progression of the indexation 1201 * @return the indexation result. 1202 * @throws Exception if an error occurs committing the results. 1203 */ 1204 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1205 { 1206 int successCount = 0; 1207 int errorCount = 0; 1208 int numberOfContents = IterableUtils.size(contents); 1209 1210 progressionTracker.setSize(numberOfContents); 1211 1212 for (Content content : contents) 1213 { 1214 try 1215 { 1216 doIndexContent(content, workspaceName, solrClient); 1217 doIndexContentWorkflow(content, workspaceName, solrClient); 1218 if (indexAttachments) 1219 { 1220 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1221 } 1222 successCount++; 1223 } 1224 catch (Exception e) 1225 { 1226 getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e); 1227 errorCount++; 1228 } 1229 1230 progressionTracker.increment(); 1231 } 1232 1233 return new IndexationResult(successCount, errorCount); 1234 } 1235 1236 /** 1237 * Update the value of a specific system property in a content document. 1238 * @param content The content to update. 1239 * @param propertyId The system property ID. 1240 * @param workspaceName The workspace name 1241 * @throws Exception if an error occurs while indexing. 1242 */ 1243 public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception 1244 { 1245 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1246 updateSystemProperty(content, propertyId, workspaceName, solrClient); 1247 } 1248 1249 /** 1250 * Update the value of a specific system property in a content document. 1251 * @param content The content to update. 1252 * @param propertyId The system property ID. 1253 * @param workspaceName The workspace name 1254 * @param solrClient The solr client to use 1255 * @throws Exception if an error occurs while indexing. 1256 */ 1257 public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1258 { 1259 getLogger().debug("Updating the system property '{}' for content {} into Solr.", propertyId, content); 1260 1261 SolrInputDocument document = new SolrInputDocument(); 1262 boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 1263 1264 if (!hasUpdate) 1265 { 1266 getLogger().debug("Did not index '{}' system property for content {} in Solr because no update to apply.", propertyId, content); 1267 return; 1268 } 1269 1270 int status = _pushSolrDocument(document, workspaceName, solrClient); 1271 if (status != 0) 1272 { 1273 throw new IOException("Indexing of system property '" + propertyId + "': got status code '" + status + "'."); 1274 } 1275 1276 getLogger().debug("Succesfully indexed '{}' system property for content {} in Solr.", propertyId, content); 1277 } 1278 1279 /** 1280 * Update the value of a specific property in a content document. 1281 * @param content The content to update. 1282 * @param propertyId The system property ID. 1283 * @param workspaceName The workspace name 1284 * @throws Exception if an error occurs while indexing. 1285 */ 1286 public void updateProperty(Content content, String propertyId, String workspaceName) throws Exception 1287 { 1288 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1289 updateProperty(content, propertyId, workspaceName, solrClient); 1290 } 1291 1292 /** 1293 * Update the value of a specific property in a content document. 1294 * @param content The content to update. 1295 * @param propertyId The system property ID. 1296 * @param workspaceName The workspace name 1297 * @param solrClient The solr client to use 1298 * @throws Exception if an error occurs while indexing. 1299 */ 1300 public void updateProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1301 { 1302 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 1303 1304 SolrInputDocument document = new SolrInputDocument(); 1305 boolean hasUpdate = _solrContentIndexer.indexPartialProperty(content, propertyId, document); 1306 1307 if (!hasUpdate) 1308 { 1309 getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content); 1310 return; 1311 } 1312 1313 int status = _pushSolrDocument(document, workspaceName, solrClient); 1314 if (status != 0) 1315 { 1316 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 1317 } 1318 1319 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 1320 } 1321 1322 private int _pushSolrDocument(SolrInputDocument document, String workspaceName, SolrClient solrClient) throws Exception 1323 { 1324 String collection = _solrClientProvider.getCollectionName(workspaceName); 1325 UpdateResponse solrResponse = solrClient.add(collection, document); 1326 return solrResponse.getStatus(); 1327 } 1328 1329 /** 1330 * Remove a content from Solr index for all workspaces and commit 1331 * @param contentId The id of content to unindex 1332 * @param unindexAttachments also unindex content attachments 1333 * @throws Exception if an error occurs while indexing. 1334 */ 1335 public void unindexContent(String contentId, boolean unindexAttachments) throws Exception 1336 { 1337 String[] workspaceNames = _repository.getWorkspaces(); 1338 for (String workspaceName : workspaceNames) 1339 { 1340 unindexContent(contentId, workspaceName, unindexAttachments); 1341 } 1342 } 1343 1344 /** 1345 * Remove a content from Solr index 1346 * @param contentId The id of content to unindex 1347 * @param workspaceName The workspace where to work in 1348 * @param unindexAttachments also unindex content attachments 1349 * @throws Exception if an error occurs while indexing. 1350 */ 1351 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception 1352 { 1353 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1354 unindexContent(contentId, workspaceName, unindexAttachments, solrClient); 1355 } 1356 1357 /** 1358 * Remove a content from Solr index 1359 * @param contentId The id of content to unindex 1360 * @param workspaceName The workspace where to work in 1361 * @param unindexAttachments also unindex content attachments 1362 * @param solrClient The solr client to use 1363 * @throws Exception if an error occurs while indexing. 1364 */ 1365 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 1366 { 1367 Request request = ContextHelper.getRequest(_context); 1368 1369 // Retrieve the current workspace. 1370 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1371 1372 try 1373 { 1374 // Force the workspace. 1375 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1376 1377 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 1378 1379 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1380 doUnindexDocument(contentId, workspaceName, solrClient); 1381 if (unindexAttachments) 1382 { 1383 doUnindexContentAttachments(contentId, workspaceName, solrClient); 1384 } 1385 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1386 1387 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 1388 } 1389 catch (Exception e) 1390 { 1391 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 1392 getLogger().error(error, e); 1393 throw new IndexingException(error, e); 1394 } 1395 finally 1396 { 1397 // Restore workspace 1398 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1399 } 1400 } 1401 1402 /** 1403 * Remove a content from Solr index for all workspaces and commit 1404 * @param contentIds The id of content to unindex 1405 * @throws Exception if an error occurs while indexing. 1406 */ 1407 public void unindexContents(Collection<String> contentIds) throws Exception 1408 { 1409 String[] workspaceNames = _repository.getWorkspaces(); 1410 for (String workspaceName : workspaceNames) 1411 { 1412 unindexContents(contentIds, workspaceName); 1413 } 1414 } 1415 1416 /** 1417 * Remove a content from Solr index 1418 * @param contentIds The id of content to unindex 1419 * @param workspaceName The workspace where to work in 1420 * @throws Exception if an error occurs while indexing. 1421 */ 1422 public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception 1423 { 1424 Request request = ContextHelper.getRequest(_context); 1425 1426 // Retrieve the current workspace. 1427 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1428 1429 try 1430 { 1431 // Force the workspace. 1432 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1433 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1434 1435 getLogger().debug("Unindexing several contents from Solr."); 1436 1437 for (String contentId : contentIds) 1438 { 1439 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1440 doUnindexDocument(contentId, workspaceName, solrClient); 1441 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1442 } 1443 1444 getLogger().debug("Succesfully unindexed content from Solr."); 1445 } 1446 catch (Exception e) 1447 { 1448 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1449 getLogger().error(error, e); 1450 throw new IndexingException(error, e); 1451 } 1452 finally 1453 { 1454 // Restore workspace 1455 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1456 } 1457 } 1458 1459 /** 1460 * Add or update a content into Solr index 1461 * @param content The content to index 1462 * @param workspaceName The workspace where to index 1463 * @param solrClient The solr client to use 1464 * @throws Exception if an error occurs while indexing. 1465 */ 1466 protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception 1467 { 1468 long time_0 = System.currentTimeMillis(); 1469 1470 SolrInputDocument document = new SolrInputDocument(); 1471 List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document); 1472 1473 long time_1 = System.currentTimeMillis(); 1474 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1475 1476 indexAclInitValues(content, document); 1477 1478 long time_2 = System.currentTimeMillis(); 1479 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1480 1481 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1482 documents.add(document); 1483 1484 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents); 1485 int status = solrResponse.getStatus(); 1486 1487 long time_3 = System.currentTimeMillis(); 1488 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1489 1490 if (status != 0) 1491 { 1492 throw new IOException("Content indexation: got status code '" + status + "'."); 1493 } 1494 } 1495 1496 /** 1497 * Indexes read-ACl initial values for object 1498 * @param ametysObject The object 1499 * @param document The Solr document 1500 */ 1501 public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document) 1502 { 1503 // Indexation of AmetysObject property 1504 document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true); 1505 1506 // Indexation of initValues for AllowedUsers 1507 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject); 1508 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed()); 1509 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed()); 1510 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString); 1511 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString); 1512 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString); 1513 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString); 1514 } 1515 1516 private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier) 1517 { 1518 if (values == null) 1519 { 1520 return; 1521 } 1522 1523 for (T value : values) 1524 { 1525 document.addField(fieldName, stringifier.apply(value)); 1526 } 1527 } 1528 1529 /** 1530 * Index the whole workflow of a content. 1531 * @param content The content. 1532 * @param workspaceName The workspace name 1533 * @param solrClient The solr client to use 1534 * @throws Exception if an error occurs while indexing. 1535 */ 1536 protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception 1537 { 1538 if (content instanceof WorkflowAwareContent) 1539 { 1540 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient); 1541 } 1542 } 1543 1544 /** 1545 * Index content attachments as new entries in the idnex 1546 * @param collection the collection of attachments 1547 * @param content the content whose attachments will be indexed 1548 * @throws Exception if something goes wrong when indexing the attachments of the content 1549 */ 1550 public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception 1551 { 1552 Request request = ContextHelper.getRequest(_context); 1553 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1554 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1555 indexContentAttachments(collection, content, solrClient); 1556 } 1557 1558 /** 1559 * Index content attachments as new entries in the idnex 1560 * @param collection the collection of attachments 1561 * @param content the content whose attachments will be indexed 1562 * @param solrClient The solr client to use 1563 * @throws Exception if something goes wrong when indexing the attachments of the content 1564 */ 1565 public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception 1566 { 1567 if (collection == null) 1568 { 1569 return; 1570 } 1571 1572 try (AmetysObjectIterable<AmetysObject> children = collection.getChildren()) 1573 { 1574 for (AmetysObject object : children) 1575 { 1576 if (object instanceof ResourceCollection) 1577 { 1578 indexContentAttachments((ResourceCollection) object, content, solrClient); 1579 } 1580 else if (object instanceof Resource) 1581 { 1582 Resource resource = (Resource) object; 1583 indexContentAttachment(resource, content, solrClient); 1584 } 1585 } 1586 } 1587 } 1588 1589 /** 1590 * Index a content attachment 1591 * @param resource the content attachment as a {@link Resource} 1592 * @param content the content whose attachment is going to be indexed 1593 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1594 */ 1595 public void indexContentAttachment(Resource resource, Content content) throws Exception 1596 { 1597 Request request = ContextHelper.getRequest(_context); 1598 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1599 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1600 indexContentAttachment(resource, content, solrClient); 1601 } 1602 1603 /** 1604 * Index a content attachment 1605 * @param resource the content attachment as a {@link Resource} 1606 * @param content the content whose attachment is going to be indexed 1607 * @param solrClient The solr client to use 1608 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1609 */ 1610 public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception 1611 { 1612 SolrInputDocument document = new SolrInputDocument(); 1613 1614 // Prepare resource doc 1615 _indexContentAttachment(resource, document, content); 1616 1617 // Indexation of the document 1618 _indexResourceDocument(resource, document, solrClient); 1619 } 1620 1621 private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception 1622 { 1623 String language = content.getLanguage(); 1624 1625 _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language); 1626 1627 // Need the id of the content for unindexing attachment during the unindexing of the content 1628 document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId()); 1629 } 1630 1631 /** 1632 * Index a populated solr input document of type Resource. 1633 * @param resource the resource from which the input document is created 1634 * @param document the input document 1635 * @param solrClient The solr client to use 1636 * @throws SolrServerException if there is an error on the server 1637 * @throws IOException if there is a communication error with the server 1638 */ 1639 protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException 1640 { 1641 String resourceId = resource.getId(); 1642 1643 // Retrieve appropriate collection name 1644 Request request = ContextHelper.getRequest(_context); 1645 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1646 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 1647 1648 // Add document 1649 UpdateResponse solrResponse = solrClient.add(collectionName, document); 1650 int status = solrResponse.getStatus(); 1651 1652 if (status != 0) 1653 { 1654 throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId); 1655 } 1656 1657 getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId); 1658 } 1659 1660 /** 1661 * Delete repeater documents of a specified content. 1662 * @param workspaceName The workspace name 1663 * @param contentId the content ID. 1664 * @param solrClient The solr client to use 1665 * @throws Exception if an error occurs while indexing. 1666 */ 1667 protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception 1668 { 1669 long time_0 = System.currentTimeMillis(); 1670 1671 // _documentType:repeater AND id:content\://xxx/* 1672 StringBuilder query = new StringBuilder(); 1673 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1674 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1675 1676 solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1677 1678 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1679 } 1680 1681 /** 1682 * Index all the resources in a given workspace. 1683 * @param workspaceName The workspace where to index 1684 * @param solrClient The solr client to use 1685 * @param progressionTracker The progression of the indexation 1686 * @return The indexation result as a Map. 1687 * @throws Exception if an error occurs while indexing. 1688 */ 1689 public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1690 { 1691 Request request = ContextHelper.getRequest(_context); 1692 1693 // Retrieve the current workspace. 1694 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1695 1696 try 1697 { 1698 // Force the workspace. 1699 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1700 1701 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1702 1703 long start = System.currentTimeMillis(); 1704 1705 // Delete all resources 1706 _unindexAllResources(workspaceName, solrClient); 1707 1708 Map<String, Object> results = new HashMap<>(); 1709 1710 try 1711 { 1712 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1713 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1714 1715 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient, progressionTracker); 1716 1717 long end = System.currentTimeMillis(); 1718 1719 if (!result.hasErrors()) 1720 { 1721 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1722 } 1723 else 1724 { 1725 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1726 } 1727 1728 results.put("successCount", result.getSuccessCount()); 1729 if (result.hasErrors()) 1730 { 1731 results.put("errorCount", result.getErrorCount()); 1732 } 1733 } 1734 catch (UnknownAmetysObjectException e) 1735 { 1736 getLogger().info("There is no root for resources in current workspace."); 1737 progressionTracker.setSize(0); 1738 } 1739 1740 return results; 1741 } 1742 catch (Exception e) 1743 { 1744 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1745 getLogger().error(error, e); 1746 throw new IndexingException(error, e); 1747 } 1748 finally 1749 { 1750 // Restore context 1751 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1752 } 1753 } 1754 1755 /** 1756 * Send a collection of contents for indexation in the solr server. 1757 * @param resources the collection of contents to index. 1758 * @param documentType The document type of the resource 1759 * @param workspaceName The workspace where to index 1760 * @return the indexation result. 1761 * @throws Exception if an error occurs while indexing. 1762 */ 1763 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception 1764 { 1765 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1766 return indexResources(resources, documentType, null, workspaceName, solrClient); 1767 } 1768 1769 /** 1770 * Send a collection of contents for indexation in the solr server. 1771 * @param resources the collection of contents to index. 1772 * @param documentType The document type of the resource 1773 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1774 * @param workspaceName The workspace where to index 1775 * @param solrClient The solr client to use 1776 * @return the indexation result. 1777 * @throws Exception if an error occurs while indexing. 1778 */ 1779 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1780 { 1781 return indexResources(resources, documentType, resourceRoot, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resources", getLogger())); 1782 } 1783 1784 /** 1785 * Send a collection of contents for indexation in the solr server. 1786 * @param resources the collection of contents to index. 1787 * @param documentType The document type of the resource 1788 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1789 * @param workspaceName The workspace where to index 1790 * @param solrClient The solr client to use 1791 * @param progressionTracker The progression of the indexation 1792 * @return the indexation result. 1793 * @throws Exception if an error occurs while indexing. 1794 */ 1795 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1796 { 1797 Request request = ContextHelper.getRequest(_context); 1798 1799 // Retrieve the current workspace. 1800 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1801 1802 try 1803 { 1804 // Force the workspace. 1805 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1806 1807 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1808 1809 long start = System.currentTimeMillis(); 1810 1811 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient, progressionTracker); 1812 1813 long end = System.currentTimeMillis(); 1814 1815 if (!result.hasErrors()) 1816 { 1817 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1818 } 1819 else 1820 { 1821 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1822 } 1823 1824 return result; 1825 } 1826 catch (Exception e) 1827 { 1828 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1829 getLogger().error(error, e); 1830 throw new IndexingException(error, e); 1831 } 1832 finally 1833 { 1834 // Restore context 1835 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1836 } 1837 } 1838 1839 /** 1840 * Send some resources for indexation in the solr server. 1841 * @param resources the resources to index. 1842 * @param documentType The document type of the resource 1843 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1844 * @param workspaceName The workspace where to index 1845 * @param solrClient The solr client to use 1846 * @param progressionTracker The progression of the indexation 1847 * @return the indexation result. 1848 * @throws Exception if an error occurs committing the results. 1849 */ 1850 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1851 { 1852 int successCount = 0; 1853 int errorCount = 0; 1854 int numberOfResources = IterableUtils.size(resources); 1855 1856 progressionTracker.setSize(numberOfResources); 1857 1858 for (AmetysObject resource : resources) 1859 { 1860 try 1861 { 1862 doIndexExplorerItem(resource, documentType, resourceRoot, solrClient); 1863 successCount++; 1864 } 1865 catch (Exception e) 1866 { 1867 getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e); 1868 errorCount++; 1869 } 1870 1871 progressionTracker.increment(); 1872 } 1873 1874 return new IndexationResult(successCount, errorCount); 1875 } 1876 1877 /** 1878 * Add or update a resource into Solr index 1879 * @param resource The resource to index 1880 * @param documentType The document type of the resource 1881 * @param workspaceName The workspace where to index 1882 * @throws Exception if an error occurs while indexing. 1883 */ 1884 public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception 1885 { 1886 Request request = ContextHelper.getRequest(_context); 1887 1888 // Retrieve the current workspace. 1889 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1890 1891 try 1892 { 1893 // Force the workspace. 1894 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1895 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1896 1897 getLogger().debug("Indexing resource {} into Solr.", resource.getId()); 1898 1899 doIndexResource(resource, documentType, null, solrClient); 1900 1901 getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId()); 1902 } 1903 catch (Exception e) 1904 { 1905 String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName); 1906 getLogger().error(error, e); 1907 throw new IndexingException(error, e); 1908 } 1909 finally 1910 { 1911 // Restore context 1912 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1913 } 1914 } 1915 1916 private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1917 { 1918 getLogger().debug("Unindexing all resources from Solr."); 1919 1920 String collection = _solrClientProvider.getCollectionName(workspaceName); 1921 solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1922 1923 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1924 } 1925 1926 /** 1927 * Delete all resource documents at a given path for all workspaces and commit 1928 * @param rootId The resource root ID, must not be null. 1929 * @param path The resource path relative to the given root, must start with a slash. 1930 * @throws Exception If an error occurs while unindexing. 1931 */ 1932 public void unindexResourcesByPath(String rootId, String path) throws Exception 1933 { 1934 String[] workspaceNames = _repository.getWorkspaces(); 1935 for (String workspaceName : workspaceNames) 1936 { 1937 unindexResourcesByPath(rootId, path, workspaceName); 1938 } 1939 } 1940 1941 /** 1942 * Delete all resource documents at a given path. 1943 * @param rootId The resource root ID, must not be null. 1944 * @param path The resource path relative to the given root, must start with a slash. 1945 * @param workspaceName The workspace where to work in 1946 * @throws Exception If an error occurs while unindexing. 1947 */ 1948 public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception 1949 { 1950 Request request = ContextHelper.getRequest(_context); 1951 1952 // Retrieve the current workspace. 1953 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1954 1955 try 1956 { 1957 // Force the workspace. 1958 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1959 1960 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1961 1962 Query query = new ResourceLocationQuery(rootId, path); 1963 1964 String collection = _solrClientProvider.getCollectionName(workspaceName); 1965 _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1966 1967 getLogger().debug("Succesfully deleted resource document from Solr."); 1968 } 1969 catch (Exception e) 1970 { 1971 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1972 getLogger().error(error, e); 1973 throw new IndexingException(error, e); 1974 } 1975 finally 1976 { 1977 // Restore workspace 1978 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1979 } 1980 } 1981 1982 /** 1983 * Remove a resource from Solr index for all workspaces and commit 1984 * @param resourceId The id of resource to unindex 1985 * @throws Exception if an error occurs while indexing. 1986 */ 1987 public void unindexResource(String resourceId) throws Exception 1988 { 1989 String[] workspaceNames = _repository.getWorkspaces(); 1990 for (String workspaceName : workspaceNames) 1991 { 1992 unindexResource(resourceId, workspaceName); 1993 } 1994 } 1995 1996 /** 1997 * Remove a resource from the Solr index. 1998 * @param resourceId The id of resource to unindex 1999 * @param workspaceName The workspace where to work in 2000 * @throws Exception if an error occurs while unindexing. 2001 */ 2002 public void unindexResource(String resourceId, String workspaceName) throws Exception 2003 { 2004 Request request = ContextHelper.getRequest(_context); 2005 2006 // Retrieve the current workspace. 2007 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 2008 2009 try 2010 { 2011 // Force the workspace. 2012 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 2013 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 2014 2015 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 2016 2017 doUnindexDocument(resourceId, workspaceName, solrClient); 2018 2019 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 2020 } 2021 catch (Exception e) 2022 { 2023 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 2024 getLogger().error(error, e); 2025 throw new IndexingException(error, e); 2026 } 2027 finally 2028 { 2029 // Restore workspace 2030 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 2031 } 2032 } 2033 2034 /** 2035 * Add or update a resource into Solr index 2036 * @param node The resource to index 2037 * @param documentType The document type of the resource 2038 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 2039 * @param solrClient The solr client to use 2040 * @throws Exception if an error occurs while indexing. 2041 */ 2042 protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2043 { 2044 if (node instanceof ResourceCollection) 2045 { 2046 try (AmetysObjectIterable<AmetysObject> children = ((ResourceCollection) node).getChildren()) 2047 { 2048 for (AmetysObject child : children) 2049 { 2050 doIndexExplorerItem(child, documentType, resourceRoot, solrClient); 2051 } 2052 } 2053 } 2054 else if (node instanceof Resource) 2055 { 2056 doIndexResource((Resource) node, documentType, resourceRoot, solrClient); 2057 } 2058 } 2059 2060 /** 2061 * Add or update a resource into Solr index 2062 * @param resource The resource to index 2063 * @param documentType The document type of the resource 2064 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 2065 * @param solrClient The solr client to use 2066 * @throws Exception if an error occurs while indexing. 2067 */ 2068 protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2069 { 2070 SolrInputDocument document = new SolrInputDocument(); 2071 2072 _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot); 2073 2074 String workspaceName = _workspaceSelector.getWorkspace(); 2075 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document); 2076 int status = solrResponse.getStatus(); 2077 2078 if (status != 0) 2079 { 2080 throw new IOException("Resource indexation: got status code '" + status + "'."); 2081 } 2082 } 2083 2084 /** 2085 * Process a Solr commit operation in all workspaces. 2086 * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient} 2087 * @throws SolrServerException if there is an error on the server 2088 * @throws IOException if there is a communication error with the server 2089 */ 2090 public void commit() throws SolrServerException, IOException 2091 { 2092 String[] workspaceNames; 2093 try 2094 { 2095 workspaceNames = _repository.getWorkspaces(); 2096 for (String workspaceName : workspaceNames) 2097 { 2098 SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName); 2099 commit(workspaceName, solrClient); 2100 } 2101 } 2102 catch (RepositoryException e) 2103 { 2104 throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e); 2105 } 2106 } 2107 2108 /** 2109 * Process a Solr commit operation in given workspace. 2110 * @param workspaceName The workspace's name 2111 * @param solrClient The solr client to use 2112 * @throws SolrServerException if there is an error on the server 2113 * @throws IOException if there is a communication error with the server 2114 */ 2115 public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2116 { 2117 long time_0 = System.currentTimeMillis(); 2118 2119 // Commit 2120 UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName)); 2121 int status = solrResponse.getStatus(); 2122 2123 if (status != 0) 2124 { 2125 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2126 } 2127 2128 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 2129 } 2130 2131 /** 2132 * Launch a solr index optimization. 2133 * @param workspaceName The workspace's name 2134 * @param solrClient The solr client to use 2135 * @throws SolrServerException if there is an error on the server 2136 * @throws IOException if there is a communication error with the server 2137 */ 2138 public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2139 { 2140 UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName)); 2141 int status = solrResponse.getStatus(); 2142 2143 if (status != 0) 2144 { 2145 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2146 } 2147 2148 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 2149 } 2150 2151 /** 2152 * Delete all documents from the solr index. 2153 * @param workspaceName The workspace name 2154 * @param solrClient The solr client to use 2155 * @throws Exception if an error occurs while unindexing. 2156 */ 2157 public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception 2158 { 2159 getLogger().debug("Deleting all documents from Solr."); 2160 2161 String collection = _solrClientProvider.getCollectionName(workspaceName); 2162 solrClient.deleteByQuery(collection, "*:*"); 2163 2164 getLogger().debug("Successfully deleted all documents from Solr."); 2165 } 2166 2167 /** 2168 * Delete a document from the Solr server. 2169 * @param id The id of the document to delete from Solr 2170 * @param workspaceName The workspace name 2171 * @param solrClient The solr client to use 2172 * @throws Exception if an error occurs while indexing. 2173 */ 2174 protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception 2175 { 2176 UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 2177 int status = solrResponse.getStatus(); 2178 2179 if (status != 0) 2180 { 2181 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 2182 } 2183 } 2184 2185 /** 2186 * Delete content attachments documents of a given content from the Solr server. 2187 * @param contentId The id of the content 2188 * @param workspaceName The workspace name 2189 * @param solrClient The solr client to use 2190 * @throws Exception if an error occurs while indexing. 2191 */ 2192 protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception 2193 { 2194 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 2195 2196 Query query = new ContentAttachmentQuery(contentId); 2197 UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build()); 2198 int status = solrResponse.getStatus(); 2199 2200 if (status != 0) 2201 { 2202 throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'."); 2203 } 2204 } 2205 2206// /** 2207// * Get the collection to use. 2208// * @return The name of the collection to index into. 2209// */ 2210// protected String getCollection() 2211// { 2212// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 2213// } 2214 2215 class IndexationResult 2216 { 2217 protected int _successCount; 2218 2219 protected int _errorCount; 2220 2221 /** 2222 * Constructor 2223 */ 2224 public IndexationResult() 2225 { 2226 this(0, 0); 2227 } 2228 2229 /** 2230 * Constructor 2231 * @param successCount The success count. 2232 * @param errorCount The error count. 2233 */ 2234 public IndexationResult(int successCount, int errorCount) 2235 { 2236 this._successCount = successCount; 2237 this._errorCount = errorCount; 2238 } 2239 2240 /** 2241 * Get the successCount. 2242 * @return the successCount 2243 */ 2244 public int getSuccessCount() 2245 { 2246 return _successCount; 2247 } 2248 2249 /** 2250 * Set the successCount. 2251 * @param successCount the successCount to set 2252 */ 2253 public void setSuccessCount(int successCount) 2254 { 2255 this._successCount = successCount; 2256 } 2257 2258 /** 2259 * Test if the indexation had errors. 2260 * @return true if the indexation had errors, false otherwise. 2261 */ 2262 public boolean hasErrors() 2263 { 2264 return _errorCount > 0; 2265 } 2266 2267 /** 2268 * Get the errorCount. 2269 * @return the errorCount 2270 */ 2271 public int getErrorCount() 2272 { 2273 return _errorCount; 2274 } 2275 2276 /** 2277 * Set the errorCount. 2278 * @param errorCount the errorCount to set 2279 */ 2280 public void setErrorCount(int errorCount) 2281 { 2282 this._errorCount = errorCount; 2283 } 2284 } 2285 2286 private static final class SchemaRequestComparator implements Comparator<SchemaRequest.Update> 2287 { 2288 public int compare(SchemaRequest.Update u1, SchemaRequest.Update u2) 2289 { 2290 return Integer.compare(_getOrder(u1), _getOrder(u2)); 2291 } 2292 2293 private int _getOrder(SchemaRequest.Update update) 2294 { 2295 // Field types are needed first to define fields 2296 if (update instanceof SchemaRequest.AddFieldType) 2297 { 2298 return 1; 2299 } 2300 2301 // Fields or dynamic fields can be defined at the same time 2302 if (update instanceof SchemaRequest.AddField || update instanceof SchemaRequest.AddDynamicField) 2303 { 2304 return 2; 2305 } 2306 2307 // Copy fields can be based on fields or dynamic fields 2308 if (update instanceof SchemaRequest.AddCopyField) 2309 { 2310 return 3; 2311 } 2312 2313 return 0; 2314 } 2315 } 2316}