001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.nio.ByteBuffer; 020import java.nio.CharBuffer; 021import java.nio.charset.CharsetDecoder; 022import java.nio.charset.CodingErrorAction; 023import java.nio.charset.StandardCharsets; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Collection; 028import java.util.Comparator; 029import java.util.HashMap; 030import java.util.HashSet; 031import java.util.List; 032import java.util.Map; 033import java.util.Objects; 034import java.util.Set; 035import java.util.function.Function; 036import java.util.stream.Collectors; 037import java.util.stream.StreamSupport; 038 039import javax.jcr.RepositoryException; 040 041import org.apache.avalon.framework.activity.Initializable; 042import org.apache.avalon.framework.component.Component; 043import org.apache.avalon.framework.context.Context; 044import org.apache.avalon.framework.context.ContextException; 045import org.apache.avalon.framework.context.Contextualizable; 046import org.apache.avalon.framework.service.ServiceException; 047import org.apache.avalon.framework.service.ServiceManager; 048import org.apache.avalon.framework.service.Serviceable; 049import org.apache.cocoon.components.ContextHelper; 050import org.apache.cocoon.environment.Request; 051import org.apache.commons.lang3.ObjectUtils; 052import org.apache.commons.lang3.StringUtils; 053import org.apache.solr.client.solrj.SolrClient; 054import org.apache.solr.client.solrj.SolrQuery; 055import org.apache.solr.client.solrj.SolrResponse; 056import org.apache.solr.client.solrj.SolrServerException; 057import org.apache.solr.client.solrj.request.CoreAdminRequest; 058import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 059import org.apache.solr.client.solrj.request.schema.SchemaRequest; 060import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 061import org.apache.solr.client.solrj.response.CoreAdminResponse; 062import org.apache.solr.client.solrj.response.QueryResponse; 063import org.apache.solr.client.solrj.response.SolrResponseBase; 064import org.apache.solr.client.solrj.response.UpdateResponse; 065import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 066import org.apache.solr.client.solrj.response.schema.SchemaResponse; 067import org.apache.solr.client.solrj.util.ClientUtils; 068import org.apache.solr.common.SolrInputDocument; 069import org.apache.solr.common.util.NamedList; 070import org.slf4j.Logger; 071 072import org.ametys.cms.indexing.IndexingException; 073import org.ametys.cms.indexing.solr.ReloadAclCacheRequest; 074import org.ametys.cms.indexing.solr.UpdateAclCacheRequest; 075import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest; 076import org.ametys.cms.repository.Content; 077import org.ametys.cms.repository.ContentQueryHelper; 078import org.ametys.cms.repository.WorkflowAwareContent; 079import org.ametys.cms.rights.solrchecking.ReadAccessHelper; 080import org.ametys.cms.search.query.ContentAttachmentQuery; 081import org.ametys.cms.search.query.DocumentTypeQuery; 082import org.ametys.cms.search.query.OrQuery; 083import org.ametys.cms.search.query.Query; 084import org.ametys.cms.search.query.ResourceLocationQuery; 085import org.ametys.cms.search.solr.NoAutoCommitUpdateClient; 086import org.ametys.cms.search.solr.SolrClientProvider; 087import org.ametys.cms.search.solr.schema.CopyFieldDefinition; 088import org.ametys.cms.search.solr.schema.FieldDefinition; 089import org.ametys.cms.search.solr.schema.SchemaDefinition; 090import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 091import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 092import org.ametys.cms.search.solr.schema.SchemaFields; 093import org.ametys.cms.search.solr.schema.SchemaHelper; 094import org.ametys.core.group.GroupIdentity; 095import org.ametys.core.right.AllowedUsers; 096import org.ametys.core.user.UserIdentity; 097import org.ametys.core.util.DateUtils; 098import org.ametys.plugins.explorer.resources.Resource; 099import org.ametys.plugins.explorer.resources.ResourceCollection; 100import org.ametys.plugins.repository.AmetysObject; 101import org.ametys.plugins.repository.AmetysObjectIterable; 102import org.ametys.plugins.repository.AmetysObjectResolver; 103import org.ametys.plugins.repository.RepositoryConstants; 104import org.ametys.plugins.repository.TraversableAmetysObject; 105import org.ametys.plugins.repository.UnknownAmetysObjectException; 106import org.ametys.plugins.repository.collection.AmetysObjectCollection; 107import org.ametys.plugins.repository.provider.AbstractRepository; 108import org.ametys.plugins.repository.provider.JackrabbitRepository; 109import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector; 110import org.ametys.plugins.repository.provider.WorkspaceSelector; 111import org.ametys.runtime.config.Config; 112import org.ametys.runtime.plugin.component.AbstractLogEnabled; 113 114/** 115 * Solr indexer. 116 */ 117public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 118{ 119 /** The component role. */ 120 public static final String ROLE = SolrIndexer.class.getName(); 121 122 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 123 124 private static final String _CONFIGSET_NAME_PREFIX = "configset-"; 125 126 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_"); 127 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general"); 128 129 private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766; 130 131 /** The ametys object resolver. */ 132 protected AmetysObjectResolver _resolver; 133 /** The schema definition provider extension point. */ 134 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 135 /** The schema helper. */ 136 protected SchemaHelper _schemaHelper; 137 /** Solr Ametys contents indexer */ 138 protected SolrContentIndexer _solrContentIndexer; 139 /** Solr workflow indexer. */ 140 protected SolrWorkflowIndexer _solrWorkflowIndexer; 141 /** Solr resource indexer. */ 142 protected SolrResourceIndexer _solrResourceIndexer; 143 144 /** The Solr client provider */ 145 protected SolrClientProvider _solrClientProvider; 146 147 /** The solr core prefix. */ 148 protected String _solrCorePrefix; 149 /** The Ametys internal URL used by Solr to query Ametys */ 150 protected String _ametysInternalUrl; 151 152 /** The workspace selector. */ 153 protected WorkspaceSelector _workspaceSelector; 154 /** The JCR repository */ 155 protected JackrabbitRepository _repository; 156 157 /** The helper for read access */ 158 protected ReadAccessHelper _readAccessHelper; 159 160 /** The avalon context */ 161 protected Context _context; 162 163 /** 164 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 165 * @return The date format for indexing dates 166 * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead 167 */ 168 @Deprecated 169 public static SimpleDateFormat dateFormat() 170 { 171 if (__DATE_FORMAT.get() == null) 172 { 173 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 174 } 175 return __DATE_FORMAT.get(); 176 } 177 178 /** 179 * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr. 180 * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept. 181 * @param value The string value to index 182 * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log. 183 * @param documentId The id of the document being indexed. Can be null if you do not want to log. 184 * @param fieldName The name of the field being indexed. Can be null if you do not want to log. 185 * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes) 186 */ 187 public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName) 188 { 189 if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT) 190 { 191 // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length()) 192 // As a result, value.getBytes().length <= limit 193 return value; 194 } 195 196 // There is a doubt, the string may need to be truncated (or not) 197 byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); 198 int bytesLength = valueBytes.length; 199 if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT) 200 { 201 return value; 202 } 203 204 if (ObjectUtils.allNotNull(logger, documentId, fieldName)) 205 { 206 logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT); 207 } 208 209 // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974) 210 CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT); 211 CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() 212 .onMalformedInput(CodingErrorAction.IGNORE); 213 decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true); 214 decoder.flush(charBuffer); 215 return new String(charBuffer.array(), 0, charBuffer.position()); 216 } 217 218 @Override 219 public void service(ServiceManager serviceManager) throws ServiceException 220 { 221 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 222 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 223 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 224 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 225 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 226 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 227 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 228 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 229 _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE); 230 _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE); 231 } 232 233 @Override 234 public void initialize() throws Exception 235 { 236 Config config = Config.getInstance(); 237 _solrCorePrefix = config.getValue("cms.solr.core.prefix"); 238 239 _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url"); 240 if (StringUtils.isBlank(_ametysInternalUrl)) 241 { 242 // fallback to CMS URL as internal URL is an optional parameter 243 _ametysInternalUrl = config.getValue("cms.url"); 244 } 245 } 246 247 @Override 248 public void contextualize(Context context) throws ContextException 249 { 250 _context = context; 251 } 252 253 /** 254 * Gets the 'autocommit' Solr client 255 * @param workspaceName The name of the workspace 256 * @return the Solr client 257 */ 258 protected SolrClient _getAutoCommitSolrClient(String workspaceName) 259 { 260 return _solrClientProvider.getUpdateClient(workspaceName, true); 261 } 262 263 /** 264 * Gets the 'no autocommit' Solr client 265 * @param workspaceName The name of the workspace 266 * @return the Solr client 267 */ 268 protected SolrClient _getNoAutoCommitSolrClient(String workspaceName) 269 { 270 return _solrClientProvider.getUpdateClient(workspaceName, false); 271 } 272 273 // for admin operations 274 private SolrClient _defaultSolrClient() 275 { 276 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 277 } 278 279 /** 280 * Get the names of the Solr cores. 281 * @return The names of the Solr cores. 282 * @throws IOException If an I/O error occurs. 283 * @throws SolrServerException If a Solr error occurs. 284 */ 285 @SuppressWarnings("unchecked") 286 public Set<String> getCoreNames() throws IOException, SolrServerException 287 { 288 Set<String> coreNames = new HashSet<>(); 289 290 getLogger().debug("Getting core list."); 291 292 SolrQuery query = new SolrQuery(); 293 query.setRequestHandler("/admin/cores"); 294 query.setParam("action", "STATUS"); 295 296 QueryResponse response = _defaultSolrClient().query(query); 297 298 NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status"); 299 for (Map.Entry<String, NamedList<?>> core : status) 300 { 301 String fullName = (String) core.getValue().get("name"); 302 if (fullName.startsWith(_solrCorePrefix)) 303 { 304 coreNames.add(fullName.substring(_solrCorePrefix.length())); 305 } 306 } 307 308 return coreNames; 309 } 310 311 /** 312 * Get the names of the Solr cores. 313 * @return The names of the Solr cores. 314 * @throws IOException If an I/O error occurs. 315 * @throws SolrServerException If a Solr error occurs. 316 */ 317 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 318 { 319 Set<String> coreNames = new HashSet<>(); 320 321 getLogger().debug("Getting core list."); 322 323 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 324 325 NamedList<NamedList<Object>> status = response.getCoreStatus(); 326 for (Map.Entry<String, NamedList<Object>> core : status) 327 { 328 String fullName = (String) core.getValue().get("name"); 329 if (fullName.startsWith(_solrCorePrefix)) 330 { 331 coreNames.add(fullName.substring(_solrCorePrefix.length())); 332 } 333 } 334 335 return coreNames; 336 } 337 338 /** 339 * Create a Solr core. 340 * @param name The name of the core to create. 341 * @throws IOException If an I/O error occurs. 342 * @throws SolrServerException If a Solr error occurs. 343 */ 344 public void createCore(String name) throws IOException, SolrServerException 345 { 346 String fullName = _solrCorePrefix + name; 347 String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix); 348 _createConfigset(configsetName); 349 350 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 351 352 SolrQuery query = new SolrQuery(); 353 query.setRequestHandler("/admin/cores"); 354 query.setParam("action", "CREATE"); 355 query.setParam("name", fullName); 356 query.setParam("configSet", configsetName); 357 query.setParam("property.ametys.url", _ametysInternalUrl); 358 359 QueryResponse response = _defaultSolrClient().query(query); 360 NamedList<?> results = response.getResponse(); 361 362 NamedList<?> error = (NamedList<?>) results.get("error"); 363 if (error != null) 364 { 365 throw new IOException("Error creating the core: " + error.get("msg")); 366 } 367 } 368 369 /** 370 * Updates the ametys.url property of the Solr cores. 371 */ 372 public void updateAmetysUrlCoreProperty() 373 { 374 Set<String> coreNames; 375 try 376 { 377 coreNames = getCoreNames(); 378 } 379 catch (SolrServerException | IOException e) 380 { 381 getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e); 382 return; 383 } 384 385 for (String coreName : coreNames) 386 { 387 String collection = _solrClientProvider.getCollectionName(coreName); 388 SolrResponseBase response; 389 try 390 { 391 response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection); 392 } 393 catch (SolrServerException | IOException e) 394 { 395 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e); 396 continue; 397 } 398 399 NamedList<Object> responseParams = response.getResponse(); 400 if ("ok".equals(responseParams.get("result"))) 401 { 402 Boolean valueChanged = responseParams.getBooleanArg("valueChanged"); 403 if (valueChanged) 404 { 405 getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName); 406 } 407 else 408 { 409 getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName); 410 } 411 } 412 else 413 { 414 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName); 415 } 416 } 417 } 418 419 private void _createConfigset(String name) throws IOException, SolrServerException 420 { 421 getLogger().info("Creating (if necessary) configset '{}'", name); 422 423 // This request handler will check if configset exists. If not it will be created. 424 SolrQuery query = new SolrQuery(); 425 query.setRequestHandler("/admin/cores"); 426 query.setParam("action", "createConfigset"); 427 query.setParam("name", name); 428 429 QueryResponse response = _defaultSolrClient().query(query); 430 NamedList<?> results = response.getResponse(); 431 432 NamedList<?> error = (NamedList<?>) results.get("error"); 433 if (error != null) 434 { 435 throw new IOException("Error creating the core: " + error.get("msg")); 436 } 437 } 438 439 /** 440 * Delete a Solr core. 441 * @param name The name of the core to delete. 442 * @throws IOException If an I/O error occurs. 443 * @throws SolrServerException If a Solr error occurs. 444 */ 445 public void deleteCore(String name) throws IOException, SolrServerException 446 { 447 String fullName = _solrCorePrefix + name; 448 449 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 450 451 SolrQuery query = new SolrQuery(); 452 query.setRequestHandler("/admin/cores"); 453 query.setParam("action", "UNLOAD"); 454 query.setParam("core", fullName); 455 query.setParam("deleteInstanceDir", "true"); 456 457 QueryResponse response = _defaultSolrClient().query(query); 458 NamedList<?> results = response.getResponse(); 459 460 NamedList<?> error = (NamedList<?>) results.get("error"); 461 if (error != null) 462 { 463 throw new IOException("Error deleting core" + name + ": " + error.get("msg")); 464 } 465 } 466 467 /** 468 * Send the schema. 469 * @throws IOException If a communication error occurs. 470 * @throws SolrServerException If a solr error occurs. 471 */ 472 public void sendSchema() throws IOException, SolrServerException 473 { 474 getLogger().info("Computing and sending the schema to the solr server."); 475 476 String workspaceName = _workspaceSelector.getWorkspace(); 477 String collection = _solrClientProvider.getCollectionName(workspaceName); 478 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 479 480// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 481 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 482 483 // TODO Clear the schema except fields marked ametysReadOnly="true". 484 485 // Clear the current schema. 486 clearSchema(solrClient, collection); 487 488 SchemaRequest schemaRequest = new SchemaRequest(); 489 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 490 491 // The cleared schema contains only the basic fields which can't be deleted. 492 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 493 SchemaFields schemaFields = new SchemaFields(clearedSchema); 494 495 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 496 497 // Add the static schema types and fields. 498 List<SchemaRequest.Update> updates = new ArrayList<>(); 499 500 // Set "add field" definitions from the static schema to the update list. 501 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 502 503 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 504 505 // Set "add field" definitions from the static schema to the update list. 506 addCustomUpdates(updates, schemaFields); 507 508 reorderUpdates(updates); 509 510 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 511 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 512 513 getLogger().debug("Send schema response: {}", updateResponse.toString()); 514 Object errors = updateResponse.getResponse().get("errors"); 515 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 516 { 517 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 518 throw new SolrServerException(msg); 519 } 520 521 getLogger().info("Schema sent to the solr server."); 522 523 reloadCores(); 524 } 525 526 /** 527 * Compute the list of {@link Update} directives from the static schema. 528 * @param updates The list of {@link Update} directives to fill. 529 * @param staticSchema The static schema representation. 530 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 531 */ 532 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 533 { 534 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 535 for (FieldTypeDefinition fieldType : fieldTypes) 536 { 537 String name = (String) fieldType.getAttributes().get("name"); 538 if (!schemaFields.hasFieldType(name)) 539 { 540 updates.add(new SchemaRequest.AddFieldType(fieldType)); 541 schemaFields.addFieldType(name); 542 } 543 } 544 for (Map<String, Object> field : staticSchema.getFields()) 545 { 546 String name = (String) field.get("name"); 547 if (!schemaFields.hasField(name)) 548 { 549 updates.add(new SchemaRequest.AddField(field)); 550 schemaFields.addField(name); 551 } 552 } 553 for (Map<String, Object> field : staticSchema.getDynamicFields()) 554 { 555 String name = (String) field.get("name"); 556 if (!schemaFields.hasDynamicField(name)) 557 { 558 updates.add(new SchemaRequest.AddDynamicField(field)); 559 schemaFields.addDynamicField(name); 560 } 561 } 562 for (Map<String, Object> field : staticSchema.getCopyFields()) 563 { 564 String source = (String) field.get("source"); 565 String dest = (String) field.get("dest"); 566 if (!schemaFields.hasCopyField(source, dest)) 567 { 568 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 569 schemaFields.addCopyField(source, dest); 570 } 571 } 572 } 573 574 /** 575 * Compute the list of custom {@link Update} directives. 576 * @param updates The list of {@link Update} directives to fill. 577 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 578 */ 579 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 580 { 581 // Add all our property-managed fields. 582 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 583 { 584 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 585 586 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 587 { 588 if (!definitionExists(definition, schemaFields)) 589 { 590 SchemaRequest.Update update = getSchemaUpdate(definition); 591 if (update != null) 592 { 593 updates.add(update); 594 } 595 } 596 } 597 } 598 599// for (String propId : _sysPropEP.getExtensionsIds()) 600// { 601// Collection<SchemaDefinition> definitions = _sysPropEP.getExtension(propId).getSchemaDefinitions(); 602// 603// for (SchemaDefinition definition : definitions) 604// { 605// if (!definitionExists(definition, schemaFields)) 606// { 607// SchemaRequest.Update update = getSchemaUpdate(definition); 608// if (update != null) 609// { 610// updates.add(update); 611// } 612// } 613// } 614// } 615 } 616 617 /** 618 * Reorder the list of {@link Update} directives. 619 * @param updates The list of {@link Update} directives to reorder. 620 */ 621 protected void reorderUpdates(List<SchemaRequest.Update> updates) 622 { 623 updates.sort(Comparator.comparing(u -> 624 { 625 if (u instanceof SchemaRequest.AddCopyField) 626 { 627 // copyField at the end of the list to avoid an exception of an undeclared 'source' or 'dest' whereas it is declared later 628 return 1; 629 } 630 else 631 { 632 return 0; 633 } 634 })); 635 } 636 637 /** 638 * Test if the given schema definition exists in the given {@link SchemaFields} reference. 639 * @param definition The schema definition to test. 640 * @param schemaFields The current schema fields. 641 * @return true if the SchemaFields contain the schema definition. 642 */ 643 protected boolean definitionExists(SchemaDefinition definition, SchemaFields schemaFields) 644 { 645 if (definition instanceof FieldDefinition) 646 { 647 FieldDefinition fieldDef = (FieldDefinition) definition; 648 if (fieldDef.isDynamic()) 649 { 650 return schemaFields.hasField(fieldDef.getName()); 651 } 652 else 653 { 654 return schemaFields.hasDynamicField(fieldDef.getName()); 655 } 656 } 657 else if (definition instanceof CopyFieldDefinition) 658 { 659 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 660 return schemaFields.hasCopyField(fieldDef.getSource(), fieldDef.getDestination()); 661 } 662 663 return false; 664 } 665 666 /** 667 * Delete all the fields of the existing schema in the given collection. 668 * @param solrClient The Solr client 669 * @param collection The collection. 670 * @throws IOException If a communication error occurs. 671 * @throws SolrServerException If a solr error occurs. 672 */ 673 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 674 { 675 try 676 { 677 getLogger().info("Clearing the existing schema on the solr server."); 678 679 SchemaRequest schemaRequest = new SchemaRequest(); 680 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 681 682 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 683 684 List<SchemaRequest.Update> deletions = new ArrayList<>(); 685 686 // First the copy fields, then dynamic and simple fields, and field types in the end. 687 for (Map<String, Object> field : schema.getCopyFields()) 688 { 689 String source = (String) field.get("source"); 690 String dest = (String) field.get("dest"); 691 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 692 } 693 for (Map<String, Object> field : schema.getDynamicFields()) 694 { 695 String name = (String) field.get("name"); 696 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 697 } 698 for (Map<String, Object> field : schema.getFields()) 699 { 700 String name = (String) field.get("name"); 701 if (!_READ_ONLY_FIELDS.contains(name)) 702 { 703 deletions.add(new SchemaRequest.DeleteField(name)); 704 } 705 } 706 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 707 { 708 String name = (String) fieldType.getAttributes().get("name"); 709 if (!_READ_ONLY_FIELDTYPES.contains(name)) 710 { 711 deletions.add(new SchemaRequest.DeleteFieldType(name)); 712 } 713 } 714 715 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 716 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 717 718 Object errors = updateResponse.getResponse().get("errors"); 719 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 720 { 721 String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString(); 722 throw new SolrServerException(msg); 723 } 724 else 725 { 726 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 727 } 728 729 getLogger().info("Solr schema cleared."); 730 } 731 catch (SolrServerException | IOException e) 732 { 733 getLogger().error("Error clearing schema in collection " + collection, e); 734 throw e; 735 } 736 } 737 738 /** 739 * Get the schema {@link Update} directive from the given schema definition. 740 * @param definition The schema definition to add. 741 * @return The add {@link Update} directive. 742 */ 743 protected SchemaRequest.Update getSchemaUpdate(SchemaDefinition definition) 744 { 745 SchemaRequest.Update update = null; 746 747 if (definition instanceof FieldDefinition) 748 { 749 FieldDefinition fieldDef = (FieldDefinition) definition; 750 751 // Force indexed and stored attributes to true. 752 Map<String, Object> attributes = new HashMap<>(); 753 attributes.put("name", fieldDef.getName()); 754 attributes.put("type", fieldDef.getType()); 755 attributes.put("multiValued", fieldDef.isMultiValued()); 756 attributes.put("docValues", fieldDef.isDocValues()); 757 attributes.put("indexed", Boolean.TRUE); 758 attributes.put("stored", Boolean.TRUE); 759 760 update = new SchemaRequest.AddField(attributes); 761 } 762 else if (definition instanceof CopyFieldDefinition) 763 { 764 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 765 766 String source = fieldDef.getSource(); 767 String dest = fieldDef.getDestination(); 768 769 update = new SchemaRequest.AddCopyField(source, Arrays.asList(dest)); 770 } 771 772 return update; 773 } 774 775 /** 776 * Reload the solr cores. 777 * @throws IOException If a communication error occurs. 778 * @throws SolrServerException If a solr error occurs. 779 */ 780 protected void reloadCores() throws IOException, SolrServerException 781 { 782 getLogger().info("Reloading solr cores."); 783 784 for (String coreName : getCoreNames()) 785 { 786 String fullName = _solrCorePrefix + coreName; 787 788 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 789 790 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 791 } 792 793 getLogger().info("All cores reloaded."); 794 } 795 796 /** 797 * Reloads the ACL Solr cache for all users 798 * @throws IOException If an I/O error occurs. 799 * @throws SolrServerException If a Solr error occurs. 800 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 801 */ 802 public void reloadAclCache() throws IOException, SolrServerException, RepositoryException 803 { 804 String[] workspaceNames = _repository.getWorkspaces(); 805 for (String workspaceName : workspaceNames) 806 { 807 reloadAclCache(workspaceName); 808 } 809 } 810 811 /** 812 * Reloads the ACL Solr cache for all users 813 * @param workspaceName The workspace name 814 * @throws IOException If an I/O error occurs. 815 * @throws SolrServerException If a Solr error occurs. 816 */ 817 public void reloadAclCache(String workspaceName) throws IOException, SolrServerException 818 { 819 reloadAclCache(workspaceName, false); 820 } 821 822 /** 823 * Reloads the ACL Solr cache for all users 824 * @param workspaceName The workspace name 825 * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache) 826 * @throws IOException If an I/O error occurs. 827 * @throws SolrServerException If a Solr error occurs. 828 */ 829 public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException 830 { 831 getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 832 833 String collection = _solrClientProvider.getCollectionName(workspaceName); 834 SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection); 835 NamedList<Object> responseObj = responseBase.getResponse(); 836 837 if ("ok".equals(responseObj.get("result"))) 838 { 839 getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 840 } 841 else 842 { 843 Object error = responseObj.get("error"); 844 getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error); 845 } 846 } 847 848 /** 849 * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces. 850 * @param objects the {@link AmetysObject}s to update. 851 * @throws IOException If an I/O error occurs. 852 * @throws SolrServerException If a Solr error occurs. 853 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 854 */ 855 public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException 856 { 857 String[] workspaceNames = _repository.getWorkspaces(); 858 for (String workspaceName : workspaceNames) 859 { 860 updateAclCache(objects, workspaceName); 861 } 862 } 863 864 /** 865 * Updates the ACL Solr cache for some {@link AmetysObject}s. 866 * @param objects the {@link AmetysObject}s to update. 867 * @param workspaceName The workspace name 868 * @throws IOException If an I/O error occurs. 869 * @throws SolrServerException If a Solr error occurs. 870 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 871 */ 872 public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException 873 { 874 Map<String, Map<String, Object>> solrParams = new HashMap<>(); 875 876 for (AmetysObject object : objects) 877 { 878 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object); 879 880 solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(), 881 "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(), 882 "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 883 "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 884 "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()), 885 "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()))); 886 } 887 888 String collection = _solrClientProvider.getCollectionName(workspaceName); 889 SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection); 890 NamedList<Object> responseObj = response.getResponse(); 891 892 if ("ok".equals(responseObj.get("result"))) 893 { 894 if (getLogger().isInfoEnabled()) 895 { 896 getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet()); 897 } 898 } 899 else 900 { 901 @SuppressWarnings("unchecked") 902 List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects"); 903 getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects); 904 } 905 } 906 907 /** 908 * Index all the contents in a given workspace. 909 * @param workspaceName the workspace where to index 910 * @param indexAttachments to index content attachments 911 * @param solrClient The solr client to use 912 * @return The indexation result as a Map. 913 * @throws Exception if an error occurs while indexing. 914 */ 915 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 916 { 917 Request request = ContextHelper.getRequest(_context); 918 919 // Retrieve the current workspace. 920 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 921 922 try 923 { 924 // Force the workspace. 925 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 926 927 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 928 929 long start = System.currentTimeMillis(); 930 931 // Delete all contents 932 unindexAllContents(workspaceName, indexAttachments, solrClient); 933 934 String query = ContentQueryHelper.getContentXPathQuery(null); 935 AmetysObjectIterable<Content> contents = _resolver.query(query); 936 937 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient); 938 939 long end = System.currentTimeMillis(); 940 941 if (!result.hasErrors()) 942 { 943 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 944 } 945 else 946 { 947 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 948 } 949 950 Map<String, Object> results = new HashMap<>(); 951 results.put("successCount", result.getSuccessCount()); 952 if (result.hasErrors()) 953 { 954 results.put("errorCount", result.getErrorCount()); 955 } 956 957 return results; 958 } 959 catch (Exception e) 960 { 961 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 962 getLogger().error(error, e); 963 throw new IndexingException(error, e); 964 } 965 finally 966 { 967 // Restore context 968 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 969 } 970 } 971 972 /** 973 * Unindex all content documents. 974 * @param workspaceName The workspace name 975 * @param unindexAttachments also unindex content attachments 976 * @param solrClient The solr client to use 977 * @throws Exception if an error occurs while unindexing. 978 */ 979 protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 980 { 981 String collection = _solrClientProvider.getCollectionName(workspaceName); 982 983 Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT); 984 Query query; 985 if (unindexAttachments) 986 { 987 Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE); 988 Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE); 989 query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents); 990 } 991 else 992 { 993 query = contents; 994 } 995 solrClient.deleteByQuery(collection, query.build()); 996 } 997 998 /** 999 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 1000 * @param collectionId The id of collection 1001 * @param indexAttachments to index content attachments 1002 * @throws Exception if an error occurs while indexing. 1003 */ 1004 public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception 1005 { 1006 String[] workspaceNames = _repository.getWorkspaces(); 1007 for (String workspaceName : workspaceNames) 1008 { 1009 indexSubcontents(collectionId, workspaceName, indexAttachments); 1010 } 1011 } 1012 1013 /** 1014 * Index the child contents of a {@link AmetysObjectCollection} 1015 * @param collectionId The id of collection 1016 * @param workspaceName the workspace where to index 1017 * @param indexAttachments to index content attachments 1018 * @throws Exception if an error occurs while unindexing. 1019 */ 1020 public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception 1021 { 1022 Request request = ContextHelper.getRequest(_context); 1023 1024 // Retrieve the current workspace. 1025 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1026 1027 try 1028 { 1029 // Force the workspace. 1030 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1031 1032 if (_resolver.hasAmetysObjectForId(collectionId)) 1033 { 1034 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1035 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 1036 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 1037 1038 for (AmetysObject child : children) 1039 { 1040 if (child instanceof Content) 1041 { 1042 Content content = (Content) child; 1043 1044 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1045 } 1046 } 1047 } 1048 } 1049 finally 1050 { 1051 // Restore context 1052 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1053 } 1054 } 1055 1056 /** 1057 * Add or update a content into Solr index on all workspaces and commit 1058 * @param contentId The id of the content to index 1059 * @param indexAttachments to index content attachments 1060 * @throws Exception if an error occurs while indexing. 1061 */ 1062 public void indexContent(String contentId, boolean indexAttachments) throws Exception 1063 { 1064 String[] workspaceNames = _repository.getWorkspaces(); 1065 for (String workspaceName : workspaceNames) 1066 { 1067 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1068 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1069 } 1070 } 1071 1072 /** 1073 * Add or update a content into Solr index 1074 * @param contentId The id of the content to index 1075 * @param workspaceName the workspace where to index 1076 * @param indexAttachments to index content attachments 1077 * @throws Exception if an error occurs while indexing. 1078 */ 1079 public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception 1080 { 1081 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1082 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1083 } 1084 1085 /** 1086 * Add or update a content into Solr index 1087 * @param contentId The id of the content to index 1088 * @param workspaceName the workspace where to index 1089 * @param indexAttachments to index content attachments 1090 * @param solrClient The solr client to use 1091 * @throws Exception if an error occurs while indexing. 1092 */ 1093 public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1094 { 1095 Request request = ContextHelper.getRequest(_context); 1096 1097 // Retrieve the current workspace. 1098 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1099 1100 try 1101 { 1102 // Force the workspace. 1103 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1104 1105 if (_resolver.hasAmetysObjectForId(contentId)) 1106 { 1107 Content content = _resolver.resolveById(contentId); 1108 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1109 } 1110 } 1111 finally 1112 { 1113 // Restore context 1114 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1115 } 1116 } 1117 1118 private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException 1119 { 1120 try 1121 { 1122 long time_0 = System.currentTimeMillis(); 1123 1124 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 1125 1126 deleteRepeaterDocs(content.getId(), workspaceName, solrClient); 1127 doIndexContent(content, workspaceName, solrClient); 1128 doIndexContentWorkflow(content, workspaceName, solrClient); 1129 if (indexAttachments) 1130 { 1131 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1132 } 1133 1134 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 1135 } 1136 catch (Exception e) 1137 { 1138 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 1139 getLogger().error(error, e); 1140 throw new IndexingException(error, e); 1141 } 1142 } 1143 1144 /** 1145 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1146 * @param contents the collection of contents to index. 1147 * @return the indexation result. 1148 * @throws Exception if an error occurs while indexing. 1149 */ 1150 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 1151 { 1152 IndexationResult result = new IndexationResult(); 1153 1154 String[] workspaceNames = _repository.getWorkspaces(); 1155 for (String workspaceName : workspaceNames) 1156 { 1157 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1158 IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient); 1159 result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount()); 1160 } 1161 1162 return result; 1163 } 1164 1165 /** 1166 * Send a collection of contents for indexation in the solr server. 1167 * @param contents the collection of contents to index. 1168 * @param workspaceName the workspace where to index 1169 * @param indexAttachments to index content attachments 1170 * @param solrClient The solr client to use 1171 * @return the indexation result. 1172 * @throws Exception if an error occurs while indexing. 1173 */ 1174 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1175 { 1176 Request request = ContextHelper.getRequest(_context); 1177 1178 // Retrieve the current workspace. 1179 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1180 1181 try 1182 { 1183 // Force the workspace. 1184 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1185 1186 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 1187 1188 long start = System.currentTimeMillis(); 1189 1190 List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false) 1191 .map(Content::getId) 1192 .map(this::_resolveSilently) 1193 .filter(Objects::nonNull) 1194 .collect(Collectors.toList()); 1195 1196 IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient); 1197 1198 long end = System.currentTimeMillis(); 1199 1200 if (!result.hasErrors()) 1201 { 1202 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1203 } 1204 else 1205 { 1206 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1207 } 1208 1209 return result; 1210 } 1211 catch (Exception e) 1212 { 1213 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 1214 getLogger().error(error, e); 1215 throw new IndexingException(error, e); 1216 } 1217 finally 1218 { 1219 // Restore context 1220 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1221 } 1222 1223 } 1224 1225 private Content _resolveSilently(String contentId) 1226 { 1227 try 1228 { 1229 return _resolver.resolveById(contentId); 1230 } 1231 catch (UnknownAmetysObjectException e) 1232 { 1233 return null; 1234 } 1235 } 1236 1237 /** 1238 * Send some contents for indexation in the solr server. 1239 * @param contents the contents to index. 1240 * @param workspaceName The workspace name 1241 * @param indexAttachments to index content attachments 1242 * @param solrClient The solr client to use 1243 * @return the indexation result. 1244 * @throws Exception if an error occurs committing the results. 1245 */ 1246 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1247 { 1248 int successCount = 0; 1249 int errorCount = 0; 1250 for (Content content : contents) 1251 { 1252 try 1253 { 1254 doIndexContent(content, workspaceName, solrClient); 1255 doIndexContentWorkflow(content, workspaceName, solrClient); 1256 if (indexAttachments) 1257 { 1258 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1259 } 1260 successCount++; 1261 } 1262 catch (Exception e) 1263 { 1264 getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e); 1265 errorCount++; 1266 } 1267 } 1268 1269 return new IndexationResult(successCount, errorCount); 1270 } 1271 1272 /** 1273 * Update the value of a specific system property in a content document. 1274 * @param content The content to update. 1275 * @param propertyId The system property ID. 1276 * @param workspaceName The workspace name 1277 * @throws Exception if an error occurs while indexing. 1278 */ 1279 public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception 1280 { 1281 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1282 updateSystemProperty(content, propertyId, workspaceName, solrClient); 1283 } 1284 1285 /** 1286 * Update the value of a specific system property in a content document. 1287 * @param content The content to update. 1288 * @param propertyId The system property ID. 1289 * @param workspaceName The workspace name 1290 * @param solrClient The solr client to use 1291 * @throws Exception if an error occurs while indexing. 1292 */ 1293 public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1294 { 1295 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 1296 1297 SolrInputDocument document = new SolrInputDocument(); 1298 boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 1299 1300 if (!hasUpdate) 1301 { 1302 getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content); 1303 return; 1304 } 1305 1306 String collection = _solrClientProvider.getCollectionName(workspaceName); 1307 UpdateResponse solrResponse = solrClient.add(collection, document); 1308 int status = solrResponse.getStatus(); 1309 1310 if (status != 0) 1311 { 1312 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 1313 } 1314 1315 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 1316 } 1317 1318 /** 1319 * Remove a content from Solr index for all workspaces and commit 1320 * @param contentId The id of content to unindex 1321 * @param unindexAttachments also unindex content attachments 1322 * @throws Exception if an error occurs while indexing. 1323 */ 1324 public void unindexContent(String contentId, boolean unindexAttachments) throws Exception 1325 { 1326 String[] workspaceNames = _repository.getWorkspaces(); 1327 for (String workspaceName : workspaceNames) 1328 { 1329 unindexContent(contentId, workspaceName, unindexAttachments); 1330 } 1331 } 1332 1333 /** 1334 * Remove a content from Solr index 1335 * @param contentId The id of content to unindex 1336 * @param workspaceName The workspace where to work in 1337 * @param unindexAttachments also unindex content attachments 1338 * @throws Exception if an error occurs while indexing. 1339 */ 1340 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception 1341 { 1342 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1343 unindexContent(contentId, workspaceName, unindexAttachments, solrClient); 1344 } 1345 1346 /** 1347 * Remove a content from Solr index 1348 * @param contentId The id of content to unindex 1349 * @param workspaceName The workspace where to work in 1350 * @param unindexAttachments also unindex content attachments 1351 * @param solrClient The solr client to use 1352 * @throws Exception if an error occurs while indexing. 1353 */ 1354 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 1355 { 1356 Request request = ContextHelper.getRequest(_context); 1357 1358 // Retrieve the current workspace. 1359 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1360 1361 try 1362 { 1363 // Force the workspace. 1364 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1365 1366 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 1367 1368 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1369 doUnindexDocument(contentId, workspaceName, solrClient); 1370 if (unindexAttachments) 1371 { 1372 doUnindexContentAttachments(contentId, workspaceName, solrClient); 1373 } 1374 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1375 1376 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 1377 } 1378 catch (Exception e) 1379 { 1380 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 1381 getLogger().error(error, e); 1382 throw new IndexingException(error, e); 1383 } 1384 finally 1385 { 1386 // Restore workspace 1387 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1388 } 1389 } 1390 1391 /** 1392 * Remove a content from Solr index for all workspaces and commit 1393 * @param contentIds The id of content to unindex 1394 * @throws Exception if an error occurs while indexing. 1395 */ 1396 public void unindexContents(Collection<String> contentIds) throws Exception 1397 { 1398 String[] workspaceNames = _repository.getWorkspaces(); 1399 for (String workspaceName : workspaceNames) 1400 { 1401 unindexContents(contentIds, workspaceName); 1402 } 1403 } 1404 1405 /** 1406 * Remove a content from Solr index 1407 * @param contentIds The id of content to unindex 1408 * @param workspaceName The workspace where to work in 1409 * @throws Exception if an error occurs while indexing. 1410 */ 1411 public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception 1412 { 1413 Request request = ContextHelper.getRequest(_context); 1414 1415 // Retrieve the current workspace. 1416 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1417 1418 try 1419 { 1420 // Force the workspace. 1421 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1422 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1423 1424 getLogger().debug("Unindexing several contents from Solr."); 1425 1426 for (String contentId : contentIds) 1427 { 1428 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1429 doUnindexDocument(contentId, workspaceName, solrClient); 1430 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1431 } 1432 1433 getLogger().debug("Succesfully unindexed content from Solr."); 1434 } 1435 catch (Exception e) 1436 { 1437 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1438 getLogger().error(error, e); 1439 throw new IndexingException(error, e); 1440 } 1441 finally 1442 { 1443 // Restore workspace 1444 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1445 } 1446 } 1447 1448 /** 1449 * Add or update a content into Solr index 1450 * @param content The content to index 1451 * @param workspaceName The workspace where to index 1452 * @param solrClient The solr client to use 1453 * @throws Exception if an error occurs while indexing. 1454 */ 1455 protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception 1456 { 1457 long time_0 = System.currentTimeMillis(); 1458 1459 SolrInputDocument document = new SolrInputDocument(); 1460 List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document); 1461 1462 long time_1 = System.currentTimeMillis(); 1463 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1464 1465 indexAclInitValues(content, document); 1466 1467 long time_2 = System.currentTimeMillis(); 1468 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1469 1470 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1471 documents.add(document); 1472 1473 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents); 1474 int status = solrResponse.getStatus(); 1475 1476 long time_3 = System.currentTimeMillis(); 1477 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1478 1479 if (status != 0) 1480 { 1481 throw new IOException("Content indexation: got status code '" + status + "'."); 1482 } 1483 } 1484 1485 /** 1486 * Indexes read-ACl initial values for object 1487 * @param ametysObject The object 1488 * @param document The Solr document 1489 */ 1490 public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document) 1491 { 1492 // Indexation of AmetysObject property 1493 document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true); 1494 1495 // Indexation of initValues for AllowedUsers 1496 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject); 1497 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed()); 1498 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed()); 1499 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString); 1500 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString); 1501 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString); 1502 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString); 1503 } 1504 1505 private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier) 1506 { 1507 if (values == null) 1508 { 1509 return; 1510 } 1511 1512 for (T value : values) 1513 { 1514 document.addField(fieldName, stringifier.apply(value)); 1515 } 1516 } 1517 1518 /** 1519 * Index the whole workflow of a content. 1520 * @param content The content. 1521 * @param workspaceName The workspace name 1522 * @param solrClient The solr client to use 1523 * @throws Exception if an error occurs while indexing. 1524 */ 1525 protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception 1526 { 1527 if (content instanceof WorkflowAwareContent) 1528 { 1529 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient); 1530 } 1531 } 1532 1533 /** 1534 * Index content attachments as new entries in the idnex 1535 * @param collection the collection of attachments 1536 * @param content the content whose attachments will be indexed 1537 * @throws Exception if something goes wrong when indexing the attachments of the content 1538 */ 1539 public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception 1540 { 1541 Request request = ContextHelper.getRequest(_context); 1542 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1543 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1544 indexContentAttachments(collection, content, solrClient); 1545 } 1546 1547 /** 1548 * Index content attachments as new entries in the idnex 1549 * @param collection the collection of attachments 1550 * @param content the content whose attachments will be indexed 1551 * @param solrClient The solr client to use 1552 * @throws Exception if something goes wrong when indexing the attachments of the content 1553 */ 1554 public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception 1555 { 1556 if (collection == null) 1557 { 1558 return; 1559 } 1560 1561 try (AmetysObjectIterable<AmetysObject> children = collection.getChildren()) 1562 { 1563 for (AmetysObject object : children) 1564 { 1565 if (object instanceof ResourceCollection) 1566 { 1567 indexContentAttachments((ResourceCollection) object, content, solrClient); 1568 } 1569 else if (object instanceof Resource) 1570 { 1571 Resource resource = (Resource) object; 1572 indexContentAttachment(resource, content, solrClient); 1573 } 1574 } 1575 } 1576 } 1577 1578 /** 1579 * Index a content attachment 1580 * @param resource the content attachment as a {@link Resource} 1581 * @param content the content whose attachment is going to be indexed 1582 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1583 */ 1584 public void indexContentAttachment(Resource resource, Content content) throws Exception 1585 { 1586 Request request = ContextHelper.getRequest(_context); 1587 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1588 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1589 indexContentAttachment(resource, content, solrClient); 1590 } 1591 1592 /** 1593 * Index a content attachment 1594 * @param resource the content attachment as a {@link Resource} 1595 * @param content the content whose attachment is going to be indexed 1596 * @param solrClient The solr client to use 1597 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1598 */ 1599 public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception 1600 { 1601 SolrInputDocument document = new SolrInputDocument(); 1602 1603 // Prepare resource doc 1604 _indexContentAttachment(resource, document, content); 1605 1606 // Indexation of the document 1607 _indexResourceDocument(resource, document, solrClient); 1608 } 1609 1610 private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception 1611 { 1612 String language = content.getLanguage(); 1613 1614 _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language); 1615 1616 // Need the id of the content for unindexing attachment during the unindexing of the content 1617 document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId()); 1618 } 1619 1620 /** 1621 * Index a populated solr input document of type Resource. 1622 * @param resource the resource from which the input document is created 1623 * @param document the input document 1624 * @param solrClient The solr client to use 1625 * @throws SolrServerException if there is an error on the server 1626 * @throws IOException if there is a communication error with the server 1627 */ 1628 protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException 1629 { 1630 String resourceId = resource.getId(); 1631 1632 // Retrieve appropriate collection name 1633 Request request = ContextHelper.getRequest(_context); 1634 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1635 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 1636 1637 // Add document 1638 UpdateResponse solrResponse = solrClient.add(collectionName, document); 1639 int status = solrResponse.getStatus(); 1640 1641 if (status != 0) 1642 { 1643 throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId); 1644 } 1645 1646 getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId); 1647 } 1648 1649 /** 1650 * Delete repeater documents of a specified content. 1651 * @param workspaceName The workspace name 1652 * @param contentId the content ID. 1653 * @param solrClient The solr client to use 1654 * @throws Exception if an error occurs while indexing. 1655 */ 1656 protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception 1657 { 1658 long time_0 = System.currentTimeMillis(); 1659 1660 // _documentType:repeater AND id:content\://xxx/* 1661 StringBuilder query = new StringBuilder(); 1662 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1663 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1664 1665 solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1666 1667 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1668 } 1669 1670 /** 1671 * Index all the resources in a given workspace. 1672 * @param workspaceName The workspace where to index 1673 * @param solrClient The solr client to use 1674 * @return The indexation result as a Map. 1675 * @throws Exception if an error occurs while indexing. 1676 */ 1677 public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1678 { 1679 Request request = ContextHelper.getRequest(_context); 1680 1681 // Retrieve the current workspace. 1682 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1683 1684 try 1685 { 1686 // Force the workspace. 1687 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1688 1689 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1690 1691 long start = System.currentTimeMillis(); 1692 1693 // Delete all resources 1694 _unindexAllResources(workspaceName, solrClient); 1695 1696 Map<String, Object> results = new HashMap<>(); 1697 1698 try 1699 { 1700 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1701 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1702 1703 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient); 1704 1705 long end = System.currentTimeMillis(); 1706 1707 if (!result.hasErrors()) 1708 { 1709 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1710 } 1711 else 1712 { 1713 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1714 } 1715 1716 results.put("successCount", result.getSuccessCount()); 1717 if (result.hasErrors()) 1718 { 1719 results.put("errorCount", result.getErrorCount()); 1720 } 1721 } 1722 catch (UnknownAmetysObjectException e) 1723 { 1724 getLogger().info("There is no root for resources in current workspace."); 1725 } 1726 1727 return results; 1728 } 1729 catch (Exception e) 1730 { 1731 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1732 getLogger().error(error, e); 1733 throw new IndexingException(error, e); 1734 } 1735 finally 1736 { 1737 // Restore context 1738 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1739 } 1740 } 1741 1742 /** 1743 * Send a collection of contents for indexation in the solr server. 1744 * @param resources the collection of contents to index. 1745 * @param documentType The document type of the resource 1746 * @param workspaceName The workspace where to index 1747 * @return the indexation result. 1748 * @throws Exception if an error occurs while indexing. 1749 */ 1750 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception 1751 { 1752 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1753 return indexResources(resources, documentType, null, workspaceName, solrClient); 1754 } 1755 1756 /** 1757 * Send a collection of contents for indexation in the solr server. 1758 * @param resources the collection of contents to index. 1759 * @param documentType The document type of the resource 1760 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1761 * @param workspaceName The workspace where to index 1762 * @param solrClient The solr client to use 1763 * @return the indexation result. 1764 * @throws Exception if an error occurs while indexing. 1765 */ 1766 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1767 { 1768 Request request = ContextHelper.getRequest(_context); 1769 1770 // Retrieve the current workspace. 1771 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1772 1773 try 1774 { 1775 // Force the workspace. 1776 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1777 1778 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1779 1780 long start = System.currentTimeMillis(); 1781 1782 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient); 1783 1784 long end = System.currentTimeMillis(); 1785 1786 if (!result.hasErrors()) 1787 { 1788 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1789 } 1790 else 1791 { 1792 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1793 } 1794 1795 return result; 1796 } 1797 catch (Exception e) 1798 { 1799 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1800 getLogger().error(error, e); 1801 throw new IndexingException(error, e); 1802 } 1803 finally 1804 { 1805 // Restore context 1806 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1807 } 1808 } 1809 1810 /** 1811 * Send some resources for indexation in the solr server. 1812 * @param resources the resources to index. 1813 * @param documentType The document type of the resource 1814 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1815 * @param workspaceName The workspace where to index 1816 * @param solrClient The solr client to use 1817 * @return the indexation result. 1818 * @throws Exception if an error occurs committing the results. 1819 */ 1820 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1821 { 1822 int successCount = 0; 1823 int errorCount = 0; 1824 for (AmetysObject resource : resources) 1825 { 1826 try 1827 { 1828 doIndexExplorerItem(resource, documentType, resourceRoot, solrClient); 1829 successCount++; 1830 } 1831 catch (Exception e) 1832 { 1833 getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e); 1834 errorCount++; 1835 } 1836 } 1837 1838 return new IndexationResult(successCount, errorCount); 1839 } 1840 1841 /** 1842 * Add or update a resource into Solr index 1843 * @param resource The resource to index 1844 * @param documentType The document type of the resource 1845 * @param workspaceName The workspace where to index 1846 * @throws Exception if an error occurs while indexing. 1847 */ 1848 public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception 1849 { 1850 Request request = ContextHelper.getRequest(_context); 1851 1852 // Retrieve the current workspace. 1853 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1854 1855 try 1856 { 1857 // Force the workspace. 1858 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1859 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1860 1861 getLogger().debug("Indexing resource {} into Solr.", resource.getId()); 1862 1863 doIndexResource(resource, documentType, null, solrClient); 1864 1865 getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId()); 1866 } 1867 catch (Exception e) 1868 { 1869 String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName); 1870 getLogger().error(error, e); 1871 throw new IndexingException(error, e); 1872 } 1873 finally 1874 { 1875 // Restore context 1876 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1877 } 1878 } 1879 1880 private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1881 { 1882 getLogger().debug("Unindexing all resources from Solr."); 1883 1884 String collection = _solrClientProvider.getCollectionName(workspaceName); 1885 solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1886 1887 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1888 } 1889 1890 /** 1891 * Delete all resource documents at a given path for all workspaces and commit 1892 * @param rootId The resource root ID, must not be null. 1893 * @param path The resource path relative to the given root, must start with a slash. 1894 * @throws Exception If an error occurs while unindexing. 1895 */ 1896 public void unindexResourcesByPath(String rootId, String path) throws Exception 1897 { 1898 String[] workspaceNames = _repository.getWorkspaces(); 1899 for (String workspaceName : workspaceNames) 1900 { 1901 unindexResourcesByPath(rootId, path, workspaceName); 1902 } 1903 } 1904 1905 /** 1906 * Delete all resource documents at a given path. 1907 * @param rootId The resource root ID, must not be null. 1908 * @param path The resource path relative to the given root, must start with a slash. 1909 * @param workspaceName The workspace where to work in 1910 * @throws Exception If an error occurs while unindexing. 1911 */ 1912 public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception 1913 { 1914 Request request = ContextHelper.getRequest(_context); 1915 1916 // Retrieve the current workspace. 1917 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1918 1919 try 1920 { 1921 // Force the workspace. 1922 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1923 1924 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1925 1926 Query query = new ResourceLocationQuery(rootId, path); 1927 1928 String collection = _solrClientProvider.getCollectionName(workspaceName); 1929 _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1930 1931 getLogger().debug("Succesfully deleted resource document from Solr."); 1932 } 1933 catch (Exception e) 1934 { 1935 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1936 getLogger().error(error, e); 1937 throw new IndexingException(error, e); 1938 } 1939 finally 1940 { 1941 // Restore workspace 1942 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1943 } 1944 } 1945 1946 /** 1947 * Remove a resource from Solr index for all workspaces and commit 1948 * @param resourceId The id of resource to unindex 1949 * @throws Exception if an error occurs while indexing. 1950 */ 1951 public void unindexResource(String resourceId) throws Exception 1952 { 1953 String[] workspaceNames = _repository.getWorkspaces(); 1954 for (String workspaceName : workspaceNames) 1955 { 1956 unindexResource(resourceId, workspaceName); 1957 } 1958 } 1959 1960 /** 1961 * Remove a resource from the Solr index. 1962 * @param resourceId The id of resource to unindex 1963 * @param workspaceName The workspace where to work in 1964 * @throws Exception if an error occurs while unindexing. 1965 */ 1966 public void unindexResource(String resourceId, String workspaceName) throws Exception 1967 { 1968 Request request = ContextHelper.getRequest(_context); 1969 1970 // Retrieve the current workspace. 1971 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1972 1973 try 1974 { 1975 // Force the workspace. 1976 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1977 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1978 1979 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 1980 1981 doUnindexDocument(resourceId, workspaceName, solrClient); 1982 1983 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 1984 } 1985 catch (Exception e) 1986 { 1987 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 1988 getLogger().error(error, e); 1989 throw new IndexingException(error, e); 1990 } 1991 finally 1992 { 1993 // Restore workspace 1994 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1995 } 1996 } 1997 1998 /** 1999 * Add or update a resource into Solr index 2000 * @param node The resource to index 2001 * @param documentType The document type of the resource 2002 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 2003 * @param solrClient The solr client to use 2004 * @throws Exception if an error occurs while indexing. 2005 */ 2006 protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2007 { 2008 if (node instanceof ResourceCollection) 2009 { 2010 try (AmetysObjectIterable<AmetysObject> children = ((ResourceCollection) node).getChildren()) 2011 { 2012 for (AmetysObject child : children) 2013 { 2014 doIndexExplorerItem(child, documentType, resourceRoot, solrClient); 2015 } 2016 } 2017 } 2018 else if (node instanceof Resource) 2019 { 2020 doIndexResource((Resource) node, documentType, resourceRoot, solrClient); 2021 } 2022 } 2023 2024 /** 2025 * Add or update a resource into Solr index 2026 * @param resource The resource to index 2027 * @param documentType The document type of the resource 2028 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 2029 * @param solrClient The solr client to use 2030 * @throws Exception if an error occurs while indexing. 2031 */ 2032 protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2033 { 2034 SolrInputDocument document = new SolrInputDocument(); 2035 2036 _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot); 2037 2038 String workspaceName = _workspaceSelector.getWorkspace(); 2039 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document); 2040 int status = solrResponse.getStatus(); 2041 2042 if (status != 0) 2043 { 2044 throw new IOException("Resource indexation: got status code '" + status + "'."); 2045 } 2046 } 2047 2048 /** 2049 * Process a Solr commit operation in all workspaces. 2050 * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient} 2051 * @throws SolrServerException if there is an error on the server 2052 * @throws IOException if there is a communication error with the server 2053 */ 2054 public void commit() throws SolrServerException, IOException 2055 { 2056 String[] workspaceNames; 2057 try 2058 { 2059 workspaceNames = _repository.getWorkspaces(); 2060 for (String workspaceName : workspaceNames) 2061 { 2062 SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName); 2063 commit(workspaceName, solrClient); 2064 } 2065 } 2066 catch (RepositoryException e) 2067 { 2068 throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e); 2069 } 2070 } 2071 2072 /** 2073 * Process a Solr commit operation in given workspace. 2074 * @param workspaceName The workspace's name 2075 * @param solrClient The solr client to use 2076 * @throws SolrServerException if there is an error on the server 2077 * @throws IOException if there is a communication error with the server 2078 */ 2079 public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2080 { 2081 long time_0 = System.currentTimeMillis(); 2082 2083 // Commit 2084 UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName)); 2085 int status = solrResponse.getStatus(); 2086 2087 if (status != 0) 2088 { 2089 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2090 } 2091 2092 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 2093 } 2094 2095 /** 2096 * Launch a solr index optimization. 2097 * @param workspaceName The workspace's name 2098 * @param solrClient The solr client to use 2099 * @throws SolrServerException if there is an error on the server 2100 * @throws IOException if there is a communication error with the server 2101 */ 2102 public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2103 { 2104 UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName)); 2105 int status = solrResponse.getStatus(); 2106 2107 if (status != 0) 2108 { 2109 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2110 } 2111 2112 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 2113 } 2114 2115 /** 2116 * Delete all documents from the solr index. 2117 * @param workspaceName The workspace name 2118 * @param solrClient The solr client to use 2119 * @throws Exception if an error occurs while unindexing. 2120 */ 2121 public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception 2122 { 2123 getLogger().debug("Deleting all documents from Solr."); 2124 2125 String collection = _solrClientProvider.getCollectionName(workspaceName); 2126 solrClient.deleteByQuery(collection, "*:*"); 2127 2128 getLogger().debug("Successfully deleted all documents from Solr."); 2129 } 2130 2131 /** 2132 * Delete a document from the Solr server. 2133 * @param id The id of the document to delete from Solr 2134 * @param workspaceName The workspace name 2135 * @param solrClient The solr client to use 2136 * @throws Exception if an error occurs while indexing. 2137 */ 2138 protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception 2139 { 2140 UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 2141 int status = solrResponse.getStatus(); 2142 2143 if (status != 0) 2144 { 2145 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 2146 } 2147 } 2148 2149 /** 2150 * Delete content attachments documents of a given content from the Solr server. 2151 * @param contentId The id of the content 2152 * @param workspaceName The workspace name 2153 * @param solrClient The solr client to use 2154 * @throws Exception if an error occurs while indexing. 2155 */ 2156 protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception 2157 { 2158 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 2159 2160 Query query = new ContentAttachmentQuery(contentId); 2161 UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build()); 2162 int status = solrResponse.getStatus(); 2163 2164 if (status != 0) 2165 { 2166 throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'."); 2167 } 2168 } 2169 2170// /** 2171// * Get the collection to use. 2172// * @return The name of the collection to index into. 2173// */ 2174// protected String getCollection() 2175// { 2176// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 2177// } 2178 2179 class IndexationResult 2180 { 2181 protected int _successCount; 2182 2183 protected int _errorCount; 2184 2185 /** 2186 * Constructor 2187 */ 2188 public IndexationResult() 2189 { 2190 this(0, 0); 2191 } 2192 2193 /** 2194 * Constructor 2195 * @param successCount The success count. 2196 * @param errorCount The error count. 2197 */ 2198 public IndexationResult(int successCount, int errorCount) 2199 { 2200 this._successCount = successCount; 2201 this._errorCount = errorCount; 2202 } 2203 2204 /** 2205 * Get the successCount. 2206 * @return the successCount 2207 */ 2208 public int getSuccessCount() 2209 { 2210 return _successCount; 2211 } 2212 2213 /** 2214 * Set the successCount. 2215 * @param successCount the successCount to set 2216 */ 2217 public void setSuccessCount(int successCount) 2218 { 2219 this._successCount = successCount; 2220 } 2221 2222 /** 2223 * Test if the indexation had errors. 2224 * @return true if the indexation had errors, false otherwise. 2225 */ 2226 public boolean hasErrors() 2227 { 2228 return _errorCount > 0; 2229 } 2230 2231 /** 2232 * Get the errorCount. 2233 * @return the errorCount 2234 */ 2235 public int getErrorCount() 2236 { 2237 return _errorCount; 2238 } 2239 2240 /** 2241 * Set the errorCount. 2242 * @param errorCount the errorCount to set 2243 */ 2244 public void setErrorCount(int errorCount) 2245 { 2246 this._errorCount = errorCount; 2247 } 2248 } 2249 2250}