001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.nio.ByteBuffer; 020import java.nio.CharBuffer; 021import java.nio.charset.CharsetDecoder; 022import java.nio.charset.CodingErrorAction; 023import java.nio.charset.StandardCharsets; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Collection; 028import java.util.Comparator; 029import java.util.HashMap; 030import java.util.HashSet; 031import java.util.List; 032import java.util.Map; 033import java.util.Objects; 034import java.util.Set; 035import java.util.function.Function; 036import java.util.stream.Collectors; 037import java.util.stream.StreamSupport; 038 039import javax.jcr.RepositoryException; 040 041import org.apache.avalon.framework.activity.Initializable; 042import org.apache.avalon.framework.component.Component; 043import org.apache.avalon.framework.context.Context; 044import org.apache.avalon.framework.context.ContextException; 045import org.apache.avalon.framework.context.Contextualizable; 046import org.apache.avalon.framework.service.ServiceException; 047import org.apache.avalon.framework.service.ServiceManager; 048import org.apache.avalon.framework.service.Serviceable; 049import org.apache.cocoon.components.ContextHelper; 050import org.apache.cocoon.environment.Request; 051import org.apache.commons.lang3.ObjectUtils; 052import org.apache.commons.lang3.StringUtils; 053import org.apache.solr.client.solrj.SolrClient; 054import org.apache.solr.client.solrj.SolrQuery; 055import org.apache.solr.client.solrj.SolrServerException; 056import org.apache.solr.client.solrj.request.CoreAdminRequest; 057import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 058import org.apache.solr.client.solrj.request.schema.SchemaRequest; 059import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 060import org.apache.solr.client.solrj.response.CoreAdminResponse; 061import org.apache.solr.client.solrj.response.QueryResponse; 062import org.apache.solr.client.solrj.response.SolrResponseBase; 063import org.apache.solr.client.solrj.response.UpdateResponse; 064import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 065import org.apache.solr.client.solrj.response.schema.SchemaResponse; 066import org.apache.solr.client.solrj.util.ClientUtils; 067import org.apache.solr.common.SolrInputDocument; 068import org.apache.solr.common.util.NamedList; 069import org.slf4j.Logger; 070 071import org.ametys.cms.indexing.IndexingException; 072import org.ametys.cms.indexing.solr.ReloadAclCacheRequest; 073import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest; 074import org.ametys.cms.repository.Content; 075import org.ametys.cms.repository.ContentQueryHelper; 076import org.ametys.cms.repository.WorkflowAwareContent; 077import org.ametys.cms.rights.solrchecking.ReadAccessHelper; 078import org.ametys.cms.search.query.ContentAttachmentQuery; 079import org.ametys.cms.search.query.DocumentTypeQuery; 080import org.ametys.cms.search.query.OrQuery; 081import org.ametys.cms.search.query.Query; 082import org.ametys.cms.search.query.ResourceLocationQuery; 083import org.ametys.cms.search.solr.NoAutoCommitUpdateClient; 084import org.ametys.cms.search.solr.SolrClientProvider; 085import org.ametys.cms.search.solr.schema.CopyFieldDefinition; 086import org.ametys.cms.search.solr.schema.FieldDefinition; 087import org.ametys.cms.search.solr.schema.SchemaDefinition; 088import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 089import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 090import org.ametys.cms.search.solr.schema.SchemaFields; 091import org.ametys.cms.search.solr.schema.SchemaHelper; 092import org.ametys.core.group.GroupIdentity; 093import org.ametys.core.right.AllowedUsers; 094import org.ametys.core.user.UserIdentity; 095import org.ametys.plugins.explorer.resources.Resource; 096import org.ametys.plugins.explorer.resources.ResourceCollection; 097import org.ametys.plugins.repository.AmetysObject; 098import org.ametys.plugins.repository.AmetysObjectIterable; 099import org.ametys.plugins.repository.AmetysObjectResolver; 100import org.ametys.plugins.repository.RepositoryConstants; 101import org.ametys.plugins.repository.TraversableAmetysObject; 102import org.ametys.plugins.repository.UnknownAmetysObjectException; 103import org.ametys.plugins.repository.collection.AmetysObjectCollection; 104import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector; 105import org.ametys.plugins.repository.provider.WorkspaceSelector; 106import org.ametys.runtime.config.Config; 107import org.ametys.runtime.plugin.component.AbstractLogEnabled; 108 109/** 110 * Solr indexer. 111 */ 112public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 113{ 114 /** The component role. */ 115 public static final String ROLE = SolrIndexer.class.getName(); 116 117 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 118 119 private static final String _CONFIGSET_NAME_PREFIX = "configset-"; 120 121 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_root_", "_version_", "text"); 122 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general"); 123 124 private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766; 125 126 /** The ametys object resolver. */ 127 protected AmetysObjectResolver _resolver; 128 /** The schema definition provider extension point. */ 129 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 130 /** The schema helper. */ 131 protected SchemaHelper _schemaHelper; 132 /** Solr Ametys contents indexer */ 133 protected SolrContentIndexer _solrContentIndexer; 134 /** Solr workflow indexer. */ 135 protected SolrWorkflowIndexer _solrWorkflowIndexer; 136 /** Solr resource indexer. */ 137 protected SolrResourceIndexer _solrResourceIndexer; 138 139 /** The Solr client provider */ 140 protected SolrClientProvider _solrClientProvider; 141 142 /** The solr core prefix. */ 143 protected String _solrCorePrefix; 144 /** The Ametys internal URL used by Solr to query Ametys */ 145 protected String _ametysInternalUrl; 146 147 /** The workspace selector. */ 148 protected WorkspaceSelector _workspaceSelector; 149 150 /** The helper for read access */ 151 protected ReadAccessHelper _readAccessHelper; 152 153 /** The avalon context */ 154 protected Context _context; 155 156 /** 157 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 158 * @return The date format for indexing dates 159 */ 160 public static SimpleDateFormat dateFormat() 161 { 162 if (__DATE_FORMAT.get() == null) 163 { 164 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 165 } 166 return __DATE_FORMAT.get(); 167 } 168 169 /** 170 * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr. 171 * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept. 172 * @param value The string value to index 173 * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log. 174 * @param documentId The id of the document being indexed. Can be null if you do not want to log. 175 * @param fieldName The name of the field being indexed. Can be null if you do not want to log. 176 * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes) 177 */ 178 public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName) 179 { 180 if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT) 181 { 182 // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length()) 183 // As a result, value.getBytes().length <= limit 184 return value; 185 } 186 187 // There is a doubt, the string may need to be truncated (or not) 188 byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); 189 int bytesLength = valueBytes.length; 190 if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT) 191 { 192 return value; 193 } 194 195 if (ObjectUtils.allNotNull(logger, documentId, fieldName)) 196 { 197 logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT); 198 } 199 200 // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974) 201 CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT); 202 CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() 203 .onMalformedInput(CodingErrorAction.IGNORE); 204 decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true); 205 decoder.flush(charBuffer); 206 return new String(charBuffer.array(), 0, charBuffer.position()); 207 } 208 209 @Override 210 public void service(ServiceManager serviceManager) throws ServiceException 211 { 212 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 213 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 214 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 215 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 216 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 217 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 218 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 219 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 220 _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE); 221 } 222 223 @Override 224 public void initialize() throws Exception 225 { 226 Config config = Config.getInstance(); 227 _solrCorePrefix = config.getValue("cms.solr.core.prefix"); 228 229 _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url"); 230 if (StringUtils.isBlank(_ametysInternalUrl)) 231 { 232 // fallback to CMS URL as internal URL is an optional parameter 233 _ametysInternalUrl = config.getValue("cms.url"); 234 } 235 } 236 237 @Override 238 public void contextualize(Context context) throws ContextException 239 { 240 _context = context; 241 } 242 243 /** 244 * Gets the 'autocommit' Solr client 245 * @param workspaceName The name of the workspace 246 * @return the Solr client 247 */ 248 protected SolrClient _getAutoCommitSolrClient(String workspaceName) 249 { 250 return _solrClientProvider.getUpdateClient(workspaceName, true); 251 } 252 253 /** 254 * Gets the 'no autocommit' Solr client 255 * @param workspaceName The name of the workspace 256 * @return the Solr client 257 */ 258 protected SolrClient _getNoAutoCommitSolrClient(String workspaceName) 259 { 260 return _solrClientProvider.getUpdateClient(workspaceName, false); 261 } 262 263 // for admin operations 264 private SolrClient _defaultSolrClient() 265 { 266 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 267 } 268 269 /** 270 * Get the names of the Solr cores. 271 * @return The names of the Solr cores. 272 * @throws IOException If an I/O error occurs. 273 * @throws SolrServerException If a Solr error occurs. 274 */ 275 @SuppressWarnings("unchecked") 276 public Set<String> getCoreNames() throws IOException, SolrServerException 277 { 278 Set<String> coreNames = new HashSet<>(); 279 280 getLogger().debug("Getting core list."); 281 282 SolrQuery query = new SolrQuery(); 283 query.setRequestHandler("/admin/cores"); 284 query.setParam("action", "STATUS"); 285 286 QueryResponse response = _defaultSolrClient().query(query); 287 288 NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status"); 289 for (Map.Entry<String, NamedList<?>> core : status) 290 { 291 String fullName = (String) core.getValue().get("name"); 292 if (fullName.startsWith(_solrCorePrefix)) 293 { 294 coreNames.add(fullName.substring(_solrCorePrefix.length())); 295 } 296 } 297 298 return coreNames; 299 } 300 301 /** 302 * Get the names of the Solr cores. 303 * @return The names of the Solr cores. 304 * @throws IOException If an I/O error occurs. 305 * @throws SolrServerException If a Solr error occurs. 306 */ 307 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 308 { 309 Set<String> coreNames = new HashSet<>(); 310 311 getLogger().debug("Getting core list."); 312 313 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 314 315 NamedList<NamedList<Object>> status = response.getCoreStatus(); 316 for (Map.Entry<String, NamedList<Object>> core : status) 317 { 318 String fullName = (String) core.getValue().get("name"); 319 if (fullName.startsWith(_solrCorePrefix)) 320 { 321 coreNames.add(fullName.substring(_solrCorePrefix.length())); 322 } 323 } 324 325 return coreNames; 326 } 327 328 /** 329 * Create a Solr core. 330 * @param name The name of the core to create. 331 * @throws IOException If an I/O error occurs. 332 * @throws SolrServerException If a Solr error occurs. 333 */ 334 public void createCore(String name) throws IOException, SolrServerException 335 { 336 String fullName = _solrCorePrefix + name; 337 String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix); 338 _createConfigset(configsetName); 339 340 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 341 342 SolrQuery query = new SolrQuery(); 343 query.setRequestHandler("/admin/cores"); 344 query.setParam("action", "CREATE"); 345 query.setParam("name", fullName); 346 query.setParam("configSet", configsetName); 347 query.setParam("property.ametys.url", _ametysInternalUrl); 348 349 QueryResponse response = _defaultSolrClient().query(query); 350 NamedList<?> results = response.getResponse(); 351 352 NamedList<?> error = (NamedList<?>) results.get("error"); 353 if (error != null) 354 { 355 throw new IOException("Error creating the core: " + error.get("msg")); 356 } 357 } 358 359 /** 360 * Updates the ametys.url property of the Solr cores. 361 */ 362 public void updateAmetysUrlCoreProperty() 363 { 364 Set<String> coreNames; 365 try 366 { 367 coreNames = getCoreNames(); 368 } 369 catch (SolrServerException | IOException e) 370 { 371 getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e); 372 return; 373 } 374 375 for (String coreName : coreNames) 376 { 377 String collection = _solrClientProvider.getCollectionName(coreName); 378 SolrResponseBase response; 379 try 380 { 381 response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection); 382 } 383 catch (SolrServerException | IOException e) 384 { 385 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e); 386 continue; 387 } 388 389 NamedList<Object> responseParams = response.getResponse(); 390 if ("ok".equals(responseParams.get("result"))) 391 { 392 Boolean valueChanged = responseParams.getBooleanArg("valueChanged"); 393 if (valueChanged) 394 { 395 getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName); 396 } 397 else 398 { 399 getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName); 400 } 401 } 402 else 403 { 404 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName); 405 } 406 } 407 } 408 409 private void _createConfigset(String name) throws IOException, SolrServerException 410 { 411 getLogger().info("Creating (if necessary) configset '{}'", name); 412 413 // This request handler will check if configset exists. If not it will be created. 414 SolrQuery query = new SolrQuery(); 415 query.setRequestHandler("/admin/cores"); 416 query.setParam("action", "createConfigset"); 417 query.setParam("name", name); 418 419 QueryResponse response = _defaultSolrClient().query(query); 420 NamedList<?> results = response.getResponse(); 421 422 NamedList<?> error = (NamedList<?>) results.get("error"); 423 if (error != null) 424 { 425 throw new IOException("Error creating the core: " + error.get("msg")); 426 } 427 } 428 429 /** 430 * Delete a Solr core. 431 * @param name The name of the core to delete. 432 * @throws IOException If an I/O error occurs. 433 * @throws SolrServerException If a Solr error occurs. 434 */ 435 public void deleteCore(String name) throws IOException, SolrServerException 436 { 437 String fullName = _solrCorePrefix + name; 438 439 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 440 441 SolrQuery query = new SolrQuery(); 442 query.setRequestHandler("/admin/cores"); 443 query.setParam("action", "UNLOAD"); 444 query.setParam("core", fullName); 445 query.setParam("deleteInstanceDir", "true"); 446 447 QueryResponse response = _defaultSolrClient().query(query); 448 NamedList<?> results = response.getResponse(); 449 450 NamedList<?> error = (NamedList<?>) results.get("error"); 451 if (error != null) 452 { 453 throw new IOException("Error deleting core" + name + ": " + error.get("msg")); 454 } 455 } 456 457 /** 458 * Send the schema. 459 * @throws IOException If a communication error occurs. 460 * @throws SolrServerException If a solr error occurs. 461 */ 462 public void sendSchema() throws IOException, SolrServerException 463 { 464 getLogger().info("Computing and sending the schema to the solr server."); 465 466 String workspaceName = _workspaceSelector.getWorkspace(); 467 String collection = _solrClientProvider.getCollectionName(workspaceName); 468 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 469 470// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 471 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 472 473 // TODO Clear the schema except fields marked ametysReadOnly="true". 474 475 // Clear the current schema. 476 clearSchema(solrClient, collection); 477 478 SchemaRequest schemaRequest = new SchemaRequest(); 479 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 480 481 // The cleared schema contains only the basic fields which can't be deleted. 482 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 483 SchemaFields schemaFields = new SchemaFields(clearedSchema); 484 485 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 486 487 // Add the static schema types and fields. 488 List<SchemaRequest.Update> updates = new ArrayList<>(); 489 490 // Set "add field" definitions from the static schema to the update list. 491 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 492 493 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 494 495 // Set "add field" definitions from the static schema to the update list. 496 addCustomUpdates(updates, schemaFields); 497 498 reorderUpdates(updates); 499 500 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 501 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 502 503 getLogger().debug("Send schema response: {}", updateResponse.toString()); 504 Object errors = updateResponse.getResponse().get("errors"); 505 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 506 { 507 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 508 throw new SolrServerException(msg); 509 } 510 511 getLogger().info("Schema sent to the solr server."); 512 513 reloadCores(); 514 } 515 516 /** 517 * Compute the list of {@link Update} directives from the static schema. 518 * @param updates The list of {@link Update} directives to fill. 519 * @param staticSchema The static schema representation. 520 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 521 */ 522 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 523 { 524 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 525 for (FieldTypeDefinition fieldType : fieldTypes) 526 { 527 String name = (String) fieldType.getAttributes().get("name"); 528 if (!schemaFields.hasFieldType(name)) 529 { 530 updates.add(new SchemaRequest.AddFieldType(fieldType)); 531 schemaFields.addFieldType(name); 532 } 533 } 534 for (Map<String, Object> field : staticSchema.getFields()) 535 { 536 String name = (String) field.get("name"); 537 if (!schemaFields.hasField(name)) 538 { 539 updates.add(new SchemaRequest.AddField(field)); 540 schemaFields.addField(name); 541 } 542 } 543 for (Map<String, Object> field : staticSchema.getDynamicFields()) 544 { 545 String name = (String) field.get("name"); 546 if (!schemaFields.hasDynamicField(name)) 547 { 548 updates.add(new SchemaRequest.AddDynamicField(field)); 549 schemaFields.addDynamicField(name); 550 } 551 } 552 for (Map<String, Object> field : staticSchema.getCopyFields()) 553 { 554 String source = (String) field.get("source"); 555 String dest = (String) field.get("dest"); 556 if (!schemaFields.hasCopyField(source, dest)) 557 { 558 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 559 schemaFields.addCopyField(source, dest); 560 } 561 } 562 } 563 564 /** 565 * Compute the list of custom {@link Update} directives. 566 * @param updates The list of {@link Update} directives to fill. 567 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 568 */ 569 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 570 { 571 // Add all our property-managed fields. 572 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 573 { 574 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 575 576 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 577 { 578 if (!definitionExists(definition, schemaFields)) 579 { 580 SchemaRequest.Update update = getSchemaUpdate(definition); 581 if (update != null) 582 { 583 updates.add(update); 584 } 585 } 586 } 587 } 588 589// for (String propId : _sysPropEP.getExtensionsIds()) 590// { 591// Collection<SchemaDefinition> definitions = _sysPropEP.getExtension(propId).getSchemaDefinitions(); 592// 593// for (SchemaDefinition definition : definitions) 594// { 595// if (!definitionExists(definition, schemaFields)) 596// { 597// SchemaRequest.Update update = getSchemaUpdate(definition); 598// if (update != null) 599// { 600// updates.add(update); 601// } 602// } 603// } 604// } 605 } 606 607 /** 608 * Reorder the list of {@link Update} directives. 609 * @param updates The list of {@link Update} directives to reorder. 610 */ 611 protected void reorderUpdates(List<SchemaRequest.Update> updates) 612 { 613 updates.sort(Comparator.comparing(u -> 614 { 615 if (u instanceof SchemaRequest.AddCopyField) 616 { 617 // copyField at the end of the list to avoid an exception of an undeclared 'source' or 'dest' whereas it is declared later 618 return 1; 619 } 620 else 621 { 622 return 0; 623 } 624 })); 625 } 626 627 /** 628 * Test if the given schema definition exists in the given {@link SchemaFields} reference. 629 * @param definition The schema definition to test. 630 * @param schemaFields The current schema fields. 631 * @return true if the SchemaFields contain the schema definition. 632 */ 633 protected boolean definitionExists(SchemaDefinition definition, SchemaFields schemaFields) 634 { 635 if (definition instanceof FieldDefinition) 636 { 637 FieldDefinition fieldDef = (FieldDefinition) definition; 638 if (fieldDef.isDynamic()) 639 { 640 return schemaFields.hasField(fieldDef.getName()); 641 } 642 else 643 { 644 return schemaFields.hasDynamicField(fieldDef.getName()); 645 } 646 } 647 else if (definition instanceof CopyFieldDefinition) 648 { 649 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 650 return schemaFields.hasCopyField(fieldDef.getSource(), fieldDef.getDestination()); 651 } 652 653 return false; 654 } 655 656 /** 657 * Delete all the fields of the existing schema in the given collection. 658 * @param solrClient The Solr client 659 * @param collection The collection. 660 * @throws IOException If a communication error occurs. 661 * @throws SolrServerException If a solr error occurs. 662 */ 663 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 664 { 665 try 666 { 667 getLogger().info("Clearing the existing schema on the solr server."); 668 669 SchemaRequest schemaRequest = new SchemaRequest(); 670 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 671 672 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 673 674 List<SchemaRequest.Update> deletions = new ArrayList<>(); 675 676 // First the copy fields, then dynamic and simple fields, and field types in the end. 677 for (Map<String, Object> field : schema.getCopyFields()) 678 { 679 String source = (String) field.get("source"); 680 String dest = (String) field.get("dest"); 681 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 682 } 683 for (Map<String, Object> field : schema.getDynamicFields()) 684 { 685 String name = (String) field.get("name"); 686 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 687 } 688 for (Map<String, Object> field : schema.getFields()) 689 { 690 String name = (String) field.get("name"); 691 if (!_READ_ONLY_FIELDS.contains(name)) 692 { 693 deletions.add(new SchemaRequest.DeleteField(name)); 694 } 695 } 696 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 697 { 698 String name = (String) fieldType.getAttributes().get("name"); 699 if (!_READ_ONLY_FIELDTYPES.contains(name)) 700 { 701 deletions.add(new SchemaRequest.DeleteFieldType(name)); 702 } 703 } 704 705 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 706 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 707 708 Object errors = updateResponse.getResponse().get("errors"); 709 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 710 { 711 String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString(); 712 throw new SolrServerException(msg); 713 } 714 else 715 { 716 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 717 } 718 719 getLogger().info("Solr schema cleared."); 720 } 721 catch (SolrServerException | IOException e) 722 { 723 getLogger().error("Error clearing schema in collection " + collection, e); 724 throw e; 725 } 726 } 727 728 /** 729 * Get the schema {@link Update} directive from the given schema definition. 730 * @param definition The schema definition to add. 731 * @return The add {@link Update} directive. 732 */ 733 protected SchemaRequest.Update getSchemaUpdate(SchemaDefinition definition) 734 { 735 SchemaRequest.Update update = null; 736 737 if (definition instanceof FieldDefinition) 738 { 739 FieldDefinition fieldDef = (FieldDefinition) definition; 740 741 // Force indexed and stored attributes to true. 742 Map<String, Object> attributes = new HashMap<>(); 743 attributes.put("name", fieldDef.getName()); 744 attributes.put("type", fieldDef.getType()); 745 attributes.put("multiValued", fieldDef.isMultiValued()); 746 attributes.put("docValues", fieldDef.isDocValues()); 747 attributes.put("indexed", Boolean.TRUE); 748 attributes.put("stored", Boolean.TRUE); 749 750 update = new SchemaRequest.AddField(attributes); 751 } 752 else if (definition instanceof CopyFieldDefinition) 753 { 754 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 755 756 String source = fieldDef.getSource(); 757 String dest = fieldDef.getDestination(); 758 759 update = new SchemaRequest.AddCopyField(source, Arrays.asList(dest)); 760 } 761 762 return update; 763 } 764 765 /** 766 * Reload the solr cores. 767 * @throws IOException If a communication error occurs. 768 * @throws SolrServerException If a solr error occurs. 769 */ 770 protected void reloadCores() throws IOException, SolrServerException 771 { 772 getLogger().info("Reloading solr cores."); 773 774 for (String coreName : getCoreNames()) 775 { 776 String fullName = _solrCorePrefix + coreName; 777 778 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 779 780 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 781 } 782 783 getLogger().info("All cores reloaded."); 784 } 785 786 /** 787 * Reloads the ACL Solr cache for all users 788 * @throws IOException If an I/O error occurs. 789 * @throws SolrServerException If a Solr error occurs. 790 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 791 */ 792 public void reloadAclCache() throws IOException, SolrServerException, RepositoryException 793 { 794 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 795 for (String workspaceName : workspaceNames) 796 { 797 reloadAclCache(workspaceName); 798 } 799 } 800 801 /** 802 * Reloads the ACL Solr cache for all users 803 * @param workspaceName The workspace name 804 * @throws IOException If an I/O error occurs. 805 * @throws SolrServerException If a Solr error occurs. 806 */ 807 public void reloadAclCache(String workspaceName) throws IOException, SolrServerException 808 { 809 reloadAclCache(workspaceName, false); 810 } 811 812 /** 813 * Reloads the ACL Solr cache for all users 814 * @param workspaceName The workspace name 815 * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache) 816 * @throws IOException If an I/O error occurs. 817 * @throws SolrServerException If a Solr error occurs. 818 */ 819 public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException 820 { 821 getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 822 823 String collection = _solrClientProvider.getCollectionName(workspaceName); 824 SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection); 825 NamedList<Object> responseObj = responseBase.getResponse(); 826 827 if ("ok".equals(responseObj.get("result"))) 828 { 829 getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 830 } 831 else 832 { 833 Object error = responseObj.get("error"); 834 getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error); 835 } 836 } 837 838 /** 839 * Index all the contents in a given workspace. 840 * @param workspaceName the workspace where to index 841 * @param indexAttachments to index content attachments 842 * @param solrClient The solr client to use 843 * @return The indexation result as a Map. 844 * @throws Exception if an error occurs while indexing. 845 */ 846 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 847 { 848 Request request = ContextHelper.getRequest(_context); 849 850 // Retrieve the current workspace. 851 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 852 853 try 854 { 855 // Force the workspace. 856 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 857 858 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 859 860 long start = System.currentTimeMillis(); 861 862 // Delete all contents 863 unindexAllContents(workspaceName, indexAttachments, solrClient); 864 865 String query = ContentQueryHelper.getContentXPathQuery(null); 866 AmetysObjectIterable<Content> contents = _resolver.query(query); 867 868 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient); 869 870 long end = System.currentTimeMillis(); 871 872 if (!result.hasErrors()) 873 { 874 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 875 } 876 else 877 { 878 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 879 } 880 881 Map<String, Object> results = new HashMap<>(); 882 results.put("successCount", result.getSuccessCount()); 883 if (result.hasErrors()) 884 { 885 results.put("errorCount", result.getErrorCount()); 886 } 887 888 return results; 889 } 890 catch (Exception e) 891 { 892 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 893 getLogger().error(error, e); 894 throw new IndexingException(error, e); 895 } 896 finally 897 { 898 // Restore context 899 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 900 } 901 } 902 903 /** 904 * Unindex all content documents. 905 * @param workspaceName The workspace name 906 * @param unindexAttachments also unindex content attachments 907 * @param solrClient The solr client to use 908 * @throws Exception if an error occurs while unindexing. 909 */ 910 protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 911 { 912 String collection = _solrClientProvider.getCollectionName(workspaceName); 913 914 Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT); 915 Query query; 916 if (unindexAttachments) 917 { 918 Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE); 919 Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE); 920 query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents); 921 } 922 else 923 { 924 query = contents; 925 } 926 solrClient.deleteByQuery(collection, query.build()); 927 } 928 929 /** 930 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 931 * @param collectionId The id of collection 932 * @param indexAttachments to index content attachments 933 * @throws Exception if an error occurs while indexing. 934 */ 935 public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception 936 { 937 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 938 for (String workspaceName : workspaceNames) 939 { 940 indexSubcontents(collectionId, workspaceName, indexAttachments); 941 } 942 } 943 944 /** 945 * Index the child contents of a {@link AmetysObjectCollection} 946 * @param collectionId The id of collection 947 * @param workspaceName the workspace where to index 948 * @param indexAttachments to index content attachments 949 * @throws Exception if an error occurs while unindexing. 950 */ 951 public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception 952 { 953 Request request = ContextHelper.getRequest(_context); 954 955 // Retrieve the current workspace. 956 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 957 958 try 959 { 960 // Force the workspace. 961 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 962 963 if (_resolver.hasAmetysObjectForId(collectionId)) 964 { 965 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 966 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 967 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 968 969 for (AmetysObject child : children) 970 { 971 if (child instanceof Content) 972 { 973 Content content = (Content) child; 974 975 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 976 } 977 } 978 } 979 } 980 finally 981 { 982 // Restore context 983 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 984 } 985 } 986 987 /** 988 * Add or update a content into Solr index on all workspaces and commit 989 * @param contentId The id of the content to index 990 * @param indexAttachments to index content attachments 991 * @throws Exception if an error occurs while indexing. 992 */ 993 public void indexContent(String contentId, boolean indexAttachments) throws Exception 994 { 995 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 996 for (String workspaceName : workspaceNames) 997 { 998 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 999 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1000 } 1001 } 1002 1003 /** 1004 * Add or update a content into Solr index 1005 * @param contentId The id of the content to index 1006 * @param workspaceName the workspace where to index 1007 * @param indexAttachments to index content attachments 1008 * @throws Exception if an error occurs while indexing. 1009 */ 1010 public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception 1011 { 1012 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1013 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1014 } 1015 1016 /** 1017 * Add or update a content into Solr index 1018 * @param contentId The id of the content to index 1019 * @param workspaceName the workspace where to index 1020 * @param indexAttachments to index content attachments 1021 * @param solrClient The solr client to use 1022 * @throws Exception if an error occurs while indexing. 1023 */ 1024 public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1025 { 1026 Request request = ContextHelper.getRequest(_context); 1027 1028 // Retrieve the current workspace. 1029 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1030 1031 try 1032 { 1033 // Force the workspace. 1034 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1035 1036 if (_resolver.hasAmetysObjectForId(contentId)) 1037 { 1038 Content content = _resolver.resolveById(contentId); 1039 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1040 } 1041 } 1042 finally 1043 { 1044 // Restore context 1045 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1046 } 1047 } 1048 1049 private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException 1050 { 1051 try 1052 { 1053 long time_0 = System.currentTimeMillis(); 1054 1055 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 1056 1057 deleteRepeaterDocs(content.getId(), workspaceName, solrClient); 1058 doIndexContent(content, workspaceName, solrClient); 1059 doIndexContentWorkflow(content, workspaceName, solrClient); 1060 if (indexAttachments) 1061 { 1062 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1063 } 1064 1065 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 1066 } 1067 catch (Exception e) 1068 { 1069 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 1070 getLogger().error(error, e); 1071 throw new IndexingException(error, e); 1072 } 1073 } 1074 1075 /** 1076 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1077 * @param contents the collection of contents to index. 1078 * @return the indexation result. 1079 * @throws Exception if an error occurs while indexing. 1080 */ 1081 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 1082 { 1083 IndexationResult result = new IndexationResult(); 1084 1085 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1086 for (String workspaceName : workspaceNames) 1087 { 1088 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1089 IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient); 1090 result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount()); 1091 } 1092 1093 return result; 1094 } 1095 1096 /** 1097 * Send a collection of contents for indexation in the solr server. 1098 * @param contents the collection of contents to index. 1099 * @param workspaceName the workspace where to index 1100 * @param indexAttachments to index content attachments 1101 * @param solrClient The solr client to use 1102 * @return the indexation result. 1103 * @throws Exception if an error occurs while indexing. 1104 */ 1105 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1106 { 1107 Request request = ContextHelper.getRequest(_context); 1108 1109 // Retrieve the current workspace. 1110 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1111 1112 try 1113 { 1114 // Force the workspace. 1115 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1116 1117 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 1118 1119 long start = System.currentTimeMillis(); 1120 1121 List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false) 1122 .map(Content::getId) 1123 .map(this::_resolveSilently) 1124 .filter(Objects::nonNull) 1125 .collect(Collectors.toList()); 1126 1127 IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient); 1128 1129 long end = System.currentTimeMillis(); 1130 1131 if (!result.hasErrors()) 1132 { 1133 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1134 } 1135 else 1136 { 1137 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1138 } 1139 1140 return result; 1141 } 1142 catch (Exception e) 1143 { 1144 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 1145 getLogger().error(error, e); 1146 throw new IndexingException(error, e); 1147 } 1148 finally 1149 { 1150 // Restore context 1151 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1152 } 1153 1154 } 1155 1156 private Content _resolveSilently(String contentId) 1157 { 1158 try 1159 { 1160 return _resolver.resolveById(contentId); 1161 } 1162 catch (UnknownAmetysObjectException e) 1163 { 1164 return null; 1165 } 1166 } 1167 1168 /** 1169 * Send some contents for indexation in the solr server. 1170 * @param contents the contents to index. 1171 * @param workspaceName The workspace name 1172 * @param indexAttachments to index content attachments 1173 * @param solrClient The solr client to use 1174 * @return the indexation result. 1175 * @throws Exception if an error occurs committing the results. 1176 */ 1177 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1178 { 1179 int successCount = 0; 1180 int errorCount = 0; 1181 for (Content content : contents) 1182 { 1183 try 1184 { 1185 doIndexContent(content, workspaceName, solrClient); 1186 doIndexContentWorkflow(content, workspaceName, solrClient); 1187 if (indexAttachments) 1188 { 1189 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1190 } 1191 successCount++; 1192 } 1193 catch (Exception e) 1194 { 1195 getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e); 1196 errorCount++; 1197 } 1198 } 1199 1200 return new IndexationResult(successCount, errorCount); 1201 } 1202 1203 /** 1204 * Update the value of a specific system property in a content document. 1205 * @param content The content to update. 1206 * @param propertyId The system property ID. 1207 * @param workspaceName The workspace name 1208 * @throws Exception if an error occurs while indexing. 1209 */ 1210 public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception 1211 { 1212 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1213 updateSystemProperty(content, propertyId, workspaceName, solrClient); 1214 } 1215 1216 /** 1217 * Update the value of a specific system property in a content document. 1218 * @param content The content to update. 1219 * @param propertyId The system property ID. 1220 * @param workspaceName The workspace name 1221 * @param solrClient The solr client to use 1222 * @throws Exception if an error occurs while indexing. 1223 */ 1224 public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1225 { 1226 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 1227 1228 SolrInputDocument document = new SolrInputDocument(); 1229 boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 1230 1231 if (!hasUpdate) 1232 { 1233 getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content); 1234 return; 1235 } 1236 1237 String collection = _solrClientProvider.getCollectionName(workspaceName); 1238 UpdateResponse solrResponse = solrClient.add(collection, document); 1239 int status = solrResponse.getStatus(); 1240 1241 if (status != 0) 1242 { 1243 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 1244 } 1245 1246 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 1247 } 1248 1249 /** 1250 * Remove a content from Solr index for all workspaces and commit 1251 * @param contentId The id of content to unindex 1252 * @param unindexAttachments also unindex content attachments 1253 * @throws Exception if an error occurs while indexing. 1254 */ 1255 public void unindexContent(String contentId, boolean unindexAttachments) throws Exception 1256 { 1257 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1258 for (String workspaceName : workspaceNames) 1259 { 1260 unindexContent(contentId, workspaceName, unindexAttachments); 1261 } 1262 } 1263 1264 /** 1265 * Remove a content from Solr index 1266 * @param contentId The id of content to unindex 1267 * @param workspaceName The workspace where to work in 1268 * @param unindexAttachments also unindex content attachments 1269 * @throws Exception if an error occurs while indexing. 1270 */ 1271 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception 1272 { 1273 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1274 unindexContent(contentId, workspaceName, unindexAttachments, solrClient); 1275 } 1276 1277 /** 1278 * Remove a content from Solr index 1279 * @param contentId The id of content to unindex 1280 * @param workspaceName The workspace where to work in 1281 * @param unindexAttachments also unindex content attachments 1282 * @param solrClient The solr client to use 1283 * @throws Exception if an error occurs while indexing. 1284 */ 1285 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 1286 { 1287 Request request = ContextHelper.getRequest(_context); 1288 1289 // Retrieve the current workspace. 1290 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1291 1292 try 1293 { 1294 // Force the workspace. 1295 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1296 1297 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 1298 1299 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1300 doUnindexDocument(contentId, workspaceName, solrClient); 1301 if (unindexAttachments) 1302 { 1303 doUnindexContentAttachments(contentId, workspaceName, solrClient); 1304 } 1305 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1306 1307 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 1308 } 1309 catch (Exception e) 1310 { 1311 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 1312 getLogger().error(error, e); 1313 throw new IndexingException(error, e); 1314 } 1315 finally 1316 { 1317 // Restore workspace 1318 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1319 } 1320 } 1321 1322 /** 1323 * Remove a content from Solr index for all workspaces and commit 1324 * @param contentIds The id of content to unindex 1325 * @throws Exception if an error occurs while indexing. 1326 */ 1327 public void unindexContents(Collection<String> contentIds) throws Exception 1328 { 1329 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1330 for (String workspaceName : workspaceNames) 1331 { 1332 unindexContents(contentIds, workspaceName); 1333 } 1334 } 1335 1336 /** 1337 * Remove a content from Solr index 1338 * @param contentIds The id of content to unindex 1339 * @param workspaceName The workspace where to work in 1340 * @throws Exception if an error occurs while indexing. 1341 */ 1342 public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception 1343 { 1344 Request request = ContextHelper.getRequest(_context); 1345 1346 // Retrieve the current workspace. 1347 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1348 1349 try 1350 { 1351 // Force the workspace. 1352 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1353 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1354 1355 getLogger().debug("Unindexing several contents from Solr."); 1356 1357 for (String contentId : contentIds) 1358 { 1359 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1360 doUnindexDocument(contentId, workspaceName, solrClient); 1361 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1362 } 1363 1364 getLogger().debug("Succesfully unindexed content from Solr."); 1365 } 1366 catch (Exception e) 1367 { 1368 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1369 getLogger().error(error, e); 1370 throw new IndexingException(error, e); 1371 } 1372 finally 1373 { 1374 // Restore workspace 1375 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1376 } 1377 } 1378 1379 /** 1380 * Add or update a content into Solr index 1381 * @param content The content to index 1382 * @param workspaceName The workspace where to index 1383 * @param solrClient The solr client to use 1384 * @throws Exception if an error occurs while indexing. 1385 */ 1386 protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception 1387 { 1388 long time_0 = System.currentTimeMillis(); 1389 1390 SolrInputDocument document = new SolrInputDocument(); 1391 List<SolrInputDocument> additionalDocuments = new ArrayList<>(); 1392 1393 _solrContentIndexer.indexContent(content, document, additionalDocuments); 1394 1395 long time_1 = System.currentTimeMillis(); 1396 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1397 1398 indexAclInitValues(content, document); 1399 1400 long time_2 = System.currentTimeMillis(); 1401 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1402 1403 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1404 documents.add(document); 1405 1406 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents); 1407 int status = solrResponse.getStatus(); 1408 1409 long time_3 = System.currentTimeMillis(); 1410 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1411 1412 if (status != 0) 1413 { 1414 throw new IOException("Content indexation: got status code '" + status + "'."); 1415 } 1416 } 1417 1418 /** 1419 * Indexes read-ACl initial values for object 1420 * @param ametysObject The object 1421 * @param document The Solr document 1422 */ 1423 public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document) 1424 { 1425 // Indexation of AmetysObject property 1426 document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true); 1427 1428 // Indexation of initValues for AllowedUsers 1429 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject); 1430 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed()); 1431 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed()); 1432 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString); 1433 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString); 1434 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString); 1435 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString); 1436 } 1437 1438 private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier) 1439 { 1440 if (values == null) 1441 { 1442 return; 1443 } 1444 1445 for (T value : values) 1446 { 1447 document.addField(fieldName, stringifier.apply(value)); 1448 } 1449 } 1450 1451 /** 1452 * Index the whole workflow of a content. 1453 * @param content The content. 1454 * @param workspaceName The workspace name 1455 * @param solrClient The solr client to use 1456 * @throws Exception if an error occurs while indexing. 1457 */ 1458 protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception 1459 { 1460 if (content instanceof WorkflowAwareContent) 1461 { 1462 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient); 1463 } 1464 } 1465 1466 /** 1467 * Index content attachments as new entries in the idnex 1468 * @param collection the collection of attachments 1469 * @param content the content whose attachments will be indexed 1470 * @throws Exception if something goes wrong when indexing the attachments of the content 1471 */ 1472 public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception 1473 { 1474 Request request = ContextHelper.getRequest(_context); 1475 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1476 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1477 indexContentAttachments(collection, content, solrClient); 1478 } 1479 1480 /** 1481 * Index content attachments as new entries in the idnex 1482 * @param collection the collection of attachments 1483 * @param content the content whose attachments will be indexed 1484 * @param solrClient The solr client to use 1485 * @throws Exception if something goes wrong when indexing the attachments of the content 1486 */ 1487 public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception 1488 { 1489 if (collection == null) 1490 { 1491 return; 1492 } 1493 1494 for (AmetysObject object : collection.getChildren()) 1495 { 1496 if (object instanceof ResourceCollection) 1497 { 1498 indexContentAttachments((ResourceCollection) object, content, solrClient); 1499 } 1500 else if (object instanceof Resource) 1501 { 1502 Resource resource = (Resource) object; 1503 indexContentAttachment(resource, content, solrClient); 1504 } 1505 } 1506 } 1507 1508 /** 1509 * Index a content attachment 1510 * @param resource the content attachment as a {@link Resource} 1511 * @param content the content whose attachment is going to be indexed 1512 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1513 */ 1514 public void indexContentAttachment(Resource resource, Content content) throws Exception 1515 { 1516 Request request = ContextHelper.getRequest(_context); 1517 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1518 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1519 indexContentAttachment(resource, content, solrClient); 1520 } 1521 1522 /** 1523 * Index a content attachment 1524 * @param resource the content attachment as a {@link Resource} 1525 * @param content the content whose attachment is going to be indexed 1526 * @param solrClient The solr client to use 1527 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1528 */ 1529 public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception 1530 { 1531 SolrInputDocument document = new SolrInputDocument(); 1532 1533 // Prepare resource doc 1534 _indexContentAttachment(resource, document, content); 1535 1536 // Indexation of the document 1537 _indexResourceDocument(resource, document, solrClient); 1538 } 1539 1540 private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception 1541 { 1542 String language = content.getLanguage(); 1543 1544 _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language); 1545 1546 // Need the id of the content for unindexing attachment during the unindexing of the content 1547 document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId()); 1548 } 1549 1550 /** 1551 * Index a populated solr input document of type Resource. 1552 * @param resource the resource from which the input document is created 1553 * @param document the input document 1554 * @param solrClient The solr client to use 1555 * @throws SolrServerException if there is an error on the server 1556 * @throws IOException if there is a communication error with the server 1557 */ 1558 protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException 1559 { 1560 String resourceId = resource.getId(); 1561 1562 // Retrieve appropriate collection name 1563 Request request = ContextHelper.getRequest(_context); 1564 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1565 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 1566 1567 // Add document 1568 UpdateResponse solrResponse = solrClient.add(collectionName, document); 1569 int status = solrResponse.getStatus(); 1570 1571 if (status != 0) 1572 { 1573 throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId); 1574 } 1575 1576 getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId); 1577 } 1578 1579 /** 1580 * Delete repeater documents of a specified content. 1581 * @param workspaceName The workspace name 1582 * @param contentId the content ID. 1583 * @param solrClient The solr client to use 1584 * @throws Exception if an error occurs while indexing. 1585 */ 1586 protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception 1587 { 1588 long time_0 = System.currentTimeMillis(); 1589 1590 // _documentType:repeater AND id:content\://xxx/* 1591 StringBuilder query = new StringBuilder(); 1592 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1593 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1594 1595 solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1596 1597 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1598 } 1599 1600 /** 1601 * Index all the resources in a given workspace. 1602 * @param workspaceName The workspace where to index 1603 * @param solrClient The solr client to use 1604 * @return The indexation result as a Map. 1605 * @throws Exception if an error occurs while indexing. 1606 */ 1607 public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1608 { 1609 Request request = ContextHelper.getRequest(_context); 1610 1611 // Retrieve the current workspace. 1612 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1613 1614 try 1615 { 1616 // Force the workspace. 1617 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1618 1619 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1620 1621 long start = System.currentTimeMillis(); 1622 1623 // Delete all resources 1624 _unindexAllResources(workspaceName, solrClient); 1625 1626 Map<String, Object> results = new HashMap<>(); 1627 1628 try 1629 { 1630 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1631 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1632 1633 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient); 1634 1635 long end = System.currentTimeMillis(); 1636 1637 if (!result.hasErrors()) 1638 { 1639 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1640 } 1641 else 1642 { 1643 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1644 } 1645 1646 results.put("successCount", result.getSuccessCount()); 1647 if (result.hasErrors()) 1648 { 1649 results.put("errorCount", result.getErrorCount()); 1650 } 1651 } 1652 catch (UnknownAmetysObjectException e) 1653 { 1654 getLogger().info("There is no root for resources in current workspace."); 1655 } 1656 1657 return results; 1658 } 1659 catch (Exception e) 1660 { 1661 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1662 getLogger().error(error, e); 1663 throw new IndexingException(error, e); 1664 } 1665 finally 1666 { 1667 // Restore context 1668 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1669 } 1670 } 1671 1672 /** 1673 * Send a collection of contents for indexation in the solr server. 1674 * @param resources the collection of contents to index. 1675 * @param documentType The document type of the resource 1676 * @param workspaceName The workspace where to index 1677 * @return the indexation result. 1678 * @throws Exception if an error occurs while indexing. 1679 */ 1680 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception 1681 { 1682 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1683 return indexResources(resources, documentType, null, workspaceName, solrClient); 1684 } 1685 1686 /** 1687 * Send a collection of contents for indexation in the solr server. 1688 * @param resources the collection of contents to index. 1689 * @param documentType The document type of the resource 1690 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1691 * @param workspaceName The workspace where to index 1692 * @param solrClient The solr client to use 1693 * @return the indexation result. 1694 * @throws Exception if an error occurs while indexing. 1695 */ 1696 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1697 { 1698 Request request = ContextHelper.getRequest(_context); 1699 1700 // Retrieve the current workspace. 1701 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1702 1703 try 1704 { 1705 // Force the workspace. 1706 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1707 1708 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1709 1710 long start = System.currentTimeMillis(); 1711 1712 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient); 1713 1714 long end = System.currentTimeMillis(); 1715 1716 if (!result.hasErrors()) 1717 { 1718 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1719 } 1720 else 1721 { 1722 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1723 } 1724 1725 return result; 1726 } 1727 catch (Exception e) 1728 { 1729 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1730 getLogger().error(error, e); 1731 throw new IndexingException(error, e); 1732 } 1733 finally 1734 { 1735 // Restore context 1736 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1737 } 1738 } 1739 1740 /** 1741 * Send some resources for indexation in the solr server. 1742 * @param resources the resources to index. 1743 * @param documentType The document type of the resource 1744 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1745 * @param workspaceName The workspace where to index 1746 * @param solrClient The solr client to use 1747 * @return the indexation result. 1748 * @throws Exception if an error occurs committing the results. 1749 */ 1750 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1751 { 1752 int successCount = 0; 1753 int errorCount = 0; 1754 for (AmetysObject resource : resources) 1755 { 1756 try 1757 { 1758 doIndexExplorerItem(resource, documentType, resourceRoot, solrClient); 1759 successCount++; 1760 } 1761 catch (Exception e) 1762 { 1763 getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e); 1764 errorCount++; 1765 } 1766 } 1767 1768 return new IndexationResult(successCount, errorCount); 1769 } 1770 1771 /** 1772 * Add or update a resource into Solr index 1773 * @param resource The resource to index 1774 * @param documentType The document type of the resource 1775 * @param workspaceName The workspace where to index 1776 * @throws Exception if an error occurs while indexing. 1777 */ 1778 public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception 1779 { 1780 Request request = ContextHelper.getRequest(_context); 1781 1782 // Retrieve the current workspace. 1783 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1784 1785 try 1786 { 1787 // Force the workspace. 1788 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1789 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1790 1791 getLogger().debug("Indexing resource {} into Solr.", resource.getId()); 1792 1793 doIndexResource(resource, documentType, null, solrClient); 1794 1795 getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId()); 1796 } 1797 catch (Exception e) 1798 { 1799 String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName); 1800 getLogger().error(error, e); 1801 throw new IndexingException(error, e); 1802 } 1803 finally 1804 { 1805 // Restore context 1806 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1807 } 1808 } 1809 1810 private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1811 { 1812 getLogger().debug("Unindexing all resources from Solr."); 1813 1814 String collection = _solrClientProvider.getCollectionName(workspaceName); 1815 solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1816 1817 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1818 } 1819 1820 /** 1821 * Delete all resource documents at a given path for all workspaces and commit 1822 * @param rootId The resource root ID, must not be null. 1823 * @param path The resource path relative to the given root, must start with a slash. 1824 * @throws Exception If an error occurs while unindexing. 1825 */ 1826 public void unindexResourcesByPath(String rootId, String path) throws Exception 1827 { 1828 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1829 for (String workspaceName : workspaceNames) 1830 { 1831 unindexResourcesByPath(rootId, path, workspaceName); 1832 } 1833 } 1834 1835 /** 1836 * Delete all resource documents at a given path. 1837 * @param rootId The resource root ID, must not be null. 1838 * @param path The resource path relative to the given root, must start with a slash. 1839 * @param workspaceName The workspace where to work in 1840 * @throws Exception If an error occurs while unindexing. 1841 */ 1842 public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception 1843 { 1844 Request request = ContextHelper.getRequest(_context); 1845 1846 // Retrieve the current workspace. 1847 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1848 1849 try 1850 { 1851 // Force the workspace. 1852 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1853 1854 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1855 1856 Query query = new ResourceLocationQuery(rootId, path); 1857 1858 String collection = _solrClientProvider.getCollectionName(workspaceName); 1859 _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1860 1861 getLogger().debug("Succesfully deleted resource document from Solr."); 1862 } 1863 catch (Exception e) 1864 { 1865 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1866 getLogger().error(error, e); 1867 throw new IndexingException(error, e); 1868 } 1869 finally 1870 { 1871 // Restore workspace 1872 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1873 } 1874 } 1875 1876 /** 1877 * Remove a resource from Solr index for all workspaces and commit 1878 * @param resourceId The id of resource to unindex 1879 * @throws Exception if an error occurs while indexing. 1880 */ 1881 public void unindexResource(String resourceId) throws Exception 1882 { 1883 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1884 for (String workspaceName : workspaceNames) 1885 { 1886 unindexResource(resourceId, workspaceName); 1887 } 1888 } 1889 1890 /** 1891 * Remove a resource from the Solr index. 1892 * @param resourceId The id of resource to unindex 1893 * @param workspaceName The workspace where to work in 1894 * @throws Exception if an error occurs while unindexing. 1895 */ 1896 public void unindexResource(String resourceId, String workspaceName) throws Exception 1897 { 1898 Request request = ContextHelper.getRequest(_context); 1899 1900 // Retrieve the current workspace. 1901 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1902 1903 try 1904 { 1905 // Force the workspace. 1906 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1907 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1908 1909 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 1910 1911 doUnindexDocument(resourceId, workspaceName, solrClient); 1912 1913 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 1914 } 1915 catch (Exception e) 1916 { 1917 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 1918 getLogger().error(error, e); 1919 throw new IndexingException(error, e); 1920 } 1921 finally 1922 { 1923 // Restore workspace 1924 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1925 } 1926 } 1927 1928 /** 1929 * Add or update a resource into Solr index 1930 * @param node The resource to index 1931 * @param documentType The document type of the resource 1932 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1933 * @param solrClient The solr client to use 1934 * @throws Exception if an error occurs while indexing. 1935 */ 1936 protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 1937 { 1938 if (node instanceof ResourceCollection) 1939 { 1940 for (AmetysObject child : ((ResourceCollection) node).getChildren()) 1941 { 1942 doIndexExplorerItem(child, documentType, resourceRoot, solrClient); 1943 } 1944 } 1945 else if (node instanceof Resource) 1946 { 1947 doIndexResource((Resource) node, documentType, resourceRoot, solrClient); 1948 } 1949 } 1950 1951 /** 1952 * Add or update a resource into Solr index 1953 * @param resource The resource to index 1954 * @param documentType The document type of the resource 1955 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1956 * @param solrClient The solr client to use 1957 * @throws Exception if an error occurs while indexing. 1958 */ 1959 protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 1960 { 1961 SolrInputDocument document = new SolrInputDocument(); 1962 1963 _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot); 1964 1965 String workspaceName = _workspaceSelector.getWorkspace(); 1966 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document); 1967 int status = solrResponse.getStatus(); 1968 1969 if (status != 0) 1970 { 1971 throw new IOException("Resource indexation: got status code '" + status + "'."); 1972 } 1973 } 1974 1975 /** 1976 * Process a Solr commit operation in all workspaces. 1977 * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient} 1978 * @throws SolrServerException if there is an error on the server 1979 * @throws IOException if there is a communication error with the server 1980 */ 1981 public void commit() throws SolrServerException, IOException 1982 { 1983 String[] workspaceNames; 1984 try 1985 { 1986 workspaceNames = _workspaceSelector.getWorkspaces(); 1987 for (String workspaceName : workspaceNames) 1988 { 1989 SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName); 1990 commit(workspaceName, solrClient); 1991 } 1992 } 1993 catch (RepositoryException e) 1994 { 1995 throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e); 1996 } 1997 } 1998 1999 /** 2000 * Process a Solr commit operation in given workspace. 2001 * @param workspaceName The workspace's name 2002 * @param solrClient The solr client to use 2003 * @throws SolrServerException if there is an error on the server 2004 * @throws IOException if there is a communication error with the server 2005 */ 2006 public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2007 { 2008 long time_0 = System.currentTimeMillis(); 2009 2010 // Commit 2011 UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName)); 2012 int status = solrResponse.getStatus(); 2013 2014 if (status != 0) 2015 { 2016 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2017 } 2018 2019 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 2020 } 2021 2022 /** 2023 * Launch a solr index optimization. 2024 * @param workspaceName The workspace's name 2025 * @param solrClient The solr client to use 2026 * @throws SolrServerException if there is an error on the server 2027 * @throws IOException if there is a communication error with the server 2028 */ 2029 public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2030 { 2031 UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName)); 2032 int status = solrResponse.getStatus(); 2033 2034 if (status != 0) 2035 { 2036 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2037 } 2038 2039 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 2040 } 2041 2042 /** 2043 * Delete all documents from the solr index. 2044 * @param workspaceName The workspace name 2045 * @param solrClient The solr client to use 2046 * @throws Exception if an error occurs while unindexing. 2047 */ 2048 public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception 2049 { 2050 getLogger().debug("Deleting all documents from Solr."); 2051 2052 String collection = _solrClientProvider.getCollectionName(workspaceName); 2053 solrClient.deleteByQuery(collection, "*:*"); 2054 2055 getLogger().debug("Successfully deleted all documents from Solr."); 2056 } 2057 2058 /** 2059 * Delete a document from the Solr server. 2060 * @param id The id of the document to delete from Solr 2061 * @param workspaceName The workspace name 2062 * @param solrClient The solr client to use 2063 * @throws Exception if an error occurs while indexing. 2064 */ 2065 protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception 2066 { 2067 UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 2068 int status = solrResponse.getStatus(); 2069 2070 if (status != 0) 2071 { 2072 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 2073 } 2074 } 2075 2076 /** 2077 * Delete content attachments documents of a given content from the Solr server. 2078 * @param contentId The id of the content 2079 * @param workspaceName The workspace name 2080 * @param solrClient The solr client to use 2081 * @throws Exception if an error occurs while indexing. 2082 */ 2083 protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception 2084 { 2085 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 2086 2087 Query query = new ContentAttachmentQuery(contentId); 2088 UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build()); 2089 int status = solrResponse.getStatus(); 2090 2091 if (status != 0) 2092 { 2093 throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'."); 2094 } 2095 } 2096 2097// /** 2098// * Get the collection to use. 2099// * @return The name of the collection to index into. 2100// */ 2101// protected String getCollection() 2102// { 2103// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 2104// } 2105 2106 class IndexationResult 2107 { 2108 protected int _successCount; 2109 2110 protected int _errorCount; 2111 2112 /** 2113 * Constructor 2114 */ 2115 public IndexationResult() 2116 { 2117 this(0, 0); 2118 } 2119 2120 /** 2121 * Constructor 2122 * @param successCount The success count. 2123 * @param errorCount The error count. 2124 */ 2125 public IndexationResult(int successCount, int errorCount) 2126 { 2127 this._successCount = successCount; 2128 this._errorCount = errorCount; 2129 } 2130 2131 /** 2132 * Get the successCount. 2133 * @return the successCount 2134 */ 2135 public int getSuccessCount() 2136 { 2137 return _successCount; 2138 } 2139 2140 /** 2141 * Set the successCount. 2142 * @param successCount the successCount to set 2143 */ 2144 public void setSuccessCount(int successCount) 2145 { 2146 this._successCount = successCount; 2147 } 2148 2149 /** 2150 * Test if the indexation had errors. 2151 * @return true if the indexation had errors, false otherwise. 2152 */ 2153 public boolean hasErrors() 2154 { 2155 return _errorCount > 0; 2156 } 2157 2158 /** 2159 * Get the errorCount. 2160 * @return the errorCount 2161 */ 2162 public int getErrorCount() 2163 { 2164 return _errorCount; 2165 } 2166 2167 /** 2168 * Set the errorCount. 2169 * @param errorCount the errorCount to set 2170 */ 2171 public void setErrorCount(int errorCount) 2172 { 2173 this._errorCount = errorCount; 2174 } 2175 } 2176 2177}