001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.nio.ByteBuffer; 020import java.nio.CharBuffer; 021import java.nio.charset.CharsetDecoder; 022import java.nio.charset.CodingErrorAction; 023import java.nio.charset.StandardCharsets; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Collection; 028import java.util.Comparator; 029import java.util.HashMap; 030import java.util.HashSet; 031import java.util.List; 032import java.util.Map; 033import java.util.Set; 034import java.util.function.Function; 035 036import javax.jcr.RepositoryException; 037 038import org.apache.avalon.framework.activity.Initializable; 039import org.apache.avalon.framework.component.Component; 040import org.apache.avalon.framework.context.Context; 041import org.apache.avalon.framework.context.ContextException; 042import org.apache.avalon.framework.context.Contextualizable; 043import org.apache.avalon.framework.service.ServiceException; 044import org.apache.avalon.framework.service.ServiceManager; 045import org.apache.avalon.framework.service.Serviceable; 046import org.apache.cocoon.components.ContextHelper; 047import org.apache.cocoon.environment.Request; 048import org.apache.commons.lang3.ObjectUtils; 049import org.apache.commons.lang3.StringUtils; 050import org.apache.solr.client.solrj.SolrClient; 051import org.apache.solr.client.solrj.SolrQuery; 052import org.apache.solr.client.solrj.SolrServerException; 053import org.apache.solr.client.solrj.request.CoreAdminRequest; 054import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 055import org.apache.solr.client.solrj.request.schema.SchemaRequest; 056import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 057import org.apache.solr.client.solrj.response.CoreAdminResponse; 058import org.apache.solr.client.solrj.response.QueryResponse; 059import org.apache.solr.client.solrj.response.SolrResponseBase; 060import org.apache.solr.client.solrj.response.UpdateResponse; 061import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 062import org.apache.solr.client.solrj.response.schema.SchemaResponse; 063import org.apache.solr.client.solrj.util.ClientUtils; 064import org.apache.solr.common.SolrInputDocument; 065import org.apache.solr.common.util.NamedList; 066import org.slf4j.Logger; 067 068import org.ametys.cms.indexing.IndexingException; 069import org.ametys.cms.indexing.solr.ReloadAclCacheRequest; 070import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest; 071import org.ametys.cms.repository.Content; 072import org.ametys.cms.repository.ContentQueryHelper; 073import org.ametys.cms.repository.WorkflowAwareContent; 074import org.ametys.cms.rights.solrchecking.ReadAccessHelper; 075import org.ametys.cms.search.query.ContentAttachmentQuery; 076import org.ametys.cms.search.query.DocumentTypeQuery; 077import org.ametys.cms.search.query.OrQuery; 078import org.ametys.cms.search.query.Query; 079import org.ametys.cms.search.query.ResourceLocationQuery; 080import org.ametys.cms.search.solr.NoAutoCommitUpdateClient; 081import org.ametys.cms.search.solr.SolrClientProvider; 082import org.ametys.cms.search.solr.schema.CopyFieldDefinition; 083import org.ametys.cms.search.solr.schema.FieldDefinition; 084import org.ametys.cms.search.solr.schema.SchemaDefinition; 085import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 086import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 087import org.ametys.cms.search.solr.schema.SchemaFields; 088import org.ametys.cms.search.solr.schema.SchemaHelper; 089import org.ametys.core.group.GroupIdentity; 090import org.ametys.core.right.AllowedUsers; 091import org.ametys.core.user.UserIdentity; 092import org.ametys.plugins.explorer.resources.Resource; 093import org.ametys.plugins.explorer.resources.ResourceCollection; 094import org.ametys.plugins.repository.AmetysObject; 095import org.ametys.plugins.repository.AmetysObjectIterable; 096import org.ametys.plugins.repository.AmetysObjectResolver; 097import org.ametys.plugins.repository.RepositoryConstants; 098import org.ametys.plugins.repository.TraversableAmetysObject; 099import org.ametys.plugins.repository.UnknownAmetysObjectException; 100import org.ametys.plugins.repository.collection.AmetysObjectCollection; 101import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector; 102import org.ametys.plugins.repository.provider.WorkspaceSelector; 103import org.ametys.runtime.config.Config; 104import org.ametys.runtime.plugin.component.AbstractLogEnabled; 105 106/** 107 * Solr indexer. 108 */ 109public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 110{ 111 /** The component role. */ 112 public static final String ROLE = SolrIndexer.class.getName(); 113 114 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 115 116 private static final String _CONFIGSET_NAME_PREFIX = "configset-"; 117 118 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_root_", "_version_", "text"); 119 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general"); 120 121 private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766; 122 123 /** The ametys object resolver. */ 124 protected AmetysObjectResolver _resolver; 125 /** The schema definition provider extension point. */ 126 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 127 /** The schema helper. */ 128 protected SchemaHelper _schemaHelper; 129 /** Solr Ametys contents indexer */ 130 protected SolrContentIndexer _solrContentIndexer; 131 /** Solr workflow indexer. */ 132 protected SolrWorkflowIndexer _solrWorkflowIndexer; 133 /** Solr resource indexer. */ 134 protected SolrResourceIndexer _solrResourceIndexer; 135 136 /** The Solr client provider */ 137 protected SolrClientProvider _solrClientProvider; 138 139 /** The solr core prefix. */ 140 protected String _solrCorePrefix; 141 /** The Ametys internal URL used by Solr to query Ametys */ 142 protected String _ametysInternalUrl; 143 144 /** The workspace selector. */ 145 protected WorkspaceSelector _workspaceSelector; 146 147 /** The helper for read access */ 148 protected ReadAccessHelper _readAccessHelper; 149 150 /** The avalon context */ 151 protected Context _context; 152 153 /** 154 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 155 * @return The date format for indexing dates 156 */ 157 public static SimpleDateFormat dateFormat() 158 { 159 if (__DATE_FORMAT.get() == null) 160 { 161 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 162 } 163 return __DATE_FORMAT.get(); 164 } 165 166 /** 167 * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr. 168 * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept. 169 * @param value The string value to index 170 * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log. 171 * @param documentId The id of the document being indexed. Can be null if you do not want to log. 172 * @param fieldName The name of the field being indexed. Can be null if you do not want to log. 173 * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes) 174 */ 175 public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName) 176 { 177 if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT) 178 { 179 // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length()) 180 // As a result, value.getBytes().length <= limit 181 return value; 182 } 183 184 // There is a doubt, the string may need to be truncated (or not) 185 byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); 186 int bytesLength = valueBytes.length; 187 if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT) 188 { 189 return value; 190 } 191 192 if (ObjectUtils.allNotNull(logger, documentId, fieldName)) 193 { 194 logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT); 195 } 196 197 // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974) 198 CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT); 199 CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() 200 .onMalformedInput(CodingErrorAction.IGNORE); 201 decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true); 202 decoder.flush(charBuffer); 203 return new String(charBuffer.array(), 0, charBuffer.position()); 204 } 205 206 @Override 207 public void service(ServiceManager serviceManager) throws ServiceException 208 { 209 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 210 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 211 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 212 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 213 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 214 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 215 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 216 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 217 _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE); 218 } 219 220 @Override 221 public void initialize() throws Exception 222 { 223 Config config = Config.getInstance(); 224 _solrCorePrefix = config.getValue("cms.solr.core.prefix"); 225 226 _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url"); 227 if (StringUtils.isBlank(_ametysInternalUrl)) 228 { 229 // fallback to CMS URL as internal URL is an optional parameter 230 _ametysInternalUrl = config.getValue("cms.url"); 231 } 232 } 233 234 @Override 235 public void contextualize(Context context) throws ContextException 236 { 237 _context = context; 238 } 239 240 /** 241 * Gets the 'autocommit' Solr client 242 * @param workspaceName The name of the workspace 243 * @return the Solr client 244 */ 245 protected SolrClient _getAutoCommitSolrClient(String workspaceName) 246 { 247 return _solrClientProvider.getUpdateClient(workspaceName, true); 248 } 249 250 /** 251 * Gets the 'no autocommit' Solr client 252 * @param workspaceName The name of the workspace 253 * @return the Solr client 254 */ 255 protected SolrClient _getNoAutoCommitSolrClient(String workspaceName) 256 { 257 return _solrClientProvider.getUpdateClient(workspaceName, false); 258 } 259 260 // for admin operations 261 private SolrClient _defaultSolrClient() 262 { 263 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 264 } 265 266 /** 267 * Get the names of the Solr cores. 268 * @return The names of the Solr cores. 269 * @throws IOException If an I/O error occurs. 270 * @throws SolrServerException If a Solr error occurs. 271 */ 272 @SuppressWarnings("unchecked") 273 public Set<String> getCoreNames() throws IOException, SolrServerException 274 { 275 Set<String> coreNames = new HashSet<>(); 276 277 getLogger().debug("Getting core list."); 278 279 SolrQuery query = new SolrQuery(); 280 query.setRequestHandler("/admin/cores"); 281 query.setParam("action", "STATUS"); 282 283 QueryResponse response = _defaultSolrClient().query(query); 284 285 NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status"); 286 for (Map.Entry<String, NamedList<?>> core : status) 287 { 288 String fullName = (String) core.getValue().get("name"); 289 if (fullName.startsWith(_solrCorePrefix)) 290 { 291 coreNames.add(fullName.substring(_solrCorePrefix.length())); 292 } 293 } 294 295 return coreNames; 296 } 297 298 /** 299 * Get the names of the Solr cores. 300 * @return The names of the Solr cores. 301 * @throws IOException If an I/O error occurs. 302 * @throws SolrServerException If a Solr error occurs. 303 */ 304 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 305 { 306 Set<String> coreNames = new HashSet<>(); 307 308 getLogger().debug("Getting core list."); 309 310 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 311 312 NamedList<NamedList<Object>> status = response.getCoreStatus(); 313 for (Map.Entry<String, NamedList<Object>> core : status) 314 { 315 String fullName = (String) core.getValue().get("name"); 316 if (fullName.startsWith(_solrCorePrefix)) 317 { 318 coreNames.add(fullName.substring(_solrCorePrefix.length())); 319 } 320 } 321 322 return coreNames; 323 } 324 325 /** 326 * Create a Solr core. 327 * @param name The name of the core to create. 328 * @throws IOException If an I/O error occurs. 329 * @throws SolrServerException If a Solr error occurs. 330 */ 331 public void createCore(String name) throws IOException, SolrServerException 332 { 333 String fullName = _solrCorePrefix + name; 334 String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix); 335 _createConfigset(configsetName); 336 337 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 338 339 SolrQuery query = new SolrQuery(); 340 query.setRequestHandler("/admin/cores"); 341 query.setParam("action", "CREATE"); 342 query.setParam("name", fullName); 343 query.setParam("configSet", configsetName); 344 query.setParam("property.ametys.url", _ametysInternalUrl); 345 346 QueryResponse response = _defaultSolrClient().query(query); 347 NamedList<?> results = response.getResponse(); 348 349 NamedList<?> error = (NamedList<?>) results.get("error"); 350 if (error != null) 351 { 352 throw new IOException("Error creating the core: " + error.get("msg")); 353 } 354 } 355 356 /** 357 * Updates the ametys.url property of the Solr cores. 358 */ 359 public void updateAmetysUrlCoreProperty() 360 { 361 Set<String> coreNames; 362 try 363 { 364 coreNames = getCoreNames(); 365 } 366 catch (SolrServerException | IOException e) 367 { 368 getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e); 369 return; 370 } 371 372 for (String coreName : coreNames) 373 { 374 String collection = _solrClientProvider.getCollectionName(coreName); 375 SolrResponseBase response; 376 try 377 { 378 response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection); 379 } 380 catch (SolrServerException | IOException e) 381 { 382 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e); 383 continue; 384 } 385 386 NamedList<Object> responseParams = response.getResponse(); 387 if ("ok".equals(responseParams.get("result"))) 388 { 389 Boolean valueChanged = responseParams.getBooleanArg("valueChanged"); 390 if (valueChanged) 391 { 392 getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName); 393 } 394 else 395 { 396 getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName); 397 } 398 } 399 else 400 { 401 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName); 402 } 403 } 404 } 405 406 private void _createConfigset(String name) throws IOException, SolrServerException 407 { 408 getLogger().info("Creating (if necessary) configset '{}'", name); 409 410 // This request handler will check if configset exists. If not it will be created. 411 SolrQuery query = new SolrQuery(); 412 query.setRequestHandler("/admin/cores"); 413 query.setParam("action", "createConfigset"); 414 query.setParam("name", name); 415 416 QueryResponse response = _defaultSolrClient().query(query); 417 NamedList<?> results = response.getResponse(); 418 419 NamedList<?> error = (NamedList<?>) results.get("error"); 420 if (error != null) 421 { 422 throw new IOException("Error creating the core: " + error.get("msg")); 423 } 424 } 425 426 /** 427 * Delete a Solr core. 428 * @param name The name of the core to delete. 429 * @throws IOException If an I/O error occurs. 430 * @throws SolrServerException If a Solr error occurs. 431 */ 432 public void deleteCore(String name) throws IOException, SolrServerException 433 { 434 String fullName = _solrCorePrefix + name; 435 436 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 437 438 SolrQuery query = new SolrQuery(); 439 query.setRequestHandler("/admin/cores"); 440 query.setParam("action", "UNLOAD"); 441 query.setParam("core", fullName); 442 query.setParam("deleteInstanceDir", "true"); 443 444 QueryResponse response = _defaultSolrClient().query(query); 445 NamedList<?> results = response.getResponse(); 446 447 NamedList<?> error = (NamedList<?>) results.get("error"); 448 if (error != null) 449 { 450 throw new IOException("Error deleting core" + name + ": " + error.get("msg")); 451 } 452 } 453 454 /** 455 * Send the schema. 456 * @throws IOException If a communication error occurs. 457 * @throws SolrServerException If a solr error occurs. 458 */ 459 public void sendSchema() throws IOException, SolrServerException 460 { 461 getLogger().info("Computing and sending the schema to the solr server."); 462 463 String workspaceName = _workspaceSelector.getWorkspace(); 464 String collection = _solrClientProvider.getCollectionName(workspaceName); 465 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 466 467// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 468 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 469 470 // TODO Clear the schema except fields marked ametysReadOnly="true". 471 472 // Clear the current schema. 473 clearSchema(solrClient, collection); 474 475 SchemaRequest schemaRequest = new SchemaRequest(); 476 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 477 478 // The cleared schema contains only the basic fields which can't be deleted. 479 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 480 SchemaFields schemaFields = new SchemaFields(clearedSchema); 481 482 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 483 484 // Add the static schema types and fields. 485 List<SchemaRequest.Update> updates = new ArrayList<>(); 486 487 // Set "add field" definitions from the static schema to the update list. 488 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 489 490 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 491 492 // Set "add field" definitions from the static schema to the update list. 493 addCustomUpdates(updates, schemaFields); 494 495 reorderUpdates(updates); 496 497 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 498 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 499 500 getLogger().debug("Send schema response: {}", updateResponse.toString()); 501 Object errors = updateResponse.getResponse().get("errors"); 502 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 503 { 504 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 505 throw new SolrServerException(msg); 506 } 507 508 getLogger().info("Schema sent to the solr server."); 509 510 reloadCores(); 511 } 512 513 /** 514 * Compute the list of {@link Update} directives from the static schema. 515 * @param updates The list of {@link Update} directives to fill. 516 * @param staticSchema The static schema representation. 517 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 518 */ 519 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 520 { 521 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 522 for (FieldTypeDefinition fieldType : fieldTypes) 523 { 524 String name = (String) fieldType.getAttributes().get("name"); 525 if (!schemaFields.hasFieldType(name)) 526 { 527 updates.add(new SchemaRequest.AddFieldType(fieldType)); 528 schemaFields.addFieldType(name); 529 } 530 } 531 for (Map<String, Object> field : staticSchema.getFields()) 532 { 533 String name = (String) field.get("name"); 534 if (!schemaFields.hasField(name)) 535 { 536 updates.add(new SchemaRequest.AddField(field)); 537 schemaFields.addField(name); 538 } 539 } 540 for (Map<String, Object> field : staticSchema.getDynamicFields()) 541 { 542 String name = (String) field.get("name"); 543 if (!schemaFields.hasDynamicField(name)) 544 { 545 updates.add(new SchemaRequest.AddDynamicField(field)); 546 schemaFields.addDynamicField(name); 547 } 548 } 549 for (Map<String, Object> field : staticSchema.getCopyFields()) 550 { 551 String source = (String) field.get("source"); 552 String dest = (String) field.get("dest"); 553 if (!schemaFields.hasCopyField(source, dest)) 554 { 555 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 556 schemaFields.addCopyField(source, dest); 557 } 558 } 559 } 560 561 /** 562 * Compute the list of custom {@link Update} directives. 563 * @param updates The list of {@link Update} directives to fill. 564 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 565 */ 566 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 567 { 568 // Add all our property-managed fields. 569 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 570 { 571 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 572 573 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 574 { 575 if (!definitionExists(definition, schemaFields)) 576 { 577 SchemaRequest.Update update = getSchemaUpdate(definition); 578 if (update != null) 579 { 580 updates.add(update); 581 } 582 } 583 } 584 } 585 586// for (String propId : _sysPropEP.getExtensionsIds()) 587// { 588// Collection<SchemaDefinition> definitions = _sysPropEP.getExtension(propId).getSchemaDefinitions(); 589// 590// for (SchemaDefinition definition : definitions) 591// { 592// if (!definitionExists(definition, schemaFields)) 593// { 594// SchemaRequest.Update update = getSchemaUpdate(definition); 595// if (update != null) 596// { 597// updates.add(update); 598// } 599// } 600// } 601// } 602 } 603 604 /** 605 * Reorder the list of {@link Update} directives. 606 * @param updates The list of {@link Update} directives to reorder. 607 */ 608 protected void reorderUpdates(List<SchemaRequest.Update> updates) 609 { 610 updates.sort(Comparator.comparing(u -> 611 { 612 if (u instanceof SchemaRequest.AddCopyField) 613 { 614 // copyField at the end of the list to avoid an exception of an undeclared 'source' or 'dest' whereas it is declared later 615 return 1; 616 } 617 else 618 { 619 return 0; 620 } 621 })); 622 } 623 624 /** 625 * Test if the given schema definition exists in the given {@link SchemaFields} reference. 626 * @param definition The schema definition to test. 627 * @param schemaFields The current schema fields. 628 * @return true if the SchemaFields contain the schema definition. 629 */ 630 protected boolean definitionExists(SchemaDefinition definition, SchemaFields schemaFields) 631 { 632 if (definition instanceof FieldDefinition) 633 { 634 FieldDefinition fieldDef = (FieldDefinition) definition; 635 if (fieldDef.isDynamic()) 636 { 637 return schemaFields.hasField(fieldDef.getName()); 638 } 639 else 640 { 641 return schemaFields.hasDynamicField(fieldDef.getName()); 642 } 643 } 644 else if (definition instanceof CopyFieldDefinition) 645 { 646 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 647 return schemaFields.hasCopyField(fieldDef.getSource(), fieldDef.getDestination()); 648 } 649 650 return false; 651 } 652 653 /** 654 * Delete all the fields of the existing schema in the given collection. 655 * @param solrClient The Solr client 656 * @param collection The collection. 657 * @throws IOException If a communication error occurs. 658 * @throws SolrServerException If a solr error occurs. 659 */ 660 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 661 { 662 try 663 { 664 getLogger().info("Clearing the existing schema on the solr server."); 665 666 SchemaRequest schemaRequest = new SchemaRequest(); 667 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 668 669 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 670 671 List<SchemaRequest.Update> deletions = new ArrayList<>(); 672 673 // First the copy fields, then dynamic and simple fields, and field types in the end. 674 for (Map<String, Object> field : schema.getCopyFields()) 675 { 676 String source = (String) field.get("source"); 677 String dest = (String) field.get("dest"); 678 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 679 } 680 for (Map<String, Object> field : schema.getDynamicFields()) 681 { 682 String name = (String) field.get("name"); 683 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 684 } 685 for (Map<String, Object> field : schema.getFields()) 686 { 687 String name = (String) field.get("name"); 688 if (!_READ_ONLY_FIELDS.contains(name)) 689 { 690 deletions.add(new SchemaRequest.DeleteField(name)); 691 } 692 } 693 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 694 { 695 String name = (String) fieldType.getAttributes().get("name"); 696 if (!_READ_ONLY_FIELDTYPES.contains(name)) 697 { 698 deletions.add(new SchemaRequest.DeleteFieldType(name)); 699 } 700 } 701 702 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 703 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 704 705 Object errors = updateResponse.getResponse().get("errors"); 706 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 707 { 708 String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString(); 709 throw new SolrServerException(msg); 710 } 711 else 712 { 713 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 714 } 715 716 getLogger().info("Solr schema cleared."); 717 } 718 catch (SolrServerException | IOException e) 719 { 720 getLogger().error("Error clearing schema in collection " + collection, e); 721 throw e; 722 } 723 } 724 725 /** 726 * Get the schema {@link Update} directive from the given schema definition. 727 * @param definition The schema definition to add. 728 * @return The add {@link Update} directive. 729 */ 730 protected SchemaRequest.Update getSchemaUpdate(SchemaDefinition definition) 731 { 732 SchemaRequest.Update update = null; 733 734 if (definition instanceof FieldDefinition) 735 { 736 FieldDefinition fieldDef = (FieldDefinition) definition; 737 738 // Force indexed and stored attributes to true. 739 Map<String, Object> attributes = new HashMap<>(); 740 attributes.put("name", fieldDef.getName()); 741 attributes.put("type", fieldDef.getType()); 742 attributes.put("multiValued", fieldDef.isMultiValued()); 743 attributes.put("docValues", fieldDef.isDocValues()); 744 attributes.put("indexed", Boolean.TRUE); 745 attributes.put("stored", Boolean.TRUE); 746 747 update = new SchemaRequest.AddField(attributes); 748 } 749 else if (definition instanceof CopyFieldDefinition) 750 { 751 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 752 753 String source = fieldDef.getSource(); 754 String dest = fieldDef.getDestination(); 755 756 update = new SchemaRequest.AddCopyField(source, Arrays.asList(dest)); 757 } 758 759 return update; 760 } 761 762 /** 763 * Reload the solr cores. 764 * @throws IOException If a communication error occurs. 765 * @throws SolrServerException If a solr error occurs. 766 */ 767 protected void reloadCores() throws IOException, SolrServerException 768 { 769 getLogger().info("Reloading solr cores."); 770 771 for (String coreName : getCoreNames()) 772 { 773 String fullName = _solrCorePrefix + coreName; 774 775 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 776 777 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 778 } 779 780 getLogger().info("All cores reloaded."); 781 } 782 783 /** 784 * Reloads the ACL Solr cache for all users 785 * @throws IOException If an I/O error occurs. 786 * @throws SolrServerException If a Solr error occurs. 787 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 788 */ 789 public void reloadAclCache() throws IOException, SolrServerException, RepositoryException 790 { 791 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 792 for (String workspaceName : workspaceNames) 793 { 794 reloadAclCache(workspaceName); 795 } 796 } 797 798 /** 799 * Reloads the ACL Solr cache for all users 800 * @param workspaceName The workspace name 801 * @throws IOException If an I/O error occurs. 802 * @throws SolrServerException If a Solr error occurs. 803 */ 804 public void reloadAclCache(String workspaceName) throws IOException, SolrServerException 805 { 806 reloadAclCache(workspaceName, false); 807 } 808 809 /** 810 * Reloads the ACL Solr cache for all users 811 * @param workspaceName The workspace name 812 * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache) 813 * @throws IOException If an I/O error occurs. 814 * @throws SolrServerException If a Solr error occurs. 815 */ 816 public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException 817 { 818 getLogger().info("Reloading read ACL Solr cache for workspace '{}'", workspaceName); 819 820 String collection = _solrClientProvider.getCollectionName(workspaceName); 821 SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection); 822 NamedList<Object> responseObj = responseBase.getResponse(); 823 824 if ("ok".equals(responseObj.get("result"))) 825 { 826 getLogger().info("Read-ACL Solr cache reloaded for workspace '{}'", workspaceName); 827 } 828 else 829 { 830 Object error = responseObj.get("error"); 831 getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Error code is the following: {}", workspaceName, error); 832 } 833 } 834 835 /** 836 * Index all the contents in a given workspace. 837 * @param workspaceName the workspace where to index 838 * @param indexAttachments to index content attachments 839 * @param solrClient The solr client to use 840 * @return The indexation result as a Map. 841 * @throws Exception if an error occurs while indexing. 842 */ 843 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 844 { 845 Request request = ContextHelper.getRequest(_context); 846 847 // Retrieve the current workspace. 848 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 849 850 try 851 { 852 // Force the workspace. 853 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 854 855 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 856 857 long start = System.currentTimeMillis(); 858 859 // Delete all contents 860 unindexAllContents(workspaceName, indexAttachments, solrClient); 861 862 String query = ContentQueryHelper.getContentXPathQuery(null); 863 AmetysObjectIterable<Content> contents = _resolver.query(query); 864 865 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient); 866 867 long end = System.currentTimeMillis(); 868 869 if (!result.hasErrors()) 870 { 871 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 872 } 873 else 874 { 875 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 876 } 877 878 Map<String, Object> results = new HashMap<>(); 879 results.put("successCount", result.getSuccessCount()); 880 if (result.hasErrors()) 881 { 882 results.put("errorCount", result.getErrorCount()); 883 } 884 885 return results; 886 } 887 catch (Exception e) 888 { 889 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 890 getLogger().error(error, e); 891 throw new IndexingException(error, e); 892 } 893 finally 894 { 895 // Restore context 896 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 897 } 898 } 899 900 /** 901 * Unindex all content documents. 902 * @param workspaceName The workspace name 903 * @param unindexAttachments also unindex content attachments 904 * @param solrClient The solr client to use 905 * @throws Exception if an error occurs while unindexing. 906 */ 907 protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 908 { 909 String collection = _solrClientProvider.getCollectionName(workspaceName); 910 911 Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT); 912 Query query; 913 if (unindexAttachments) 914 { 915 Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE); 916 Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE); 917 query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents); 918 } 919 else 920 { 921 query = contents; 922 } 923 solrClient.deleteByQuery(collection, query.build()); 924 } 925 926 /** 927 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 928 * @param collectionId The id of collection 929 * @param indexAttachments to index content attachments 930 * @throws Exception if an error occurs while indexing. 931 */ 932 public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception 933 { 934 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 935 for (String workspaceName : workspaceNames) 936 { 937 indexSubcontents(collectionId, workspaceName, indexAttachments); 938 } 939 } 940 941 /** 942 * Index the child contents of a {@link AmetysObjectCollection} 943 * @param collectionId The id of collection 944 * @param workspaceName the workspace where to index 945 * @param indexAttachments to index content attachments 946 * @throws Exception if an error occurs while unindexing. 947 */ 948 public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception 949 { 950 Request request = ContextHelper.getRequest(_context); 951 952 // Retrieve the current workspace. 953 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 954 955 try 956 { 957 // Force the workspace. 958 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 959 960 if (_resolver.hasAmetysObjectForId(collectionId)) 961 { 962 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 963 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 964 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 965 966 for (AmetysObject child : children) 967 { 968 if (child instanceof Content) 969 { 970 Content content = (Content) child; 971 972 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 973 } 974 } 975 } 976 } 977 finally 978 { 979 // Restore context 980 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 981 } 982 } 983 984 /** 985 * Add or update a content into Solr index on all workspaces and commit 986 * @param contentId The id of the content to index 987 * @param indexAttachments to index content attachments 988 * @throws Exception if an error occurs while indexing. 989 */ 990 public void indexContent(String contentId, boolean indexAttachments) throws Exception 991 { 992 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 993 for (String workspaceName : workspaceNames) 994 { 995 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 996 indexContent(contentId, workspaceName, indexAttachments, solrClient); 997 } 998 } 999 1000 /** 1001 * Add or update a content into Solr index 1002 * @param contentId The id of the content to index 1003 * @param workspaceName the workspace where to index 1004 * @param indexAttachments to index content attachments 1005 * @throws Exception if an error occurs while indexing. 1006 */ 1007 public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception 1008 { 1009 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1010 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1011 } 1012 1013 /** 1014 * Add or update a content into Solr index 1015 * @param contentId The id of the content to index 1016 * @param workspaceName the workspace where to index 1017 * @param indexAttachments to index content attachments 1018 * @param solrClient The solr client to use 1019 * @throws Exception if an error occurs while indexing. 1020 */ 1021 public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1022 { 1023 Request request = ContextHelper.getRequest(_context); 1024 1025 // Retrieve the current workspace. 1026 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1027 1028 try 1029 { 1030 // Force the workspace. 1031 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1032 1033 if (_resolver.hasAmetysObjectForId(contentId)) 1034 { 1035 Content content = _resolver.resolveById(contentId); 1036 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1037 } 1038 } 1039 finally 1040 { 1041 // Restore context 1042 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1043 } 1044 } 1045 1046 private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException 1047 { 1048 try 1049 { 1050 long time_0 = System.currentTimeMillis(); 1051 1052 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 1053 1054 deleteRepeaterDocs(content.getId(), workspaceName, solrClient); 1055 doIndexContent(content, workspaceName, solrClient); 1056 doIndexContentWorkflow(content, workspaceName, solrClient); 1057 if (indexAttachments) 1058 { 1059 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1060 } 1061 1062 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 1063 } 1064 catch (Exception e) 1065 { 1066 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 1067 getLogger().error(error, e); 1068 throw new IndexingException(error, e); 1069 } 1070 } 1071 1072 /** 1073 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1074 * @param contents the collection of contents to index. 1075 * @return the indexation result. 1076 * @throws Exception if an error occurs while indexing. 1077 */ 1078 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 1079 { 1080 IndexationResult result = new IndexationResult(); 1081 1082 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1083 for (String workspaceName : workspaceNames) 1084 { 1085 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1086 IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient); 1087 result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount()); 1088 } 1089 1090 return result; 1091 } 1092 1093 /** 1094 * Send a collection of contents for indexation in the solr server. 1095 * @param contents the collection of contents to index. 1096 * @param workspaceName the workspace where to index 1097 * @param indexAttachments to index content attachments 1098 * @param solrClient The solr client to use 1099 * @return the indexation result. 1100 * @throws Exception if an error occurs while indexing. 1101 */ 1102 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1103 { 1104 Request request = ContextHelper.getRequest(_context); 1105 1106 // Retrieve the current workspace. 1107 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1108 1109 try 1110 { 1111 // Force the workspace. 1112 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1113 1114 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 1115 1116 long start = System.currentTimeMillis(); 1117 1118 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient); 1119 1120 long end = System.currentTimeMillis(); 1121 1122 if (!result.hasErrors()) 1123 { 1124 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1125 } 1126 else 1127 { 1128 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1129 } 1130 1131 return result; 1132 } 1133 catch (Exception e) 1134 { 1135 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 1136 getLogger().error(error, e); 1137 throw new IndexingException(error, e); 1138 } 1139 finally 1140 { 1141 // Restore context 1142 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1143 } 1144 1145 } 1146 1147 /** 1148 * Send some contents for indexation in the solr server. 1149 * @param contents the contents to index. 1150 * @param workspaceName The workspace name 1151 * @param indexAttachments to index content attachments 1152 * @param solrClient The solr client to use 1153 * @return the indexation result. 1154 * @throws Exception if an error occurs committing the results. 1155 */ 1156 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1157 { 1158 int successCount = 0; 1159 int errorCount = 0; 1160 for (Content content : contents) 1161 { 1162 try 1163 { 1164 doIndexContent(content, workspaceName, solrClient); 1165 doIndexContentWorkflow(content, workspaceName, solrClient); 1166 if (indexAttachments) 1167 { 1168 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1169 } 1170 successCount++; 1171 } 1172 catch (Exception e) 1173 { 1174 getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e); 1175 errorCount++; 1176 } 1177 } 1178 1179 return new IndexationResult(successCount, errorCount); 1180 } 1181 1182 /** 1183 * Update the value of a specific system property in a content document. 1184 * @param content The content to update. 1185 * @param propertyId The system property ID. 1186 * @param workspaceName The workspace name 1187 * @throws Exception if an error occurs while indexing. 1188 */ 1189 public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception 1190 { 1191 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1192 updateSystemProperty(content, propertyId, workspaceName, solrClient); 1193 } 1194 1195 /** 1196 * Update the value of a specific system property in a content document. 1197 * @param content The content to update. 1198 * @param propertyId The system property ID. 1199 * @param workspaceName The workspace name 1200 * @param solrClient The solr client to use 1201 * @throws Exception if an error occurs while indexing. 1202 */ 1203 public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1204 { 1205 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 1206 1207 SolrInputDocument document = new SolrInputDocument(); 1208 boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 1209 1210 if (!hasUpdate) 1211 { 1212 getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content); 1213 return; 1214 } 1215 1216 String collection = _solrClientProvider.getCollectionName(workspaceName); 1217 UpdateResponse solrResponse = solrClient.add(collection, document); 1218 int status = solrResponse.getStatus(); 1219 1220 if (status != 0) 1221 { 1222 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 1223 } 1224 1225 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 1226 } 1227 1228 /** 1229 * Remove a content from Solr index for all workspaces and commit 1230 * @param contentId The id of content to unindex 1231 * @param unindexAttachments also unindex content attachments 1232 * @throws Exception if an error occurs while indexing. 1233 */ 1234 public void unindexContent(String contentId, boolean unindexAttachments) throws Exception 1235 { 1236 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1237 for (String workspaceName : workspaceNames) 1238 { 1239 unindexContent(contentId, workspaceName, unindexAttachments); 1240 } 1241 } 1242 1243 /** 1244 * Remove a content from Solr index 1245 * @param contentId The id of content to unindex 1246 * @param workspaceName The workspace where to work in 1247 * @param unindexAttachments also unindex content attachments 1248 * @throws Exception if an error occurs while indexing. 1249 */ 1250 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception 1251 { 1252 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1253 unindexContent(contentId, workspaceName, unindexAttachments, solrClient); 1254 } 1255 1256 /** 1257 * Remove a content from Solr index 1258 * @param contentId The id of content to unindex 1259 * @param workspaceName The workspace where to work in 1260 * @param unindexAttachments also unindex content attachments 1261 * @param solrClient The solr client to use 1262 * @throws Exception if an error occurs while indexing. 1263 */ 1264 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 1265 { 1266 Request request = ContextHelper.getRequest(_context); 1267 1268 // Retrieve the current workspace. 1269 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1270 1271 try 1272 { 1273 // Force the workspace. 1274 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1275 1276 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 1277 1278 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1279 doUnindexDocument(contentId, workspaceName, solrClient); 1280 if (unindexAttachments) 1281 { 1282 doUnindexContentAttachments(contentId, workspaceName, solrClient); 1283 } 1284 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1285 1286 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 1287 } 1288 catch (Exception e) 1289 { 1290 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 1291 getLogger().error(error, e); 1292 throw new IndexingException(error, e); 1293 } 1294 finally 1295 { 1296 // Restore workspace 1297 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1298 } 1299 } 1300 1301 /** 1302 * Remove a content from Solr index for all workspaces and commit 1303 * @param contentIds The id of content to unindex 1304 * @throws Exception if an error occurs while indexing. 1305 */ 1306 public void unindexContents(Collection<String> contentIds) throws Exception 1307 { 1308 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1309 for (String workspaceName : workspaceNames) 1310 { 1311 unindexContents(contentIds, workspaceName); 1312 } 1313 } 1314 1315 /** 1316 * Remove a content from Solr index 1317 * @param contentIds The id of content to unindex 1318 * @param workspaceName The workspace where to work in 1319 * @throws Exception if an error occurs while indexing. 1320 */ 1321 public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception 1322 { 1323 Request request = ContextHelper.getRequest(_context); 1324 1325 // Retrieve the current workspace. 1326 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1327 1328 try 1329 { 1330 // Force the workspace. 1331 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1332 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1333 1334 getLogger().debug("Unindexing several contents from Solr."); 1335 1336 for (String contentId : contentIds) 1337 { 1338 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1339 doUnindexDocument(contentId, workspaceName, solrClient); 1340 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1341 } 1342 1343 getLogger().debug("Succesfully unindexed content from Solr."); 1344 } 1345 catch (Exception e) 1346 { 1347 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1348 getLogger().error(error, e); 1349 throw new IndexingException(error, e); 1350 } 1351 finally 1352 { 1353 // Restore workspace 1354 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1355 } 1356 } 1357 1358 /** 1359 * Add or update a content into Solr index 1360 * @param content The content to index 1361 * @param workspaceName The workspace where to index 1362 * @param solrClient The solr client to use 1363 * @throws Exception if an error occurs while indexing. 1364 */ 1365 protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception 1366 { 1367 long time_0 = System.currentTimeMillis(); 1368 1369 SolrInputDocument document = new SolrInputDocument(); 1370 List<SolrInputDocument> additionalDocuments = new ArrayList<>(); 1371 1372 _solrContentIndexer.indexContent(content, document, additionalDocuments); 1373 1374 long time_1 = System.currentTimeMillis(); 1375 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1376 1377 indexAclInitValues(content, document); 1378 1379 long time_2 = System.currentTimeMillis(); 1380 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1381 1382 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1383 documents.add(document); 1384 1385 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents); 1386 int status = solrResponse.getStatus(); 1387 1388 long time_3 = System.currentTimeMillis(); 1389 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1390 1391 if (status != 0) 1392 { 1393 throw new IOException("Content indexation: got status code '" + status + "'."); 1394 } 1395 } 1396 1397 /** 1398 * Indexes read-ACl initial values for object 1399 * @param ametysObject The object 1400 * @param document The Solr document 1401 */ 1402 public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document) 1403 { 1404 // Indexation of AmetysObject property 1405 document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true); 1406 1407 // Indexation of initValues for AllowedUsers 1408 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject); 1409 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed()); 1410 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed()); 1411 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString); 1412 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString); 1413 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString); 1414 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString); 1415 } 1416 1417 private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier) 1418 { 1419 if (values == null) 1420 { 1421 return; 1422 } 1423 1424 for (T value : values) 1425 { 1426 document.addField(fieldName, stringifier.apply(value)); 1427 } 1428 } 1429 1430 /** 1431 * Index the whole workflow of a content. 1432 * @param content The content. 1433 * @param workspaceName The workspace name 1434 * @param solrClient The solr client to use 1435 * @throws Exception if an error occurs while indexing. 1436 */ 1437 protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception 1438 { 1439 if (content instanceof WorkflowAwareContent) 1440 { 1441 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient); 1442 } 1443 } 1444 1445 /** 1446 * Index content attachments as new entries in the idnex 1447 * @param collection the collection of attachments 1448 * @param content the content whose attachments will be indexed 1449 * @throws Exception if something goes wrong when indexing the attachments of the content 1450 */ 1451 public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception 1452 { 1453 Request request = ContextHelper.getRequest(_context); 1454 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1455 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1456 indexContentAttachments(collection, content, solrClient); 1457 } 1458 1459 /** 1460 * Index content attachments as new entries in the idnex 1461 * @param collection the collection of attachments 1462 * @param content the content whose attachments will be indexed 1463 * @param solrClient The solr client to use 1464 * @throws Exception if something goes wrong when indexing the attachments of the content 1465 */ 1466 public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception 1467 { 1468 if (collection == null) 1469 { 1470 return; 1471 } 1472 1473 for (AmetysObject object : collection.getChildren()) 1474 { 1475 if (object instanceof ResourceCollection) 1476 { 1477 indexContentAttachments((ResourceCollection) object, content, solrClient); 1478 } 1479 else if (object instanceof Resource) 1480 { 1481 Resource resource = (Resource) object; 1482 indexContentAttachment(resource, content, solrClient); 1483 } 1484 } 1485 } 1486 1487 /** 1488 * Index a content attachment 1489 * @param resource the content attachment as a {@link Resource} 1490 * @param content the content whose attachment is going to be indexed 1491 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1492 */ 1493 public void indexContentAttachment(Resource resource, Content content) throws Exception 1494 { 1495 Request request = ContextHelper.getRequest(_context); 1496 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1497 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1498 indexContentAttachment(resource, content, solrClient); 1499 } 1500 1501 /** 1502 * Index a content attachment 1503 * @param resource the content attachment as a {@link Resource} 1504 * @param content the content whose attachment is going to be indexed 1505 * @param solrClient The solr client to use 1506 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1507 */ 1508 public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception 1509 { 1510 SolrInputDocument document = new SolrInputDocument(); 1511 1512 // Prepare resource doc 1513 _indexContentAttachment(resource, document, content); 1514 1515 // Indexation of the document 1516 _indexResourceDocument(resource, document, solrClient); 1517 } 1518 1519 private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception 1520 { 1521 String language = content.getLanguage(); 1522 1523 _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language); 1524 1525 // Need the id of the content for unindexing attachment during the unindexing of the content 1526 document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId()); 1527 } 1528 1529 /** 1530 * Index a populated solr input document of type Resource. 1531 * @param resource the resource from which the input document is created 1532 * @param document the input document 1533 * @param solrClient The solr client to use 1534 * @throws SolrServerException if there is an error on the server 1535 * @throws IOException if there is a communication error with the server 1536 */ 1537 protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException 1538 { 1539 String resourceId = resource.getId(); 1540 1541 // Retrieve appropriate collection name 1542 Request request = ContextHelper.getRequest(_context); 1543 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1544 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 1545 1546 // Add document 1547 UpdateResponse solrResponse = solrClient.add(collectionName, document); 1548 int status = solrResponse.getStatus(); 1549 1550 if (status != 0) 1551 { 1552 throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId); 1553 } 1554 1555 getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId); 1556 } 1557 1558 /** 1559 * Delete repeater documents of a specified content. 1560 * @param workspaceName The workspace name 1561 * @param contentId the content ID. 1562 * @param solrClient The solr client to use 1563 * @throws Exception if an error occurs while indexing. 1564 */ 1565 protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception 1566 { 1567 long time_0 = System.currentTimeMillis(); 1568 1569 // _documentType:repeater AND id:content\://xxx/* 1570 StringBuilder query = new StringBuilder(); 1571 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1572 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1573 1574 solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1575 1576 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1577 } 1578 1579 /** 1580 * Index all the resources in a given workspace. 1581 * @param workspaceName The workspace where to index 1582 * @param solrClient The solr client to use 1583 * @return The indexation result as a Map. 1584 * @throws Exception if an error occurs while indexing. 1585 */ 1586 public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1587 { 1588 Request request = ContextHelper.getRequest(_context); 1589 1590 // Retrieve the current workspace. 1591 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1592 1593 try 1594 { 1595 // Force the workspace. 1596 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1597 1598 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1599 1600 long start = System.currentTimeMillis(); 1601 1602 // Delete all resources 1603 _unindexAllResources(workspaceName, solrClient); 1604 1605 Map<String, Object> results = new HashMap<>(); 1606 1607 try 1608 { 1609 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1610 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1611 1612 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient); 1613 1614 long end = System.currentTimeMillis(); 1615 1616 if (!result.hasErrors()) 1617 { 1618 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1619 } 1620 else 1621 { 1622 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1623 } 1624 1625 results.put("successCount", result.getSuccessCount()); 1626 if (result.hasErrors()) 1627 { 1628 results.put("errorCount", result.getErrorCount()); 1629 } 1630 } 1631 catch (UnknownAmetysObjectException e) 1632 { 1633 getLogger().info("There is no root for resources in current workspace."); 1634 } 1635 1636 return results; 1637 } 1638 catch (Exception e) 1639 { 1640 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1641 getLogger().error(error, e); 1642 throw new IndexingException(error, e); 1643 } 1644 finally 1645 { 1646 // Restore context 1647 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1648 } 1649 } 1650 1651 /** 1652 * Send a collection of contents for indexation in the solr server. 1653 * @param resources the collection of contents to index. 1654 * @param documentType The document type of the resource 1655 * @param workspaceName The workspace where to index 1656 * @return the indexation result. 1657 * @throws Exception if an error occurs while indexing. 1658 */ 1659 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception 1660 { 1661 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1662 return indexResources(resources, documentType, null, workspaceName, solrClient); 1663 } 1664 1665 /** 1666 * Send a collection of contents for indexation in the solr server. 1667 * @param resources the collection of contents to index. 1668 * @param documentType The document type of the resource 1669 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1670 * @param workspaceName The workspace where to index 1671 * @param solrClient The solr client to use 1672 * @return the indexation result. 1673 * @throws Exception if an error occurs while indexing. 1674 */ 1675 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1676 { 1677 Request request = ContextHelper.getRequest(_context); 1678 1679 // Retrieve the current workspace. 1680 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1681 1682 try 1683 { 1684 // Force the workspace. 1685 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1686 1687 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1688 1689 long start = System.currentTimeMillis(); 1690 1691 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient); 1692 1693 long end = System.currentTimeMillis(); 1694 1695 if (!result.hasErrors()) 1696 { 1697 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1698 } 1699 else 1700 { 1701 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1702 } 1703 1704 return result; 1705 } 1706 catch (Exception e) 1707 { 1708 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1709 getLogger().error(error, e); 1710 throw new IndexingException(error, e); 1711 } 1712 finally 1713 { 1714 // Restore context 1715 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1716 } 1717 } 1718 1719 /** 1720 * Send some resources for indexation in the solr server. 1721 * @param resources the resources to index. 1722 * @param documentType The document type of the resource 1723 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1724 * @param workspaceName The workspace where to index 1725 * @param solrClient The solr client to use 1726 * @return the indexation result. 1727 * @throws Exception if an error occurs committing the results. 1728 */ 1729 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1730 { 1731 int successCount = 0; 1732 int errorCount = 0; 1733 for (AmetysObject resource : resources) 1734 { 1735 try 1736 { 1737 doIndexExplorerItem(resource, documentType, resourceRoot, solrClient); 1738 successCount++; 1739 } 1740 catch (Exception e) 1741 { 1742 getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e); 1743 errorCount++; 1744 } 1745 } 1746 1747 return new IndexationResult(successCount, errorCount); 1748 } 1749 1750 /** 1751 * Add or update a resource into Solr index 1752 * @param resource The resource to index 1753 * @param documentType The document type of the resource 1754 * @param workspaceName The workspace where to index 1755 * @throws Exception if an error occurs while indexing. 1756 */ 1757 public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception 1758 { 1759 Request request = ContextHelper.getRequest(_context); 1760 1761 // Retrieve the current workspace. 1762 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1763 1764 try 1765 { 1766 // Force the workspace. 1767 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1768 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1769 1770 getLogger().debug("Indexing resource {} into Solr.", resource.getId()); 1771 1772 doIndexResource(resource, documentType, null, solrClient); 1773 1774 getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId()); 1775 } 1776 catch (Exception e) 1777 { 1778 String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName); 1779 getLogger().error(error, e); 1780 throw new IndexingException(error, e); 1781 } 1782 finally 1783 { 1784 // Restore context 1785 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1786 } 1787 } 1788 1789 private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1790 { 1791 getLogger().debug("Unindexing all resources from Solr."); 1792 1793 String collection = _solrClientProvider.getCollectionName(workspaceName); 1794 solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1795 1796 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1797 } 1798 1799 /** 1800 * Delete all resource documents at a given path for all workspaces and commit 1801 * @param rootId The resource root ID, must not be null. 1802 * @param path The resource path relative to the given root, must start with a slash. 1803 * @throws Exception If an error occurs while unindexing. 1804 */ 1805 public void unindexResourcesByPath(String rootId, String path) throws Exception 1806 { 1807 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1808 for (String workspaceName : workspaceNames) 1809 { 1810 unindexResourcesByPath(rootId, path, workspaceName); 1811 } 1812 } 1813 1814 /** 1815 * Delete all resource documents at a given path. 1816 * @param rootId The resource root ID, must not be null. 1817 * @param path The resource path relative to the given root, must start with a slash. 1818 * @param workspaceName The workspace where to work in 1819 * @throws Exception If an error occurs while unindexing. 1820 */ 1821 public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception 1822 { 1823 Request request = ContextHelper.getRequest(_context); 1824 1825 // Retrieve the current workspace. 1826 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1827 1828 try 1829 { 1830 // Force the workspace. 1831 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1832 1833 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1834 1835 Query query = new ResourceLocationQuery(rootId, path); 1836 1837 String collection = _solrClientProvider.getCollectionName(workspaceName); 1838 _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1839 1840 getLogger().debug("Succesfully deleted resource document from Solr."); 1841 } 1842 catch (Exception e) 1843 { 1844 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1845 getLogger().error(error, e); 1846 throw new IndexingException(error, e); 1847 } 1848 finally 1849 { 1850 // Restore workspace 1851 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1852 } 1853 } 1854 1855 /** 1856 * Remove a resource from Solr index for all workspaces and commit 1857 * @param resourceId The id of resource to unindex 1858 * @throws Exception if an error occurs while indexing. 1859 */ 1860 public void unindexResource(String resourceId) throws Exception 1861 { 1862 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1863 for (String workspaceName : workspaceNames) 1864 { 1865 unindexResource(resourceId, workspaceName); 1866 } 1867 } 1868 1869 /** 1870 * Remove a resource from the Solr index. 1871 * @param resourceId The id of resource to unindex 1872 * @param workspaceName The workspace where to work in 1873 * @throws Exception if an error occurs while unindexing. 1874 */ 1875 public void unindexResource(String resourceId, String workspaceName) throws Exception 1876 { 1877 Request request = ContextHelper.getRequest(_context); 1878 1879 // Retrieve the current workspace. 1880 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1881 1882 try 1883 { 1884 // Force the workspace. 1885 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1886 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1887 1888 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 1889 1890 doUnindexDocument(resourceId, workspaceName, solrClient); 1891 1892 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 1893 } 1894 catch (Exception e) 1895 { 1896 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 1897 getLogger().error(error, e); 1898 throw new IndexingException(error, e); 1899 } 1900 finally 1901 { 1902 // Restore workspace 1903 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1904 } 1905 } 1906 1907 /** 1908 * Add or update a resource into Solr index 1909 * @param node The resource to index 1910 * @param documentType The document type of the resource 1911 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1912 * @param solrClient The solr client to use 1913 * @throws Exception if an error occurs while indexing. 1914 */ 1915 protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 1916 { 1917 if (node instanceof ResourceCollection) 1918 { 1919 for (AmetysObject child : ((ResourceCollection) node).getChildren()) 1920 { 1921 doIndexExplorerItem(child, documentType, resourceRoot, solrClient); 1922 } 1923 } 1924 else if (node instanceof Resource) 1925 { 1926 doIndexResource((Resource) node, documentType, resourceRoot, solrClient); 1927 } 1928 } 1929 1930 /** 1931 * Add or update a resource into Solr index 1932 * @param resource The resource to index 1933 * @param documentType The document type of the resource 1934 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1935 * @param solrClient The solr client to use 1936 * @throws Exception if an error occurs while indexing. 1937 */ 1938 protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 1939 { 1940 SolrInputDocument document = new SolrInputDocument(); 1941 1942 _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot); 1943 1944 String workspaceName = _workspaceSelector.getWorkspace(); 1945 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), document); 1946 int status = solrResponse.getStatus(); 1947 1948 if (status != 0) 1949 { 1950 throw new IOException("Resource indexation: got status code '" + status + "'."); 1951 } 1952 } 1953 1954 /** 1955 * Process a Solr commit operation in all workspaces. 1956 * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient} 1957 * @throws SolrServerException if there is an error on the server 1958 * @throws IOException if there is a communication error with the server 1959 */ 1960 public void commit() throws SolrServerException, IOException 1961 { 1962 String[] workspaceNames; 1963 try 1964 { 1965 workspaceNames = _workspaceSelector.getWorkspaces(); 1966 for (String workspaceName : workspaceNames) 1967 { 1968 SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName); 1969 commit(workspaceName, solrClient); 1970 } 1971 } 1972 catch (RepositoryException e) 1973 { 1974 throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e); 1975 } 1976 } 1977 1978 /** 1979 * Process a Solr commit operation in given workspace. 1980 * @param workspaceName The workspace's name 1981 * @param solrClient The solr client to use 1982 * @throws SolrServerException if there is an error on the server 1983 * @throws IOException if there is a communication error with the server 1984 */ 1985 public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 1986 { 1987 long time_0 = System.currentTimeMillis(); 1988 1989 // Commit 1990 UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName)); 1991 int status = solrResponse.getStatus(); 1992 1993 if (status != 0) 1994 { 1995 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 1996 } 1997 1998 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 1999 } 2000 2001 /** 2002 * Launch a solr index optimization. 2003 * @param workspaceName The workspace's name 2004 * @param solrClient The solr client to use 2005 * @throws SolrServerException if there is an error on the server 2006 * @throws IOException if there is a communication error with the server 2007 */ 2008 public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2009 { 2010 UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName)); 2011 int status = solrResponse.getStatus(); 2012 2013 if (status != 0) 2014 { 2015 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2016 } 2017 2018 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 2019 } 2020 2021 /** 2022 * Delete all documents from the solr index. 2023 * @param workspaceName The workspace name 2024 * @param solrClient The solr client to use 2025 * @throws Exception if an error occurs while unindexing. 2026 */ 2027 public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception 2028 { 2029 getLogger().debug("Deleting all documents from Solr."); 2030 2031 String collection = _solrClientProvider.getCollectionName(workspaceName); 2032 solrClient.deleteByQuery(collection, "*:*"); 2033 2034 getLogger().debug("Successfully deleted all documents from Solr."); 2035 } 2036 2037 /** 2038 * Delete a document from the Solr server. 2039 * @param id The id of the document to delete from Solr 2040 * @param workspaceName The workspace name 2041 * @param solrClient The solr client to use 2042 * @throws Exception if an error occurs while indexing. 2043 */ 2044 protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception 2045 { 2046 UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 2047 int status = solrResponse.getStatus(); 2048 2049 if (status != 0) 2050 { 2051 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 2052 } 2053 } 2054 2055 /** 2056 * Delete content attachments documents of a given content from the Solr server. 2057 * @param contentId The id of the content 2058 * @param workspaceName The workspace name 2059 * @param solrClient The solr client to use 2060 * @throws Exception if an error occurs while indexing. 2061 */ 2062 protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception 2063 { 2064 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 2065 2066 Query query = new ContentAttachmentQuery(contentId); 2067 UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build()); 2068 int status = solrResponse.getStatus(); 2069 2070 if (status != 0) 2071 { 2072 throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'."); 2073 } 2074 } 2075 2076// /** 2077// * Get the collection to use. 2078// * @return The name of the collection to index into. 2079// */ 2080// protected String getCollection() 2081// { 2082// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 2083// } 2084 2085 class IndexationResult 2086 { 2087 protected int _successCount; 2088 2089 protected int _errorCount; 2090 2091 /** 2092 * Constructor 2093 */ 2094 public IndexationResult() 2095 { 2096 this(0, 0); 2097 } 2098 2099 /** 2100 * Constructor 2101 * @param successCount The success count. 2102 * @param errorCount The error count. 2103 */ 2104 public IndexationResult(int successCount, int errorCount) 2105 { 2106 this._successCount = successCount; 2107 this._errorCount = errorCount; 2108 } 2109 2110 /** 2111 * Get the successCount. 2112 * @return the successCount 2113 */ 2114 public int getSuccessCount() 2115 { 2116 return _successCount; 2117 } 2118 2119 /** 2120 * Set the successCount. 2121 * @param successCount the successCount to set 2122 */ 2123 public void setSuccessCount(int successCount) 2124 { 2125 this._successCount = successCount; 2126 } 2127 2128 /** 2129 * Test if the indexation had errors. 2130 * @return true if the indexation had errors, false otherwise. 2131 */ 2132 public boolean hasErrors() 2133 { 2134 return _errorCount > 0; 2135 } 2136 2137 /** 2138 * Get the errorCount. 2139 * @return the errorCount 2140 */ 2141 public int getErrorCount() 2142 { 2143 return _errorCount; 2144 } 2145 2146 /** 2147 * Set the errorCount. 2148 * @param errorCount the errorCount to set 2149 */ 2150 public void setErrorCount(int errorCount) 2151 { 2152 this._errorCount = errorCount; 2153 } 2154 } 2155 2156}