001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.nio.ByteBuffer; 020import java.nio.CharBuffer; 021import java.nio.charset.CharsetDecoder; 022import java.nio.charset.CodingErrorAction; 023import java.nio.charset.StandardCharsets; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Collection; 028import java.util.Collections; 029import java.util.Comparator; 030import java.util.HashMap; 031import java.util.HashSet; 032import java.util.List; 033import java.util.Map; 034import java.util.Objects; 035import java.util.Set; 036import java.util.concurrent.Future; 037import java.util.function.Function; 038import java.util.stream.Collectors; 039import java.util.stream.StreamSupport; 040 041import javax.jcr.RepositoryException; 042 043import org.apache.avalon.framework.activity.Initializable; 044import org.apache.avalon.framework.component.Component; 045import org.apache.avalon.framework.context.Context; 046import org.apache.avalon.framework.context.ContextException; 047import org.apache.avalon.framework.context.Contextualizable; 048import org.apache.avalon.framework.service.ServiceException; 049import org.apache.avalon.framework.service.ServiceManager; 050import org.apache.avalon.framework.service.Serviceable; 051import org.apache.cocoon.Constants; 052import org.apache.cocoon.components.ContextHelper; 053import org.apache.cocoon.environment.Request; 054import org.apache.commons.collections4.IterableUtils; 055import org.apache.commons.lang3.ObjectUtils; 056import org.apache.commons.lang3.StringUtils; 057import org.apache.solr.client.solrj.SolrClient; 058import org.apache.solr.client.solrj.SolrResponse; 059import org.apache.solr.client.solrj.SolrServerException; 060import org.apache.solr.client.solrj.request.CoreAdminRequest; 061import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; 062import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 063import org.apache.solr.client.solrj.request.schema.SchemaRequest; 064import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 065import org.apache.solr.client.solrj.response.CoreAdminResponse; 066import org.apache.solr.client.solrj.response.SolrResponseBase; 067import org.apache.solr.client.solrj.response.UpdateResponse; 068import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 069import org.apache.solr.client.solrj.response.schema.SchemaResponse; 070import org.apache.solr.client.solrj.util.ClientUtils; 071import org.apache.solr.common.SolrInputDocument; 072import org.apache.solr.common.params.CoreAdminParams; 073import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; 074import org.apache.solr.common.params.ModifiableSolrParams; 075import org.apache.solr.common.params.SolrParams; 076import org.apache.solr.common.util.NamedList; 077import org.slf4j.Logger; 078 079import org.ametys.cms.indexing.IndexingException; 080import org.ametys.cms.indexing.solr.AbstractIndexerCallable; 081import org.ametys.cms.indexing.solr.IndexationResult; 082import org.ametys.cms.indexing.solr.ReloadAclCacheRequest; 083import org.ametys.cms.indexing.solr.ThreadIndexerHelper; 084import org.ametys.cms.indexing.solr.UpdateAclCacheRequest; 085import org.ametys.cms.indexing.solr.UpdateCorePropertyRequest; 086import org.ametys.cms.repository.Content; 087import org.ametys.cms.repository.ContentQueryHelper; 088import org.ametys.cms.repository.WorkflowAwareContent; 089import org.ametys.cms.rights.solrchecking.ReadAccessHelper; 090import org.ametys.cms.search.query.ContentAttachmentQuery; 091import org.ametys.cms.search.query.DocumentTypeQuery; 092import org.ametys.cms.search.query.OrQuery; 093import org.ametys.cms.search.query.Query; 094import org.ametys.cms.search.query.ResourceLocationQuery; 095import org.ametys.cms.search.solr.NoAutoCommitUpdateClient; 096import org.ametys.cms.search.solr.SolrClientProvider; 097import org.ametys.cms.search.solr.schema.SchemaDefinition; 098import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 099import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 100import org.ametys.cms.search.solr.schema.SchemaFields; 101import org.ametys.cms.search.solr.schema.SchemaHelper; 102import org.ametys.core.group.GroupIdentity; 103import org.ametys.core.right.AllowedUsers; 104import org.ametys.core.schedule.progression.ContainerProgressionTracker; 105import org.ametys.core.schedule.progression.ProgressionTrackerFactory; 106import org.ametys.core.schedule.progression.SimpleProgressionTracker; 107import org.ametys.core.user.UserIdentity; 108import org.ametys.core.util.DateUtils; 109import org.ametys.plugins.explorer.resources.Resource; 110import org.ametys.plugins.explorer.resources.ResourceCollection; 111import org.ametys.plugins.repository.AmetysObject; 112import org.ametys.plugins.repository.AmetysObjectIterable; 113import org.ametys.plugins.repository.AmetysObjectResolver; 114import org.ametys.plugins.repository.RepositoryConstants; 115import org.ametys.plugins.repository.TraversableAmetysObject; 116import org.ametys.plugins.repository.UnknownAmetysObjectException; 117import org.ametys.plugins.repository.collection.AmetysObjectCollection; 118import org.ametys.plugins.repository.provider.AbstractRepository; 119import org.ametys.plugins.repository.provider.JackrabbitRepository; 120import org.ametys.plugins.repository.provider.RequestAttributeWorkspaceSelector; 121import org.ametys.plugins.repository.provider.WorkspaceSelector; 122import org.ametys.runtime.config.Config; 123import org.ametys.runtime.i18n.I18nizableText; 124import org.ametys.runtime.plugin.component.AbstractLogEnabled; 125 126/** 127 * Solr indexer. 128 */ 129public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 130{ 131 /** The component role. */ 132 public static final String ROLE = SolrIndexer.class.getName(); 133 134 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 135 136 private static final String _CONFIGSET_NAME_PREFIX = "configset-"; 137 138 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_version_", "_text_"); 139 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "plong", "text_general"); 140 141 private static final int __SOLR_STRING_NB_BYTES_LIMIT = 32766; 142 143 /** The service manager. */ 144 protected ServiceManager _manager; 145 /** The ametys object resolver. */ 146 protected AmetysObjectResolver _resolver; 147 /** The schema definition provider extension point. */ 148 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 149 /** The schema helper. */ 150 protected SchemaHelper _schemaHelper; 151 /** Solr Ametys contents indexer */ 152 protected SolrContentIndexer _solrContentIndexer; 153 /** Solr workflow indexer. */ 154 protected SolrWorkflowIndexer _solrWorkflowIndexer; 155 /** Solr resource indexer. */ 156 protected SolrResourceIndexer _solrResourceIndexer; 157 158 /** The Solr client provider */ 159 protected SolrClientProvider _solrClientProvider; 160 161 /** The solr core prefix. */ 162 protected String _solrCorePrefix; 163 /** The Ametys internal URL used by Solr to query Ametys */ 164 protected String _ametysInternalUrl; 165 166 /** The workspace selector. */ 167 protected WorkspaceSelector _workspaceSelector; 168 /** The JCR repository */ 169 protected JackrabbitRepository _repository; 170 /** The helper for read access */ 171 protected ReadAccessHelper _readAccessHelper; 172 /** The thread indexer helper */ 173 protected ThreadIndexerHelper _threadIndexerHelper; 174 175 /** The avalon context */ 176 protected Context _context; 177 /** Cocoon Context */ 178 protected org.apache.cocoon.environment.Context _cocoonContext; 179 180 /** 181 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 182 * @return The date format for indexing dates 183 * @deprecated use {@link DateUtils#zonedDateTimeToString(java.time.ZonedDateTime, java.time.ZoneId)} using UTC zone id instead 184 */ 185 @Deprecated 186 public static SimpleDateFormat dateFormat() 187 { 188 if (__DATE_FORMAT.get() == null) 189 { 190 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 191 } 192 return __DATE_FORMAT.get(); 193 } 194 195 /** 196 * Truncates (if needed) the given string in order to be indexed without <i>immense term</i> error by Solr. 197 * Only the {@value #__SOLR_STRING_NB_BYTES_LIMIT} first bytes of the String will be kept. 198 * @param value The string value to index 199 * @param logger The logger for logging in WARN level in case the given string is too long and will be truncated. Can be null if you do not want to log. 200 * @param documentId The id of the document being indexed. Can be null if you do not want to log. 201 * @param fieldName The name of the field being indexed. Can be null if you do not want to log. 202 * @return The given string value, or its truncation if it is too long (greater than {@value #__SOLR_STRING_NB_BYTES_LIMIT} bytes) 203 */ 204 public static String truncateUtf8StringValue(String value, Logger logger, String documentId , String fieldName) 205 { 206 if (value.length() * 4 <= __SOLR_STRING_NB_BYTES_LIMIT) 207 { 208 // With UTF-8, a character is encoded using 1, 2, 3 or 4 bytes, so (value.length() <= value.getBytes().length <= 4 * value.length()) 209 // As a result, value.getBytes().length <= limit 210 return value; 211 } 212 213 // There is a doubt, the string may need to be truncated (or not) 214 byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); 215 int bytesLength = valueBytes.length; 216 if (bytesLength <= __SOLR_STRING_NB_BYTES_LIMIT) 217 { 218 return value; 219 } 220 221 if (ObjectUtils.allNotNull(logger, documentId, fieldName)) 222 { 223 logger.warn("The string value for document '{}' and field name '{}' is longer ({}) than the max bytes length {}. It will be truncated to prevent Solr error, but you should consider verifying why this string is so long.", documentId, fieldName, bytesLength, __SOLR_STRING_NB_BYTES_LIMIT); 224 } 225 226 // Need a truncation (inspired by https://stackoverflow.com/questions/119328/how-do-i-truncate-a-java-string-to-fit-in-a-given-number-of-bytes-once-utf-8-en#answer-35148974) 227 CharBuffer charBuffer = CharBuffer.allocate(__SOLR_STRING_NB_BYTES_LIMIT); 228 CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() 229 .onMalformedInput(CodingErrorAction.IGNORE); 230 decoder.decode(ByteBuffer.wrap(valueBytes, 0, __SOLR_STRING_NB_BYTES_LIMIT), charBuffer, true); 231 decoder.flush(charBuffer); 232 return new String(charBuffer.array(), 0, charBuffer.position()); 233 } 234 235 @Override 236 public void service(ServiceManager serviceManager) throws ServiceException 237 { 238 _manager = serviceManager; 239 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 240 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 241 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 242 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 243 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 244 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 245 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 246 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 247 _readAccessHelper = (ReadAccessHelper) serviceManager.lookup(ReadAccessHelper.ROLE); 248 _repository = (JackrabbitRepository) serviceManager.lookup(AbstractRepository.ROLE); 249 _threadIndexerHelper = (ThreadIndexerHelper) serviceManager.lookup(ThreadIndexerHelper.ROLE); 250 } 251 252 @Override 253 public void initialize() throws Exception 254 { 255 Config config = Config.getInstance(); 256 _solrCorePrefix = config.getValue("cms.solr.core.prefix"); 257 258 _ametysInternalUrl = config.getValue("cms.solr.core.ametys.internal.url"); 259 if (StringUtils.isBlank(_ametysInternalUrl)) 260 { 261 // fallback to CMS URL as internal URL is an optional parameter 262 _ametysInternalUrl = config.getValue("cms.url"); 263 } 264 } 265 266 @Override 267 public void contextualize(Context context) throws ContextException 268 { 269 _context = context; 270 _cocoonContext = (org.apache.cocoon.environment.Context) context.get(Constants.CONTEXT_ENVIRONMENT_CONTEXT); 271 } 272 273 /** 274 * Gets the 'autocommit' Solr client 275 * @param workspaceName The name of the workspace 276 * @return the Solr client 277 */ 278 protected SolrClient _getAutoCommitSolrClient(String workspaceName) 279 { 280 return _solrClientProvider.getUpdateClient(workspaceName, true); 281 } 282 283 /** 284 * Gets the 'no autocommit' Solr client 285 * @param workspaceName The name of the workspace 286 * @return the Solr client 287 */ 288 protected SolrClient _getNoAutoCommitSolrClient(String workspaceName) 289 { 290 return _solrClientProvider.getUpdateClient(workspaceName, false); 291 } 292 293 // for admin operations 294 private SolrClient _defaultSolrClient() 295 { 296 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 297 } 298 299 /** 300 * Get the names of the Solr cores. 301 * @return The names of the Solr cores. 302 * @throws IOException If an I/O error occurs. 303 * @throws SolrServerException If a Solr error occurs. 304 */ 305 public Set<String> getCoreNames() throws IOException, SolrServerException 306 { 307 Set<String> coreNames = new HashSet<>(); 308 309 getLogger().debug("Getting core list."); 310 311 CoreAdminRequest req = new CoreAdminRequest(); 312 req.setAction(CoreAdminAction.STATUS); 313 314 NamedList<NamedList<Object>> status = req.process(_defaultSolrClient()).getCoreStatus(); 315 for (Map.Entry<String, NamedList<Object>> core : status) 316 { 317 String fullName = (String) core.getValue().get("name"); 318 if (fullName.startsWith(_solrCorePrefix)) 319 { 320 coreNames.add(fullName.substring(_solrCorePrefix.length())); 321 } 322 } 323 324 return coreNames; 325 } 326 327 /** 328 * Get the names of the Solr cores. 329 * @return The names of the Solr cores. 330 * @throws IOException If an I/O error occurs. 331 * @throws SolrServerException If a Solr error occurs. 332 */ 333 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 334 { 335 Set<String> coreNames = new HashSet<>(); 336 337 getLogger().debug("Getting core list."); 338 339 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 340 341 NamedList<NamedList<Object>> status = response.getCoreStatus(); 342 for (Map.Entry<String, NamedList<Object>> core : status) 343 { 344 String fullName = (String) core.getValue().get("name"); 345 if (fullName.startsWith(_solrCorePrefix)) 346 { 347 coreNames.add(fullName.substring(_solrCorePrefix.length())); 348 } 349 } 350 351 return coreNames; 352 } 353 354 /** 355 * Create a Solr core. 356 * @param name The name of the core to create. 357 * @throws IOException If an I/O error occurs. 358 * @throws SolrServerException If a Solr error occurs. 359 */ 360 public void createCore(String name) throws IOException, SolrServerException 361 { 362 Set<String> cores = getCoreNames(); 363 if (!cores.contains(name)) 364 { 365 String fullName = _solrCorePrefix + name; 366 String configsetName = _CONFIGSET_NAME_PREFIX + (_solrCorePrefix.endsWith("-") ? _solrCorePrefix.substring(0, _solrCorePrefix.length() - 1) : _solrCorePrefix); 367 _createConfigset(configsetName); 368 369 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 370 371 Create createRequest = new Create() { 372 @Override 373 public SolrParams getParams() 374 { 375 ModifiableSolrParams params = new ModifiableSolrParams(); 376 377 params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString()); 378 params.set(CoreAdminParams.NAME, fullName); 379 params.set(CoreAdminParams.CONFIGSET, configsetName); 380 params.set("property.ametys.url", _ametysInternalUrl); 381 382 return params; 383 } 384 }; 385 386 NamedList<?> results = createRequest.process(_defaultSolrClient()).getResponse(); 387 388 NamedList<?> error = (NamedList<?>) results.get("error"); 389 if (error != null) 390 { 391 throw new IOException("Error creating the core: " + error.get("msg")); 392 } 393 } 394 else 395 { 396 if (getLogger().isDebugEnabled()) 397 { 398 getLogger().debug("Core '" + name + "' already exists, skipping it."); 399 } 400 } 401 } 402 403 /** 404 * Updates the ametys.url property of the Solr cores. 405 */ 406 public void updateAmetysUrlCoreProperty() 407 { 408 Set<String> coreNames; 409 try 410 { 411 coreNames = getCoreNames(); 412 } 413 catch (SolrServerException | IOException e) 414 { 415 getLogger().error("Cannot get Solr core names. As a result, the internal Ametys URL could not be updated on Solr server.", e); 416 return; 417 } 418 419 for (String coreName : coreNames) 420 { 421 String collection = _solrClientProvider.getCollectionName(coreName); 422 SolrResponseBase response; 423 try 424 { 425 response = new UpdateCorePropertyRequest("ametys.url", _ametysInternalUrl).process(_getAutoCommitSolrClient(coreName), collection); 426 } 427 catch (SolrServerException | IOException e) 428 { 429 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName, e); 430 continue; 431 } 432 433 NamedList<Object> responseParams = response.getResponse(); 434 if ("ok".equals(responseParams.get("result"))) 435 { 436 Boolean valueChanged = responseParams.getBooleanArg("valueChanged"); 437 if (valueChanged) 438 { 439 getLogger().info("'core.properties' file updated with the up-to-date Ametys URL for workspace '{}'", coreName); 440 } 441 else 442 { 443 getLogger().info("'core.properties' file already has the up-to-date Ametys URL for workspace '{}', it was not modified.", coreName); 444 } 445 } 446 else 447 { 448 getLogger().error("'core.properties' file updating for workspace '{}' did not succeed as expected.", coreName); 449 } 450 } 451 } 452 453 private void _createConfigset(String name) throws IOException, SolrServerException 454 { 455 getLogger().info("Creating (if necessary) configset '{}'", name); 456 457 // This request handler will check if configset exists. If not it will be created. 458 CoreAdminRequest request = new CoreAdminRequest() { 459 @Override 460 public SolrParams getParams() 461 { 462 ModifiableSolrParams params = new ModifiableSolrParams(); 463 params.set(CoreAdminParams.ACTION, "createConfigset"); 464 params.set(CoreAdminParams.NAME, name); 465 return params; 466 } 467 }; 468 469 NamedList<?> results = request.process(_defaultSolrClient()).getResponse(); 470 471 NamedList<?> error = (NamedList<?>) results.get("error"); 472 if (error != null) 473 { 474 throw new IOException("Error creating the core: " + error.get("msg")); 475 } 476 } 477 478 /** 479 * Delete a Solr core. 480 * @param name The name of the core to delete. 481 * @throws IOException If an I/O error occurs. 482 * @throws SolrServerException If a Solr error occurs. 483 */ 484 public void deleteCore(String name) throws IOException, SolrServerException 485 { 486 String fullName = _solrCorePrefix + name; 487 488 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 489 490 CoreAdminResponse response = CoreAdminRequest.unloadCore(fullName, true, true, _defaultSolrClient()); 491 NamedList<?> results = response.getResponse(); 492 493 NamedList<?> error = (NamedList<?>) results.get("error"); 494 if (error != null) 495 { 496 throw new IOException("Error deleting core" + name + ": " + error.get("msg")); 497 } 498 } 499 500 /** 501 * Send the schema. 502 * @throws IOException If a communication error occurs. 503 * @throws SolrServerException If a solr error occurs. 504 */ 505 public void sendSchema() throws IOException, SolrServerException 506 { 507 getLogger().info("Computing and sending the schema to the solr server."); 508 509 String workspaceName = _workspaceSelector.getWorkspace(); 510 String collection = _solrClientProvider.getCollectionName(workspaceName); 511 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 512 513// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 514 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 515 516 // TODO Clear the schema except fields marked ametysReadOnly="true". 517 518 // Clear the current schema. 519 clearSchema(solrClient, collection); 520 521 SchemaRequest schemaRequest = new SchemaRequest(); 522 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 523 524 // The cleared schema contains only the basic fields which can't be deleted. 525 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 526 SchemaFields schemaFields = new SchemaFields(clearedSchema); 527 528 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 529 530 // Add the static schema types and fields. 531 List<SchemaRequest.Update> updates = new ArrayList<>(); 532 533 // Set "add field" definitions from the static schema to the update list. 534 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 535 536 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 537 538 // Set "add field" definitions from the static schema to the update list. 539 addCustomUpdates(updates, schemaFields); 540 541 updates.sort(new SchemaRequestComparator()); 542 543 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 544 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 545 546 getLogger().debug("Send schema response: {}", updateResponse.toString()); 547 Object errors = updateResponse.getResponse().get("errors"); 548 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 549 { 550 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 551 throw new SolrServerException(msg); 552 } 553 554 getLogger().info("Schema sent to the solr server."); 555 556 reloadCores(); 557 } 558 559 /** 560 * Compute the list of {@link Update} directives from the static schema. 561 * @param updates The list of {@link Update} directives to fill. 562 * @param staticSchema The static schema representation. 563 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 564 */ 565 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 566 { 567 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 568 for (FieldTypeDefinition fieldType : fieldTypes) 569 { 570 String name = (String) fieldType.getAttributes().get("name"); 571 if (!schemaFields.hasFieldType(name)) 572 { 573 updates.add(new SchemaRequest.AddFieldType(fieldType)); 574 schemaFields.addFieldType(name); 575 } 576 } 577 for (Map<String, Object> field : staticSchema.getFields()) 578 { 579 String name = (String) field.get("name"); 580 if (!schemaFields.hasField(name)) 581 { 582 updates.add(new SchemaRequest.AddField(field)); 583 schemaFields.addField(name); 584 } 585 } 586 for (Map<String, Object> field : staticSchema.getDynamicFields()) 587 { 588 String name = (String) field.get("name"); 589 if (!schemaFields.hasDynamicField(name)) 590 { 591 updates.add(new SchemaRequest.AddDynamicField(field)); 592 schemaFields.addDynamicField(name); 593 } 594 } 595 for (Map<String, Object> field : staticSchema.getCopyFields()) 596 { 597 String source = (String) field.get("source"); 598 String dest = (String) field.get("dest"); 599 if (!schemaFields.hasCopyField(source, dest)) 600 { 601 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 602 schemaFields.addCopyField(source, dest); 603 } 604 } 605 } 606 607 /** 608 * Compute the list of custom {@link Update} directives. 609 * @param updates The list of {@link Update} directives to fill. 610 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 611 */ 612 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 613 { 614 // Add all our property-managed fields. 615 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 616 { 617 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 618 619 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 620 { 621 if (!definition.exists(schemaFields)) 622 { 623 SchemaRequest.Update update = definition.getSchemaUpdate(); 624 if (update != null) 625 { 626 updates.add(update); 627 } 628 } 629 } 630 } 631 } 632 633 /** 634 * Delete all the fields of the existing schema in the given collection. 635 * @param solrClient The Solr client 636 * @param collection The collection. 637 * @throws IOException If a communication error occurs. 638 * @throws SolrServerException If a solr error occurs. 639 */ 640 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 641 { 642 try 643 { 644 getLogger().info("Clearing the existing schema on the solr server."); 645 646 SchemaRequest schemaRequest = new SchemaRequest(); 647 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 648 649 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 650 651 List<SchemaRequest.Update> deletions = new ArrayList<>(); 652 653 // First the copy fields, then dynamic and simple fields, and field types in the end. 654 for (Map<String, Object> field : schema.getCopyFields()) 655 { 656 String source = (String) field.get("source"); 657 String dest = (String) field.get("dest"); 658 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 659 } 660 for (Map<String, Object> field : schema.getDynamicFields()) 661 { 662 String name = (String) field.get("name"); 663 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 664 } 665 for (Map<String, Object> field : schema.getFields()) 666 { 667 String name = (String) field.get("name"); 668 if (!_READ_ONLY_FIELDS.contains(name)) 669 { 670 deletions.add(new SchemaRequest.DeleteField(name)); 671 } 672 } 673 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 674 { 675 String name = (String) fieldType.getAttributes().get("name"); 676 if (!_READ_ONLY_FIELDTYPES.contains(name)) 677 { 678 deletions.add(new SchemaRequest.DeleteFieldType(name)); 679 } 680 } 681 682 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 683 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 684 685 Object errors = updateResponse.getResponse().get("errors"); 686 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 687 { 688 String msg = "An error occured when clearing Solr schema, it contains errors:\n" + errors.toString(); 689 throw new SolrServerException(msg); 690 } 691 else 692 { 693 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 694 } 695 696 getLogger().info("Solr schema cleared."); 697 } 698 catch (SolrServerException | IOException e) 699 { 700 getLogger().error("Error clearing schema in collection " + collection, e); 701 throw e; 702 } 703 } 704 705 /** 706 * Reload the solr cores. 707 * @throws IOException If a communication error occurs. 708 * @throws SolrServerException If a solr error occurs. 709 */ 710 protected void reloadCores() throws IOException, SolrServerException 711 { 712 getLogger().info("Reloading solr cores."); 713 714 for (String coreName : getCoreNames()) 715 { 716 String fullName = _solrCorePrefix + coreName; 717 718 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 719 720 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 721 } 722 723 getLogger().info("All cores reloaded."); 724 } 725 726 /** 727 * Reloads the ACL Solr cache for all users 728 * @throws IOException If an I/O error occurs. 729 * @throws SolrServerException If a Solr error occurs. 730 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 731 */ 732 public void reloadAclCache() throws IOException, SolrServerException, RepositoryException 733 { 734 String[] workspaceNames = _repository.getWorkspaces(); 735 for (String workspaceName : workspaceNames) 736 { 737 reloadAclCache(workspaceName); 738 } 739 } 740 741 /** 742 * Reloads the ACL Solr cache for all users 743 * @param workspaceName The workspace name 744 * @throws IOException If an I/O error occurs. 745 * @throws SolrServerException If a Solr error occurs. 746 */ 747 public void reloadAclCache(String workspaceName) throws IOException, SolrServerException 748 { 749 reloadAclCache(workspaceName, false); 750 } 751 752 /** 753 * Reloads the ACL Solr cache for all users 754 * @param workspaceName The workspace name 755 * @param checkIfNecessary true to check if the reload is necessary for each segment (i.e. reload only the segments not already in cache) 756 * @throws IOException If an I/O error occurs. 757 * @throws SolrServerException If a Solr error occurs. 758 */ 759 public void reloadAclCache(String workspaceName, boolean checkIfNecessary) throws IOException, SolrServerException 760 { 761 getLogger().info("Reloading read ACL Solr cache for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 762 763 String collection = _solrClientProvider.getCollectionName(workspaceName); 764 SolrResponseBase responseBase = new ReloadAclCacheRequest(checkIfNecessary).process(_getAutoCommitSolrClient(workspaceName), collection); 765 NamedList<Object> responseObj = responseBase.getResponse(); 766 767 if ("ok".equals(responseObj.get("result"))) 768 { 769 getLogger().info("Read-ACL Solr cache reloaded for workspace '{}' (checkIfNecessary={})", workspaceName, checkIfNecessary); 770 } 771 else 772 { 773 Object error = responseObj.get("error"); 774 getLogger().error("The reloading of Read-ACL Solr Cache for workspace '{}' (checkIfNecessary={}) did not succeed as expected.\n Error code is the following: {}", workspaceName, checkIfNecessary, error); 775 } 776 } 777 778 /** 779 * Updates the ACL Solr cache for some {@link AmetysObject}s for all workspaces. 780 * @param objects the {@link AmetysObject}s to update. 781 * @throws IOException If an I/O error occurs. 782 * @throws SolrServerException If a Solr error occurs. 783 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 784 */ 785 public void updateAclCache(Iterable<? extends AmetysObject> objects) throws IOException, SolrServerException, RepositoryException 786 { 787 String[] workspaceNames = _repository.getWorkspaces(); 788 for (String workspaceName : workspaceNames) 789 { 790 updateAclCache(objects, workspaceName); 791 } 792 } 793 794 /** 795 * Updates the ACL Solr cache for some {@link AmetysObject}s. 796 * @param objects the {@link AmetysObject}s to update. 797 * @param workspaceName The workspace name 798 * @throws IOException If an I/O error occurs. 799 * @throws SolrServerException If a Solr error occurs. 800 * @throws RepositoryException If a repository exception occurs when retrieving workspaces. 801 */ 802 public void updateAclCache(Iterable<? extends AmetysObject> objects, String workspaceName) throws IOException, SolrServerException, RepositoryException 803 { 804 Map<String, Map<String, Object>> solrParams = new HashMap<>(); 805 806 for (AmetysObject object : objects) 807 { 808 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(object); 809 810 solrParams.put(object.getId(), Map.of("anonymous", allowedUsers.isAnonymousAllowed(), 811 "anyConnectedUser", allowedUsers.isAnyConnectedUserAllowed(), 812 "allowedUsers", allowedUsers.getAllowedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 813 "deniedUsers", allowedUsers.getDeniedUsers().stream().map(UserIdentity::userIdentityToString).collect(Collectors.toList()), 814 "allowedGroups", allowedUsers.getAllowedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()), 815 "deniedGroups", allowedUsers.getDeniedGroups().stream().map(GroupIdentity::groupIdentityToString).collect(Collectors.toList()))); 816 } 817 818 String collection = _solrClientProvider.getCollectionName(workspaceName); 819 SolrResponse response = new UpdateAclCacheRequest(solrParams).process(_getAutoCommitSolrClient(workspaceName), collection); 820 NamedList<Object> responseObj = response.getResponse(); 821 822 if ("ok".equals(responseObj.get("result"))) 823 { 824 if (getLogger().isInfoEnabled()) 825 { 826 getLogger().info("Read-ACL Solr cache updated for workspace '{}' and objects {}", workspaceName, solrParams.keySet()); 827 } 828 } 829 else 830 { 831 @SuppressWarnings("unchecked") 832 List<String> unHandledObjects = (List<String>) responseObj.get("unhandled-objects"); 833 getLogger().warn("The updating of Read-ACL Solr Cache for workspace '{}' did not succeed as expected.\n Following objects have net been updated: {}", workspaceName, unHandledObjects); 834 } 835 } 836 837 /** 838 * Index all the contents in a given workspace. 839 * @param workspaceName the workspace where to index 840 * @param indexAttachments to index content attachments 841 * @param solrClient The solr client to use 842 * @return The indexation result as a Map. 843 * @throws Exception if an error occurs while indexing. 844 */ 845 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 846 { 847 return indexAllContents(workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index all contents", getLogger())); 848 } 849 850 /** 851 * Index all the contents in a given workspace. 852 * @param workspaceName the workspace where to index 853 * @param indexAttachments to index content attachments 854 * @param solrClient The solr client to use 855 * @param progressionTracker The progression of the indexation 856 * @return The indexation result as a Map. 857 * @throws Exception if an error occurs while indexing. 858 */ 859 public Map<String, Object> indexAllContents(String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 860 { 861 Request request = ContextHelper.getRequest(_context); 862 863 // Retrieve the current workspace. 864 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 865 866 try 867 { 868 // Force the workspace. 869 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 870 871 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 872 873 long start = System.currentTimeMillis(); 874 875 // Delete all contents 876 unindexAllContents(workspaceName, indexAttachments, solrClient); 877 878 String query = ContentQueryHelper.getContentXPathQuery(null); 879 AmetysObjectIterable<Content> contents = _resolver.query(query); 880 881 IndexationResult result = doIndexContents(contents, workspaceName, indexAttachments, solrClient, progressionTracker); 882 883 long end = System.currentTimeMillis(); 884 885 if (!result.hasErrors()) 886 { 887 getLogger().info("{} contents indexed without error in {} milliseconds.", result.successCount(), end - start); 888 } 889 else 890 { 891 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount()); 892 } 893 894 Map<String, Object> results = new HashMap<>(); 895 results.put("successCount", result.successCount()); 896 if (result.hasErrors()) 897 { 898 results.put("errorCount", result.errorCount()); 899 } 900 901 return results; 902 } 903 catch (Exception e) 904 { 905 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 906 getLogger().error(error, e); 907 throw new IndexingException(error, e); 908 } 909 finally 910 { 911 // Restore context 912 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 913 } 914 } 915 /** 916 * Unindex all content documents. 917 * @param workspaceName The workspace name 918 * @param unindexAttachments also unindex content attachments 919 * @param solrClient The solr client to use 920 * @throws Exception if an error occurs while unindexing. 921 */ 922 protected void unindexAllContents(String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 923 { 924 String collection = _solrClientProvider.getCollectionName(workspaceName); 925 926 Query contents = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT); 927 Query query; 928 if (unindexAttachments) 929 { 930 Query contentResourceAttachments = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE); 931 Query contentResourceAttributes = new DocumentTypeQuery(SolrFieldNames.TYPE_CONTENT_ATTRIBUTE_RESOURCE); 932 query = new OrQuery(contentResourceAttachments, contentResourceAttributes, contents); 933 } 934 else 935 { 936 query = contents; 937 } 938 solrClient.deleteByQuery(collection, query.build()); 939 } 940 941 /** 942 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 943 * @param collectionId The id of collection 944 * @param indexAttachments to index content attachments 945 * @throws Exception if an error occurs while indexing. 946 */ 947 public void indexSubcontents(String collectionId, boolean indexAttachments) throws Exception 948 { 949 String[] workspaceNames = _repository.getWorkspaces(); 950 for (String workspaceName : workspaceNames) 951 { 952 indexSubcontents(collectionId, workspaceName, indexAttachments); 953 } 954 } 955 956 /** 957 * Index the child contents of a {@link AmetysObjectCollection} 958 * @param collectionId The id of collection 959 * @param workspaceName the workspace where to index 960 * @param indexAttachments to index content attachments 961 * @throws Exception if an error occurs while unindexing. 962 */ 963 public void indexSubcontents(String collectionId, String workspaceName, boolean indexAttachments) throws Exception 964 { 965 Request request = ContextHelper.getRequest(_context); 966 967 // Retrieve the current workspace. 968 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 969 970 try 971 { 972 // Force the workspace. 973 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 974 975 if (_resolver.hasAmetysObjectForId(collectionId)) 976 { 977 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 978 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 979 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 980 981 for (AmetysObject child : children) 982 { 983 if (child instanceof Content) 984 { 985 Content content = (Content) child; 986 987 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 988 } 989 } 990 } 991 } 992 finally 993 { 994 // Restore context 995 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 996 } 997 } 998 999 /** 1000 * Add or update a content into Solr index on all workspaces and commit 1001 * @param contentId The id of the content to index 1002 * @param indexAttachments to index content attachments 1003 * @throws Exception if an error occurs while indexing. 1004 */ 1005 public void indexContent(String contentId, boolean indexAttachments) throws Exception 1006 { 1007 String[] workspaceNames = _repository.getWorkspaces(); 1008 for (String workspaceName : workspaceNames) 1009 { 1010 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1011 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1012 } 1013 } 1014 1015 /** 1016 * Add or update a content into Solr index 1017 * @param contentId The id of the content to index 1018 * @param workspaceName the workspace where to index 1019 * @param indexAttachments to index content attachments 1020 * @throws Exception if an error occurs while indexing. 1021 */ 1022 public void indexContent(String contentId, String workspaceName, boolean indexAttachments) throws Exception 1023 { 1024 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1025 indexContent(contentId, workspaceName, indexAttachments, solrClient); 1026 } 1027 1028 /** 1029 * Add or update a content into Solr index 1030 * @param contentId The id of the content to index 1031 * @param workspaceName the workspace where to index 1032 * @param indexAttachments to index content attachments 1033 * @param solrClient The solr client to use 1034 * @throws Exception if an error occurs while indexing. 1035 */ 1036 public void indexContent(String contentId, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1037 { 1038 Request request = ContextHelper.getRequest(_context); 1039 1040 // Retrieve the current workspace. 1041 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1042 1043 try 1044 { 1045 // Force the workspace. 1046 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1047 1048 if (_resolver.hasAmetysObjectForId(contentId)) 1049 { 1050 Content content = _resolver.resolveById(contentId); 1051 _doIndexContent(content, workspaceName, indexAttachments, solrClient); 1052 } 1053 } 1054 finally 1055 { 1056 // Restore context 1057 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1058 } 1059 } 1060 1061 private void _doIndexContent(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws IndexingException 1062 { 1063 try 1064 { 1065 long time_0 = System.currentTimeMillis(); 1066 1067 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 1068 1069 deleteRepeaterDocs(content.getId(), workspaceName, solrClient); 1070 doIndexContent(content, workspaceName, solrClient); 1071 doIndexContentWorkflow(content, workspaceName, solrClient); 1072 if (indexAttachments) 1073 { 1074 indexContentAttachments(content.getRootAttachments(), content, solrClient); 1075 } 1076 1077 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 1078 } 1079 catch (Exception e) 1080 { 1081 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 1082 getLogger().error(error, e); 1083 throw new IndexingException(error, e); 1084 } 1085 } 1086 1087 /** 1088 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1089 * @param contents the collection of contents to index. 1090 * @return the indexation result. 1091 * @throws Exception if an error occurs while indexing. 1092 */ 1093 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 1094 { 1095 return indexContents(contents, ProgressionTrackerFactory.createContainerProgressionTracker("Index contents", getLogger())); 1096 } 1097 1098 1099 /** 1100 * Send a collection of contents for indexation in the solr server on all workspaces and commit 1101 * @param contents the collection of contents to index. 1102 * @param progressionTracker The progression of the indexation 1103 * @return the indexation result. 1104 * @throws Exception if an error occurs while indexing. 1105 */ 1106 public IndexationResult indexContents(Iterable<Content> contents, ContainerProgressionTracker progressionTracker) throws Exception 1107 { 1108 IndexationResult result = new IndexationResult(0, 0); 1109 1110 String[] workspaceNames = _repository.getWorkspaces(); 1111 1112 for (String workspaceName : workspaceNames) 1113 { 1114 progressionTracker.addSimpleStep(workspaceName, new I18nizableText("plugin.cms", "PLUGINS_CMS_SCHEDULER_GLOBAL_INDEXATION_CONTENT_WORKSPACE_STEP_LABEL", List.of(workspaceName))); 1115 } 1116 1117 for (String workspaceName : workspaceNames) 1118 { 1119 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1120 IndexationResult wResult = indexContents(contents, workspaceName, true, solrClient, progressionTracker.getStep(workspaceName)); 1121 1122 result = new IndexationResult(result.successCount() + wResult.successCount(), result.errorCount() + wResult.errorCount()); 1123 } 1124 1125 return result; 1126 } 1127 1128 /** 1129 * Send a collection of contents for indexation in the solr server. 1130 * @param contents the collection of contents to index. 1131 * @param workspaceName the workspace where to index 1132 * @param indexAttachments to index content attachments 1133 * @param solrClient The solr client to use 1134 * @return the indexation result. 1135 * @throws Exception if an error occurs while indexing. 1136 */ 1137 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient) throws Exception 1138 { 1139 return indexContents(contents, workspaceName, indexAttachments, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index contents for workspace " + workspaceName, getLogger())); 1140 } 1141 1142 /** 1143 * Send a collection of contents for indexation in the solr server. 1144 * @param contents the collection of contents to index. 1145 * @param workspaceName the workspace where to index 1146 * @param indexAttachments to index content attachments 1147 * @param solrClient The solr client to use 1148 * @param progressionTracker The progression of the indexation 1149 * @return the indexation result. 1150 * @throws Exception if an error occurs while indexing. 1151 */ 1152 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1153 { 1154 Request request = ContextHelper.getRequest(_context); 1155 1156 // Retrieve the current workspace. 1157 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1158 1159 try 1160 { 1161 // Force the workspace. 1162 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1163 1164 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 1165 1166 long start = System.currentTimeMillis(); 1167 1168 List<Content> contentsInWorkspace = StreamSupport.stream(contents.spliterator(), false) 1169 .map(Content::getId) 1170 .map(this::_resolveSilently) 1171 .filter(Objects::nonNull) 1172 .collect(Collectors.toList()); 1173 1174 IndexationResult result = doIndexContents(contentsInWorkspace, workspaceName, indexAttachments, solrClient, progressionTracker); 1175 1176 long end = System.currentTimeMillis(); 1177 1178 if (!result.hasErrors()) 1179 { 1180 getLogger().info("{} contents indexed without error in {} milliseconds.", result.successCount(), end - start); 1181 } 1182 else 1183 { 1184 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount()); 1185 } 1186 1187 return result; 1188 } 1189 catch (Exception e) 1190 { 1191 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 1192 getLogger().error(error, e); 1193 throw new IndexingException(error, e); 1194 } 1195 finally 1196 { 1197 // Restore context 1198 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1199 } 1200 } 1201 1202 private Content _resolveSilently(String contentId) 1203 { 1204 try 1205 { 1206 return _resolver.resolveById(contentId); 1207 } 1208 catch (UnknownAmetysObjectException e) 1209 { 1210 return null; 1211 } 1212 } 1213 1214 /** 1215 * Send some contents for indexation in the solr server. 1216 * @param contents the contents to index. 1217 * @param workspaceName The workspace name 1218 * @param indexAttachments to index content attachments 1219 * @param solrClient The solr client to use 1220 * @param progressionTracker The progression of the indexation 1221 * @return the indexation result. 1222 * @throws Exception if an error occurs committing the results. 1223 */ 1224 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1225 { 1226 int numberOfContents = IterableUtils.size(contents); 1227 progressionTracker.setSize(numberOfContents); 1228 1229 // Add callable for each content to index 1230 List<Future<Void>> tasks = new ArrayList<>(); 1231 for (Content content : contents) 1232 { 1233 tasks.add(_threadIndexerHelper.submitCallable(new ContentIndexerCallable(content, workspaceName, indexAttachments, solrClient, progressionTracker))); 1234 } 1235 1236 // Now that everything is submitted, we can iterate and wait for result 1237 return IndexationResult.fromTasks(tasks, getLogger()); 1238 } 1239 1240 /** 1241 * Update the value of a specific system property in a content document. 1242 * @param content The content to update. 1243 * @param propertyId The system property ID. 1244 * @param workspaceName The workspace name 1245 * @throws Exception if an error occurs while indexing. 1246 */ 1247 public void updateSystemProperty(Content content, String propertyId, String workspaceName) throws Exception 1248 { 1249 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1250 updateSystemProperty(content, propertyId, workspaceName, solrClient); 1251 } 1252 1253 /** 1254 * Update the value of a specific system property in a content document. 1255 * @param content The content to update. 1256 * @param propertyId The system property ID. 1257 * @param workspaceName The workspace name 1258 * @param solrClient The solr client to use 1259 * @throws Exception if an error occurs while indexing. 1260 */ 1261 public void updateSystemProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1262 { 1263 getLogger().debug("Updating the system property '{}' for content {} into Solr.", propertyId, content); 1264 1265 SolrInputDocument document = new SolrInputDocument(); 1266 boolean hasUpdate = _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 1267 1268 if (!hasUpdate) 1269 { 1270 getLogger().debug("Did not index '{}' system property for content {} in Solr because no update to apply.", propertyId, content); 1271 return; 1272 } 1273 1274 int status = _pushSolrDocument(document, workspaceName, solrClient); 1275 if (status != 0) 1276 { 1277 throw new IOException("Indexing of system property '" + propertyId + "': got status code '" + status + "'."); 1278 } 1279 1280 getLogger().debug("Succesfully indexed '{}' system property for content {} in Solr.", propertyId, content); 1281 } 1282 1283 /** 1284 * Update the value of a specific property in a content document. 1285 * @param content The content to update. 1286 * @param propertyId The system property ID. 1287 * @param workspaceName The workspace name 1288 * @throws Exception if an error occurs while indexing. 1289 */ 1290 public void updateProperty(Content content, String propertyId, String workspaceName) throws Exception 1291 { 1292 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1293 updateProperty(content, propertyId, workspaceName, solrClient); 1294 } 1295 1296 /** 1297 * Update the value of a specific property in a content document. 1298 * @param content The content to update. 1299 * @param propertyId The system property ID. 1300 * @param workspaceName The workspace name 1301 * @param solrClient The solr client to use 1302 * @throws Exception if an error occurs while indexing. 1303 */ 1304 public void updateProperty(Content content, String propertyId, String workspaceName, SolrClient solrClient) throws Exception 1305 { 1306 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 1307 1308 SolrInputDocument document = new SolrInputDocument(); 1309 boolean hasUpdate = _solrContentIndexer.indexPartialProperty(content, propertyId, document); 1310 1311 if (!hasUpdate) 1312 { 1313 getLogger().debug("Did not index '{}' property for content {} in Solr because no update to apply.", propertyId, content); 1314 return; 1315 } 1316 1317 int status = _pushSolrDocument(document, workspaceName, solrClient); 1318 if (status != 0) 1319 { 1320 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 1321 } 1322 1323 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 1324 } 1325 1326 private int _pushSolrDocument(SolrInputDocument document, String workspaceName, SolrClient solrClient) throws Exception 1327 { 1328 String collection = _solrClientProvider.getCollectionName(workspaceName); 1329 UpdateResponse solrResponse = solrClient.add(collection, document); 1330 return solrResponse.getStatus(); 1331 } 1332 1333 /** 1334 * Remove a content from Solr index for all workspaces and commit 1335 * @param contentId The id of content to unindex 1336 * @param unindexAttachments also unindex content attachments 1337 * @throws Exception if an error occurs while indexing. 1338 */ 1339 public void unindexContent(String contentId, boolean unindexAttachments) throws Exception 1340 { 1341 String[] workspaceNames = _repository.getWorkspaces(); 1342 for (String workspaceName : workspaceNames) 1343 { 1344 unindexContent(contentId, workspaceName, unindexAttachments); 1345 } 1346 } 1347 1348 /** 1349 * Remove a content from Solr index 1350 * @param contentId The id of content to unindex 1351 * @param workspaceName The workspace where to work in 1352 * @param unindexAttachments also unindex content attachments 1353 * @throws Exception if an error occurs while indexing. 1354 */ 1355 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments) throws Exception 1356 { 1357 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1358 unindexContent(contentId, workspaceName, unindexAttachments, solrClient); 1359 } 1360 1361 /** 1362 * Remove a content from Solr index 1363 * @param contentId The id of content to unindex 1364 * @param workspaceName The workspace where to work in 1365 * @param unindexAttachments also unindex content attachments 1366 * @param solrClient The solr client to use 1367 * @throws Exception if an error occurs while indexing. 1368 */ 1369 public void unindexContent(String contentId, String workspaceName, boolean unindexAttachments, SolrClient solrClient) throws Exception 1370 { 1371 Request request = ContextHelper.getRequest(_context); 1372 1373 // Retrieve the current workspace. 1374 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1375 1376 try 1377 { 1378 // Force the workspace. 1379 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1380 1381 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 1382 1383 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1384 doUnindexDocument(contentId, workspaceName, solrClient); 1385 if (unindexAttachments) 1386 { 1387 doUnindexContentAttachments(contentId, workspaceName, solrClient); 1388 } 1389 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1390 1391 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 1392 } 1393 catch (Exception e) 1394 { 1395 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 1396 getLogger().error(error, e); 1397 throw new IndexingException(error, e); 1398 } 1399 finally 1400 { 1401 // Restore workspace 1402 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1403 } 1404 } 1405 1406 /** 1407 * Remove a content from Solr index for all workspaces and commit 1408 * @param contentIds The id of content to unindex 1409 * @throws Exception if an error occurs while indexing. 1410 */ 1411 public void unindexContents(Collection<String> contentIds) throws Exception 1412 { 1413 String[] workspaceNames = _repository.getWorkspaces(); 1414 for (String workspaceName : workspaceNames) 1415 { 1416 unindexContents(contentIds, workspaceName); 1417 } 1418 } 1419 1420 /** 1421 * Remove a content from Solr index 1422 * @param contentIds The id of content to unindex 1423 * @param workspaceName The workspace where to work in 1424 * @throws Exception if an error occurs while indexing. 1425 */ 1426 public void unindexContents(Collection<String> contentIds, String workspaceName) throws Exception 1427 { 1428 Request request = ContextHelper.getRequest(_context); 1429 1430 // Retrieve the current workspace. 1431 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1432 1433 try 1434 { 1435 // Force the workspace. 1436 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1437 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1438 1439 getLogger().debug("Unindexing several contents from Solr."); 1440 1441 for (String contentId : contentIds) 1442 { 1443 deleteRepeaterDocs(contentId, workspaceName, solrClient); 1444 doUnindexDocument(contentId, workspaceName, solrClient); 1445 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, solrClient); 1446 } 1447 1448 getLogger().debug("Succesfully unindexed content from Solr."); 1449 } 1450 catch (Exception e) 1451 { 1452 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1453 getLogger().error(error, e); 1454 throw new IndexingException(error, e); 1455 } 1456 finally 1457 { 1458 // Restore workspace 1459 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1460 } 1461 } 1462 1463 /** 1464 * Add or update a content into Solr index 1465 * @param content The content to index 1466 * @param workspaceName The workspace where to index 1467 * @param solrClient The solr client to use 1468 * @throws Exception if an error occurs while indexing. 1469 */ 1470 protected void doIndexContent(Content content, String workspaceName, SolrClient solrClient) throws Exception 1471 { 1472 long time_0 = System.currentTimeMillis(); 1473 1474 SolrInputDocument document = new SolrInputDocument(); 1475 List<SolrInputDocument> additionalDocuments = _solrContentIndexer.indexContent(content, document); 1476 1477 long time_1 = System.currentTimeMillis(); 1478 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1479 1480 indexAclInitValues(content, document); 1481 1482 long time_2 = System.currentTimeMillis(); 1483 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1484 1485 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1486 documents.add(document); 1487 1488 UpdateResponse solrResponse = solrClient.add(_solrClientProvider.getCollectionName(workspaceName), documents); 1489 int status = solrResponse.getStatus(); 1490 1491 long time_3 = System.currentTimeMillis(); 1492 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1493 1494 if (status != 0) 1495 { 1496 throw new IOException("Content indexation: got status code '" + status + "'."); 1497 } 1498 } 1499 1500 /** 1501 * Indexes read-ACl initial values for object 1502 * @param ametysObject The object 1503 * @param document The Solr document 1504 */ 1505 public void indexAclInitValues(AmetysObject ametysObject, SolrInputDocument document) 1506 { 1507 // Indexation of AmetysObject property 1508 document.addField(SolrFieldNames.IS_AMETYS_OBJECT, true); 1509 1510 // Indexation of initValues for AllowedUsers 1511 AllowedUsers allowedUsers = _readAccessHelper.allowedUsers(ametysObject); 1512 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANONYMOUS, allowedUsers.isAnonymousAllowed()); 1513 document.addField(SolrFieldNames.ACL_INIT_VALUE_ANYCONNECTED, allowedUsers.isAnyConnectedUserAllowed()); 1514 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_USERS, allowedUsers.getAllowedUsers(), UserIdentity::userIdentityToString); 1515 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_USERS, allowedUsers.getDeniedUsers(), UserIdentity::userIdentityToString); 1516 _addField(document, SolrFieldNames.ACL_INIT_VALUE_ALLOWED_GROUPS, allowedUsers.getAllowedGroups(), GroupIdentity::groupIdentityToString); 1517 _addField(document, SolrFieldNames.ACL_INIT_VALUE_DENIED_GROUPS, allowedUsers.getDeniedGroups(), GroupIdentity::groupIdentityToString); 1518 } 1519 1520 private <T> void _addField(SolrInputDocument document, String fieldName, Set<T> values, Function<T, String> stringifier) 1521 { 1522 if (values == null) 1523 { 1524 return; 1525 } 1526 1527 for (T value : values) 1528 { 1529 document.addField(fieldName, stringifier.apply(value)); 1530 } 1531 } 1532 1533 /** 1534 * Index the whole workflow of a content. 1535 * @param content The content. 1536 * @param workspaceName The workspace name 1537 * @param solrClient The solr client to use 1538 * @throws Exception if an error occurs while indexing. 1539 */ 1540 protected void doIndexContentWorkflow(Content content, String workspaceName, SolrClient solrClient) throws Exception 1541 { 1542 if (content instanceof WorkflowAwareContent) 1543 { 1544 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, solrClient); 1545 } 1546 } 1547 1548 /** 1549 * Index content attachments as new entries in the idnex 1550 * @param collection the collection of attachments 1551 * @param content the content whose attachments will be indexed 1552 * @throws Exception if something goes wrong when indexing the attachments of the content 1553 */ 1554 public void indexContentAttachments(ResourceCollection collection, Content content) throws Exception 1555 { 1556 Request request = ContextHelper.getRequest(_context); 1557 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1558 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1559 indexContentAttachments(collection, content, solrClient); 1560 } 1561 1562 /** 1563 * Index content attachments as new entries in the idnex 1564 * @param collection the collection of attachments 1565 * @param content the content whose attachments will be indexed 1566 * @param solrClient The solr client to use 1567 * @throws Exception if something goes wrong when indexing the attachments of the content 1568 */ 1569 public void indexContentAttachments(ResourceCollection collection, Content content, SolrClient solrClient) throws Exception 1570 { 1571 if (collection == null) 1572 { 1573 return; 1574 } 1575 1576 try (AmetysObjectIterable<AmetysObject> children = collection.getChildren()) 1577 { 1578 for (AmetysObject object : children) 1579 { 1580 if (object instanceof ResourceCollection) 1581 { 1582 indexContentAttachments((ResourceCollection) object, content, solrClient); 1583 } 1584 else if (object instanceof Resource) 1585 { 1586 Resource resource = (Resource) object; 1587 indexContentAttachment(resource, content, solrClient); 1588 } 1589 } 1590 } 1591 } 1592 1593 /** 1594 * Index a content attachment 1595 * @param resource the content attachment as a {@link Resource} 1596 * @param content the content whose attachment is going to be indexed 1597 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1598 */ 1599 public void indexContentAttachment(Resource resource, Content content) throws Exception 1600 { 1601 Request request = ContextHelper.getRequest(_context); 1602 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1603 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1604 indexContentAttachment(resource, content, solrClient); 1605 } 1606 1607 /** 1608 * Index a content attachment 1609 * @param resource the content attachment as a {@link Resource} 1610 * @param content the content whose attachment is going to be indexed 1611 * @param solrClient The solr client to use 1612 * @throws Exception if something goes wrong when processing the indexation of the content attachment 1613 */ 1614 public void indexContentAttachment(Resource resource, Content content, SolrClient solrClient) throws Exception 1615 { 1616 SolrInputDocument document = new SolrInputDocument(); 1617 1618 // Prepare resource doc 1619 _indexContentAttachment(resource, document, content); 1620 1621 // Indexation of the document 1622 _indexResourceDocument(resource, document, solrClient); 1623 } 1624 1625 private void _indexContentAttachment(Resource resource, SolrInputDocument document, Content content) throws Exception 1626 { 1627 String language = content.getLanguage(); 1628 1629 _solrResourceIndexer.indexResource(resource, document, SolrFieldNames.TYPE_CONTENT_ATTACHMENT_RESOURCE, language); 1630 1631 // Need the id of the content for unindexing attachment during the unindexing of the content 1632 document.addField(SolrFieldNames.ATTACHMENT_CONTENT_ID, content.getId()); 1633 } 1634 1635 /** 1636 * Index a populated solr input document of type Resource. 1637 * @param resource the resource from which the input document is created 1638 * @param document the input document 1639 * @param solrClient The solr client to use 1640 * @throws SolrServerException if there is an error on the server 1641 * @throws IOException if there is a communication error with the server 1642 */ 1643 protected void _indexResourceDocument(Resource resource, SolrInputDocument document, SolrClient solrClient) throws SolrServerException, IOException 1644 { 1645 String resourceId = resource.getId(); 1646 1647 // Retrieve appropriate collection name 1648 Request request = ContextHelper.getRequest(_context); 1649 String workspaceName = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1650 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 1651 1652 // Add document 1653 UpdateResponse solrResponse = solrClient.add(collectionName, document); 1654 int status = solrResponse.getStatus(); 1655 1656 if (status != 0) 1657 { 1658 throw new IOException("Ametys resource indexing - Expecting status code of '0' in the Solr response but got : '" + status + "'. Resource id : " + resourceId); 1659 } 1660 1661 getLogger().debug("Successful resource indexing. Resource identifier : {}", resourceId); 1662 } 1663 1664 /** 1665 * Delete repeater documents of a specified content. 1666 * @param workspaceName The workspace name 1667 * @param contentId the content ID. 1668 * @param solrClient The solr client to use 1669 * @throws Exception if an error occurs while indexing. 1670 */ 1671 protected void deleteRepeaterDocs(String contentId, String workspaceName, SolrClient solrClient) throws Exception 1672 { 1673 long time_0 = System.currentTimeMillis(); 1674 1675 // _documentType:repeater AND id:content\://xxx/* 1676 StringBuilder query = new StringBuilder(); 1677 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1678 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1679 1680 solrClient.deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1681 1682 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1683 } 1684 1685 /** 1686 * Index all the resources in a given workspace. 1687 * @param workspaceName The workspace where to index 1688 * @param solrClient The solr client to use 1689 * @param progressionTracker The progression of the indexation 1690 * @return The indexation result as a Map. 1691 * @throws Exception if an error occurs while indexing. 1692 */ 1693 public Map<String, Object> indexAllResources(String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1694 { 1695 Request request = ContextHelper.getRequest(_context); 1696 1697 // Retrieve the current workspace. 1698 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1699 1700 try 1701 { 1702 // Force the workspace. 1703 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1704 1705 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1706 1707 long start = System.currentTimeMillis(); 1708 1709 // Delete all resources 1710 _unindexAllResources(workspaceName, solrClient); 1711 1712 Map<String, Object> results = new HashMap<>(); 1713 1714 try 1715 { 1716 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1717 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1718 1719 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, solrClient, progressionTracker); 1720 1721 long end = System.currentTimeMillis(); 1722 1723 if (!result.hasErrors()) 1724 { 1725 getLogger().info("{} resources indexed without error in {} milliseconds.", result.successCount(), end - start); 1726 } 1727 else 1728 { 1729 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount()); 1730 } 1731 1732 results.put("successCount", result.successCount()); 1733 if (result.hasErrors()) 1734 { 1735 results.put("errorCount", result.errorCount()); 1736 } 1737 } 1738 catch (UnknownAmetysObjectException e) 1739 { 1740 getLogger().info("There is no root for resources in current workspace."); 1741 progressionTracker.setSize(0); 1742 } 1743 1744 return results; 1745 } 1746 catch (Exception e) 1747 { 1748 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1749 getLogger().error(error, e); 1750 throw new IndexingException(error, e); 1751 } 1752 finally 1753 { 1754 // Restore context 1755 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1756 } 1757 } 1758 1759 /** 1760 * Send a collection of contents for indexation in the solr server. 1761 * @param resources the collection of contents to index. 1762 * @param documentType The document type of the resource 1763 * @param workspaceName The workspace where to index 1764 * @return the indexation result. 1765 * @throws Exception if an error occurs while indexing. 1766 */ 1767 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName) throws Exception 1768 { 1769 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1770 return indexResources(resources, documentType, null, workspaceName, solrClient); 1771 } 1772 1773 /** 1774 * Send a collection of contents for indexation in the solr server. 1775 * @param resources the collection of contents to index. 1776 * @param documentType The document type of the resource 1777 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1778 * @param workspaceName The workspace where to index 1779 * @param solrClient The solr client to use 1780 * @return the indexation result. 1781 * @throws Exception if an error occurs while indexing. 1782 */ 1783 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient) throws Exception 1784 { 1785 return indexResources(resources, documentType, resourceRoot, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resources", getLogger())); 1786 } 1787 1788 /** 1789 * Send a collection of contents for indexation in the solr server. 1790 * @param resources the collection of contents to index. 1791 * @param documentType The document type of the resource 1792 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1793 * @param workspaceName The workspace where to index 1794 * @param solrClient The solr client to use 1795 * @param progressionTracker The progression of the indexation 1796 * @return the indexation result. 1797 * @throws Exception if an error occurs while indexing. 1798 */ 1799 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1800 { 1801 Request request = ContextHelper.getRequest(_context); 1802 1803 // Retrieve the current workspace. 1804 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1805 1806 try 1807 { 1808 // Force the workspace. 1809 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1810 1811 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1812 1813 long start = System.currentTimeMillis(); 1814 1815 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, solrClient, progressionTracker); 1816 1817 long end = System.currentTimeMillis(); 1818 1819 if (!result.hasErrors()) 1820 { 1821 getLogger().info("{} resources indexed without error in {} milliseconds.", result.successCount(), end - start); 1822 } 1823 else 1824 { 1825 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.errorCount()); 1826 } 1827 1828 return result; 1829 } 1830 catch (Exception e) 1831 { 1832 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1833 getLogger().error(error, e); 1834 throw new IndexingException(error, e); 1835 } 1836 finally 1837 { 1838 // Restore context 1839 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1840 } 1841 } 1842 1843 /** 1844 * Send some resources for indexation in the solr server. 1845 * @param resources the resources to index. 1846 * @param documentType The document type of the resource 1847 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1848 * @param workspaceName The workspace where to index 1849 * @param solrClient The solr client to use 1850 * @param progressionTracker The progression of the indexation 1851 * @return the indexation result. 1852 * @throws Exception if an error occurs committing the results. 1853 */ 1854 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, SolrClient solrClient, SimpleProgressionTracker progressionTracker) throws Exception 1855 { 1856 try 1857 { 1858 // Add callable for each content to index 1859 List<Future<Void>> tasks = new ArrayList<>(); 1860 for (AmetysObject resource : resources) 1861 { 1862 tasks.addAll(_asyncIndexExplorerItem(resource, documentType, resourceRoot, solrClient)); 1863 } 1864 1865 // Now that everything is submitted, we can iterate and wait for result 1866 return IndexationResult.fromTasks(tasks, getLogger()); 1867 } 1868 finally 1869 { 1870 progressionTracker.increment(); 1871 } 1872 } 1873 1874 /** 1875 * Add or update a resource into Solr index 1876 * @param resource The resource to index 1877 * @param documentType The document type of the resource 1878 * @param workspaceName The workspace where to index 1879 * @throws Exception if an error occurs while indexing. 1880 */ 1881 public void indexResource(Resource resource, String documentType, String workspaceName) throws Exception 1882 { 1883 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1884 doIndexResources(Collections.singleton(resource), documentType, null, workspaceName, solrClient, ProgressionTrackerFactory.createSimpleProgressionTracker("Index resource", getLogger())); 1885 } 1886 1887 private void _unindexAllResources(String workspaceName, SolrClient solrClient) throws Exception 1888 { 1889 getLogger().debug("Unindexing all resources from Solr."); 1890 1891 String collection = _solrClientProvider.getCollectionName(workspaceName); 1892 solrClient.deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1893 1894 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1895 } 1896 1897 /** 1898 * Delete all resource documents at a given path for all workspaces and commit 1899 * @param rootId The resource root ID, must not be null. 1900 * @param path The resource path relative to the given root, must start with a slash. 1901 * @throws Exception If an error occurs while unindexing. 1902 */ 1903 public void unindexResourcesByPath(String rootId, String path) throws Exception 1904 { 1905 String[] workspaceNames = _repository.getWorkspaces(); 1906 for (String workspaceName : workspaceNames) 1907 { 1908 unindexResourcesByPath(rootId, path, workspaceName); 1909 } 1910 } 1911 1912 /** 1913 * Delete all resource documents at a given path. 1914 * @param rootId The resource root ID, must not be null. 1915 * @param path The resource path relative to the given root, must start with a slash. 1916 * @param workspaceName The workspace where to work in 1917 * @throws Exception If an error occurs while unindexing. 1918 */ 1919 public void unindexResourcesByPath(String rootId, String path, String workspaceName) throws Exception 1920 { 1921 Request request = ContextHelper.getRequest(_context); 1922 1923 // Retrieve the current workspace. 1924 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1925 1926 try 1927 { 1928 // Force the workspace. 1929 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1930 1931 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1932 1933 Query query = new ResourceLocationQuery(rootId, path); 1934 1935 String collection = _solrClientProvider.getCollectionName(workspaceName); 1936 _getAutoCommitSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1937 1938 getLogger().debug("Succesfully deleted resource document from Solr."); 1939 } 1940 catch (Exception e) 1941 { 1942 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1943 getLogger().error(error, e); 1944 throw new IndexingException(error, e); 1945 } 1946 finally 1947 { 1948 // Restore workspace 1949 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1950 } 1951 } 1952 1953 /** 1954 * Remove a resource from Solr index for all workspaces and commit 1955 * @param resourceId The id of resource to unindex 1956 * @throws Exception if an error occurs while indexing. 1957 */ 1958 public void unindexResource(String resourceId) throws Exception 1959 { 1960 String[] workspaceNames = _repository.getWorkspaces(); 1961 for (String workspaceName : workspaceNames) 1962 { 1963 unindexResource(resourceId, workspaceName); 1964 } 1965 } 1966 1967 /** 1968 * Remove a resource from the Solr index. 1969 * @param resourceId The id of resource to unindex 1970 * @param workspaceName The workspace where to work in 1971 * @throws Exception if an error occurs while unindexing. 1972 */ 1973 public void unindexResource(String resourceId, String workspaceName) throws Exception 1974 { 1975 Request request = ContextHelper.getRequest(_context); 1976 1977 // Retrieve the current workspace. 1978 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1979 1980 try 1981 { 1982 // Force the workspace. 1983 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1984 SolrClient solrClient = _getAutoCommitSolrClient(workspaceName); 1985 1986 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 1987 1988 doUnindexDocument(resourceId, workspaceName, solrClient); 1989 1990 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 1991 } 1992 catch (Exception e) 1993 { 1994 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 1995 getLogger().error(error, e); 1996 throw new IndexingException(error, e); 1997 } 1998 finally 1999 { 2000 // Restore workspace 2001 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 2002 } 2003 } 2004 2005 private List<Future<Void>> _asyncIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) throws Exception 2006 { 2007 List<Future<Void>> tasks = new ArrayList<>(); 2008 2009 if (node instanceof ResourceCollection resourceCollection) 2010 { 2011 try (AmetysObjectIterable<AmetysObject> children = resourceCollection.getChildren()) 2012 { 2013 for (AmetysObject child : children) 2014 { 2015 tasks.addAll(_asyncIndexExplorerItem(child, documentType, resourceRoot, solrClient)); 2016 } 2017 } 2018 } 2019 else if (node instanceof Resource resource) 2020 { 2021 tasks.add(_threadIndexerHelper.submitCallable(new ResourceIndexerCallable(resource, _workspaceSelector.getWorkspace(), documentType, resourceRoot, solrClient))); 2022 } 2023 2024 return tasks; 2025 } 2026 2027 /** 2028 * Process a Solr commit operation in all workspaces. 2029 * <br>Use this only after a long operation with updates sent via {@link NoAutoCommitUpdateClient} 2030 * @throws SolrServerException if there is an error on the server 2031 * @throws IOException if there is a communication error with the server 2032 */ 2033 public void commit() throws SolrServerException, IOException 2034 { 2035 String[] workspaceNames; 2036 try 2037 { 2038 workspaceNames = _repository.getWorkspaces(); 2039 for (String workspaceName : workspaceNames) 2040 { 2041 SolrClient solrClient = _getNoAutoCommitSolrClient(workspaceName); 2042 commit(workspaceName, solrClient); 2043 } 2044 } 2045 catch (RepositoryException e) 2046 { 2047 throw new RuntimeException("An exception occured while retrieving JCR workspaces. Cannot commited to Solr.", e); 2048 } 2049 } 2050 2051 /** 2052 * Process a Solr commit operation in given workspace. 2053 * @param workspaceName The workspace's name 2054 * @param solrClient The solr client to use 2055 * @throws SolrServerException if there is an error on the server 2056 * @throws IOException if there is a communication error with the server 2057 */ 2058 public void commit(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2059 { 2060 long time_0 = System.currentTimeMillis(); 2061 2062 // Commit 2063 UpdateResponse solrResponse = solrClient.commit(_solrClientProvider.getCollectionName(workspaceName)); 2064 int status = solrResponse.getStatus(); 2065 2066 if (status != 0) 2067 { 2068 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2069 } 2070 2071 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 2072 } 2073 2074 /** 2075 * Launch a solr index optimization. 2076 * @param workspaceName The workspace's name 2077 * @param solrClient The solr client to use 2078 * @throws SolrServerException if there is an error on the server 2079 * @throws IOException if there is a communication error with the server 2080 */ 2081 public void optimize(String workspaceName, SolrClient solrClient) throws SolrServerException, IOException 2082 { 2083 UpdateResponse solrResponse = solrClient.optimize(_solrClientProvider.getCollectionName(workspaceName)); 2084 int status = solrResponse.getStatus(); 2085 2086 if (status != 0) 2087 { 2088 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 2089 } 2090 2091 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 2092 } 2093 2094 /** 2095 * Delete all documents from the solr index. 2096 * @param workspaceName The workspace name 2097 * @param solrClient The solr client to use 2098 * @throws Exception if an error occurs while unindexing. 2099 */ 2100 public void unindexAllDocuments(String workspaceName, SolrClient solrClient) throws Exception 2101 { 2102 getLogger().debug("Deleting all documents from Solr."); 2103 2104 String collection = _solrClientProvider.getCollectionName(workspaceName); 2105 solrClient.deleteByQuery(collection, "*:*"); 2106 2107 getLogger().debug("Successfully deleted all documents from Solr."); 2108 } 2109 2110 /** 2111 * Delete a document from the Solr server. 2112 * @param id The id of the document to delete from Solr 2113 * @param workspaceName The workspace name 2114 * @param solrClient The solr client to use 2115 * @throws Exception if an error occurs while indexing. 2116 */ 2117 protected void doUnindexDocument(String id, String workspaceName, SolrClient solrClient) throws Exception 2118 { 2119 UpdateResponse solrResponse = solrClient.deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 2120 int status = solrResponse.getStatus(); 2121 2122 if (status != 0) 2123 { 2124 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 2125 } 2126 } 2127 2128 /** 2129 * Delete content attachments documents of a given content from the Solr server. 2130 * @param contentId The id of the content 2131 * @param workspaceName The workspace name 2132 * @param solrClient The solr client to use 2133 * @throws Exception if an error occurs while indexing. 2134 */ 2135 protected void doUnindexContentAttachments(String contentId, String workspaceName, SolrClient solrClient) throws Exception 2136 { 2137 String collectionName = _solrClientProvider.getCollectionName(workspaceName); 2138 2139 Query query = new ContentAttachmentQuery(contentId); 2140 UpdateResponse solrResponse = solrClient.deleteByQuery(collectionName, query.build()); 2141 int status = solrResponse.getStatus(); 2142 2143 if (status != 0) 2144 { 2145 throw new IOException("Deletion of content attachments of content " + contentId + ": got status code '" + status + "'."); 2146 } 2147 } 2148 2149// /** 2150// * Get the collection to use. 2151// * @return The name of the collection to index into. 2152// */ 2153// protected String getCollection() 2154// { 2155// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 2156// } 2157 2158 private static final class SchemaRequestComparator implements Comparator<SchemaRequest.Update> 2159 { 2160 public int compare(SchemaRequest.Update u1, SchemaRequest.Update u2) 2161 { 2162 return Integer.compare(_getOrder(u1), _getOrder(u2)); 2163 } 2164 2165 private int _getOrder(SchemaRequest.Update update) 2166 { 2167 // Field types are needed first to define fields 2168 if (update instanceof SchemaRequest.AddFieldType) 2169 { 2170 return 1; 2171 } 2172 2173 // Fields or dynamic fields can be defined at the same time 2174 if (update instanceof SchemaRequest.AddField || update instanceof SchemaRequest.AddDynamicField) 2175 { 2176 return 2; 2177 } 2178 2179 // Copy fields can be based on fields or dynamic fields 2180 if (update instanceof SchemaRequest.AddCopyField) 2181 { 2182 return 3; 2183 } 2184 2185 return 0; 2186 } 2187 } 2188 2189 private class ContentIndexerCallable extends AbstractIndexerCallable<Content> 2190 { 2191 private boolean _indexAttachments; 2192 private SimpleProgressionTracker _tracker; 2193 2194 @SuppressWarnings("synthetic-access") 2195 public ContentIndexerCallable(Content content, String workspaceName, boolean indexAttachments, SolrClient solrClient, SimpleProgressionTracker progressionTracker) 2196 { 2197 super(content, workspaceName, solrClient, _manager, _cocoonContext, _resolver, getLogger()); 2198 this._indexAttachments = indexAttachments; 2199 this._tracker = progressionTracker; 2200 } 2201 2202 @Override 2203 protected void process(Content content) throws Exception 2204 { 2205 try 2206 { 2207 doIndexContent(content, _workspaceName, _solrClient); 2208 doIndexContentWorkflow(content, _workspaceName, _solrClient); 2209 if (_indexAttachments) 2210 { 2211 indexContentAttachments(content.getRootAttachments(), content, _solrClient); 2212 } 2213 } 2214 finally 2215 { 2216 _tracker.increment(); 2217 } 2218 } 2219 2220 @Override 2221 protected String getObjectLabel() 2222 { 2223 return "content"; 2224 } 2225 } 2226 2227 private class ResourceIndexerCallable extends AbstractIndexerCallable<Resource> 2228 { 2229 private String _documentType; 2230 private TraversableAmetysObject _resourceRoot; 2231 2232 @SuppressWarnings("synthetic-access") 2233 public ResourceIndexerCallable(Resource resource, String workspaceName, String documentType, TraversableAmetysObject resourceRoot, SolrClient solrClient) 2234 { 2235 super(resource, workspaceName, solrClient, _manager, _cocoonContext, _resolver, getLogger()); 2236 this._documentType = documentType; 2237 this._resourceRoot = resourceRoot; 2238 } 2239 2240 @Override 2241 protected void process(Resource resource) throws Exception 2242 { 2243 SolrInputDocument document = new SolrInputDocument(); 2244 2245 _solrResourceIndexer.indexResource(resource, document, _documentType, _resourceRoot); 2246 2247 UpdateResponse solrResponse = _solrClient.add(_solrClientProvider.getCollectionName(_workspaceName), document); 2248 int status = solrResponse.getStatus(); 2249 2250 if (status != 0) 2251 { 2252 throw new IOException("Resource indexation: got status code '" + status + "'."); 2253 } 2254 } 2255 2256 @Override 2257 protected String getObjectLabel() 2258 { 2259 return "resource"; 2260 } 2261 } 2262}