001/* 002 * Copyright 2015 Anyware Services 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.ametys.cms.content.indexing.solr; 017 018import java.io.IOException; 019import java.text.SimpleDateFormat; 020import java.util.ArrayList; 021import java.util.Arrays; 022import java.util.Collection; 023import java.util.HashMap; 024import java.util.HashSet; 025import java.util.List; 026import java.util.Map; 027import java.util.Set; 028 029import org.apache.avalon.framework.activity.Initializable; 030import org.apache.avalon.framework.component.Component; 031import org.apache.avalon.framework.context.Context; 032import org.apache.avalon.framework.context.ContextException; 033import org.apache.avalon.framework.context.Contextualizable; 034import org.apache.avalon.framework.service.ServiceException; 035import org.apache.avalon.framework.service.ServiceManager; 036import org.apache.avalon.framework.service.Serviceable; 037import org.apache.cocoon.components.ContextHelper; 038import org.apache.cocoon.environment.Request; 039import org.apache.solr.client.solrj.SolrClient; 040import org.apache.solr.client.solrj.SolrQuery; 041import org.apache.solr.client.solrj.SolrServerException; 042import org.apache.solr.client.solrj.request.CoreAdminRequest; 043import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; 044import org.apache.solr.client.solrj.request.schema.SchemaRequest; 045import org.apache.solr.client.solrj.request.schema.SchemaRequest.Update; 046import org.apache.solr.client.solrj.response.CoreAdminResponse; 047import org.apache.solr.client.solrj.response.QueryResponse; 048import org.apache.solr.client.solrj.response.UpdateResponse; 049import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; 050import org.apache.solr.client.solrj.response.schema.SchemaResponse; 051import org.apache.solr.client.solrj.util.ClientUtils; 052import org.apache.solr.common.SolrInputDocument; 053import org.apache.solr.common.util.NamedList; 054 055import org.ametys.cms.indexing.IndexingException; 056import org.ametys.cms.repository.Content; 057import org.ametys.cms.repository.ContentQueryHelper; 058import org.ametys.cms.repository.RequestAttributeWorkspaceSelector; 059import org.ametys.cms.repository.WorkflowAwareContent; 060import org.ametys.cms.search.query.Query; 061import org.ametys.cms.search.query.ResourceLocationQuery; 062import org.ametys.cms.search.solr.SolrClientProvider; 063import org.ametys.cms.search.solr.schema.CopyFieldDefinition; 064import org.ametys.cms.search.solr.schema.FieldDefinition; 065import org.ametys.cms.search.solr.schema.SchemaDefinition; 066import org.ametys.cms.search.solr.schema.SchemaDefinitionProvider; 067import org.ametys.cms.search.solr.schema.SchemaDefinitionProviderExtensionPoint; 068import org.ametys.cms.search.solr.schema.SchemaFields; 069import org.ametys.cms.search.solr.schema.SchemaHelper; 070import org.ametys.plugins.explorer.resources.Resource; 071import org.ametys.plugins.explorer.resources.ResourceCollection; 072import org.ametys.plugins.repository.AmetysObject; 073import org.ametys.plugins.repository.AmetysObjectIterable; 074import org.ametys.plugins.repository.AmetysObjectResolver; 075import org.ametys.plugins.repository.RepositoryConstants; 076import org.ametys.plugins.repository.TraversableAmetysObject; 077import org.ametys.plugins.repository.UnknownAmetysObjectException; 078import org.ametys.plugins.repository.collection.AmetysObjectCollection; 079import org.ametys.plugins.repository.provider.WorkspaceSelector; 080import org.ametys.runtime.config.Config; 081import org.ametys.runtime.plugin.component.AbstractLogEnabled; 082 083/** 084 * Solr indexer. 085 */ 086public class SolrIndexer extends AbstractLogEnabled implements Component, Serviceable, Initializable, Contextualizable 087{ 088 /** The component role. */ 089 public static final String ROLE = SolrIndexer.class.getName(); 090 091 private static final ThreadLocal<SimpleDateFormat> __DATE_FORMAT = new ThreadLocal<>(); 092 093 private static final String _CONFIGSET_NAME = "ametys-standard"; 094 095 private static final List<String> _READ_ONLY_FIELDS = Arrays.asList("id", "_root_", "_version_", "text"); 096 private static final List<String> _READ_ONLY_FIELDTYPES = Arrays.asList("string", "long", "text_general"); 097 098 /** The ametys object resolver. */ 099 protected AmetysObjectResolver _resolver; 100 /** The schema definition provider extension point. */ 101 protected SchemaDefinitionProviderExtensionPoint _schemaDefProviderEP; 102 /** The schema helper. */ 103 protected SchemaHelper _schemaHelper; 104 /** Solr Ametys contents indexer */ 105 protected SolrContentIndexer _solrContentIndexer; 106 /** Solr workflow indexer. */ 107 protected SolrWorkflowIndexer _solrWorkflowIndexer; 108 /** Solr right indexer. */ 109 protected SolrContentRightIndexer _solrContentRightIndexer; 110 /** Solr resource indexer. */ 111 protected SolrResourceIndexer _solrResourceIndexer; 112 113 /** The Solr client provider */ 114 protected SolrClientProvider _solrClientProvider; 115 116 /** The solr core prefix. */ 117 protected String _solrCorePrefix; 118 119 /** The workspace selector. */ 120 protected WorkspaceSelector _workspaceSelector; 121 122 /** The avalon context */ 123 protected Context _context; 124 125 /** 126 * Returns the formatter for indexing dates. This is used for adding a dates as formatted strings (and not with date object directly) to prevent indexing of the wrong value because of time zone 127 * @return The date format for indexing dates 128 */ 129 public static SimpleDateFormat dateFormat() 130 { 131 if (__DATE_FORMAT.get() == null) 132 { 133 __DATE_FORMAT.set(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); 134 } 135 return __DATE_FORMAT.get(); 136 } 137 138 @Override 139 public void service(ServiceManager serviceManager) throws ServiceException 140 { 141 _resolver = (AmetysObjectResolver) serviceManager.lookup(AmetysObjectResolver.ROLE); 142 _schemaDefProviderEP = (SchemaDefinitionProviderExtensionPoint) serviceManager.lookup(SchemaDefinitionProviderExtensionPoint.ROLE); 143 _schemaHelper = (SchemaHelper) serviceManager.lookup(SchemaHelper.ROLE); 144 _solrContentIndexer = (SolrContentIndexer) serviceManager.lookup(SolrContentIndexer.ROLE); 145 _solrWorkflowIndexer = (SolrWorkflowIndexer) serviceManager.lookup(SolrWorkflowIndexer.ROLE); 146 _solrContentRightIndexer = (SolrContentRightIndexer) serviceManager.lookup(SolrContentRightIndexer.ROLE); 147 _solrResourceIndexer = (SolrResourceIndexer) serviceManager.lookup(SolrResourceIndexer.ROLE); 148 _solrClientProvider = (SolrClientProvider) serviceManager.lookup(SolrClientProvider.ROLE); 149 _workspaceSelector = (WorkspaceSelector) serviceManager.lookup(WorkspaceSelector.ROLE); 150 } 151 152 @Override 153 public void initialize() throws Exception 154 { 155 _solrCorePrefix = Config.getInstance().getValueAsString("cms.solr.core.prefix"); 156 } 157 158 @Override 159 public void contextualize(Context context) throws ContextException 160 { 161 _context = context; 162 } 163 164 /** 165 * Gets the Solr client 166 * @param workspaceName The name of the workspace 167 * @return the Solr client 168 */ 169 protected SolrClient _getSolrClient(String workspaceName) 170 { 171 return _solrClientProvider.getUpdateClient(workspaceName); 172 } 173 174 // for admin operations 175 private SolrClient _defaultSolrClient() 176 { 177 return _solrClientProvider.getUpdateClient(RepositoryConstants.DEFAULT_WORKSPACE); 178 } 179 180 /** 181 * Get the names of the Solr cores. 182 * @return The names of the Solr cores. 183 * @throws IOException If an I/O error occurs. 184 * @throws SolrServerException If a Solr error occurs. 185 */ 186 @SuppressWarnings("unchecked") 187 public Set<String> getCoreNames() throws IOException, SolrServerException 188 { 189 Set<String> coreNames = new HashSet<>(); 190 191 getLogger().debug("Getting core list."); 192 193 SolrQuery query = new SolrQuery(); 194 query.setRequestHandler("/admin/cores"); 195 query.setParam("action", "STATUS"); 196 197 QueryResponse response = _defaultSolrClient().query(query); 198 199 NamedList<NamedList<?>> status = (NamedList<NamedList<?>>) response.getResponse().get("status"); 200 for (Map.Entry<String, NamedList<?>> core : status) 201 { 202 String fullName = (String) core.getValue().get("name"); 203 if (fullName.startsWith(_solrCorePrefix)) 204 { 205 coreNames.add(fullName.substring(_solrCorePrefix.length())); 206 } 207 } 208 209 return coreNames; 210 } 211 212 /** 213 * Get the names of the Solr cores. 214 * @return The names of the Solr cores. 215 * @throws IOException If an I/O error occurs. 216 * @throws SolrServerException If a Solr error occurs. 217 */ 218 protected Set<String> getRealCoreNames() throws IOException, SolrServerException 219 { 220 Set<String> coreNames = new HashSet<>(); 221 222 getLogger().debug("Getting core list."); 223 224 CoreAdminResponse response = new CoreAdminRequest().process(_defaultSolrClient()); 225 226 NamedList<NamedList<Object>> status = response.getCoreStatus(); 227 for (Map.Entry<String, NamedList<Object>> core : status) 228 { 229 String fullName = (String) core.getValue().get("name"); 230 if (fullName.startsWith(_solrCorePrefix)) 231 { 232 coreNames.add(fullName.substring(_solrCorePrefix.length())); 233 } 234 } 235 236 return coreNames; 237 } 238 239 /** 240 * Create a Solr core. 241 * @param name The name of the core to create. 242 * @throws IOException If an I/O error occurs. 243 * @throws SolrServerException If a Solr error occurs. 244 */ 245 public void createCore(String name) throws IOException, SolrServerException 246 { 247 String fullName = _solrCorePrefix + name; 248 249 getLogger().info("Creating core '{}' (full name: '{}').", name, fullName); 250 251 SolrQuery query = new SolrQuery(); 252 query.setRequestHandler("/admin/cores"); 253 query.setParam("action", "CREATE"); 254 query.setParam("name", fullName); 255 query.setParam("configSet", _CONFIGSET_NAME); 256 257 QueryResponse response = _defaultSolrClient().query(query); 258 NamedList<?> results = response.getResponse(); 259 260 NamedList<?> error = (NamedList<?>) results.get("error"); 261 if (error != null) 262 { 263 throw new IOException("Error creating the core: " + error.get("msg")); 264 } 265 } 266 267 /** 268 * Delete a Solr core. 269 * @param name The name of the core to delete. 270 * @throws IOException If an I/O error occurs. 271 * @throws SolrServerException If a Solr error occurs. 272 */ 273 public void deleteCore(String name) throws IOException, SolrServerException 274 { 275 String fullName = _solrCorePrefix + name; 276 277 getLogger().info("Deleting core '{}' (full name: '{}').", name, fullName); 278 279 SolrQuery query = new SolrQuery(); 280 query.setRequestHandler("/admin/cores"); 281 query.setParam("action", "UNLOAD"); 282 query.setParam("core", fullName); 283 query.setParam("deleteInstanceDir", "true"); 284 285 QueryResponse response = _defaultSolrClient().query(query); 286 NamedList<?> results = response.getResponse(); 287 288 NamedList<?> error = (NamedList<?>) results.get("error"); 289 if (error != null) 290 { 291 throw new IOException("Error creating the core: " + error.get("msg")); 292 } 293 } 294 295 /** 296 * Send the schema. 297 * @throws IOException If a communication error occurs. 298 * @throws SolrServerException If a solr error occurs. 299 */ 300 public void sendSchema() throws IOException, SolrServerException 301 { 302 getLogger().info("Computing and sending the schema to the solr server."); 303 304 String workspaceName = _workspaceSelector.getWorkspace(); 305 String collection = _solrClientProvider.getCollectionName(workspaceName); 306 SolrClient solrClient = _getSolrClient(workspaceName); 307 308// SchemaRepresentation staticSchema = _schemaHelper.getStaticSchema(); 309 SchemaRepresentation staticSchema = _schemaHelper.getSchema("resource://org/ametys/cms/search/solr/schema/schema.xml"); 310 311 // TODO Clear the schema except fields marked ametysReadOnly="true". 312 313 // Clear the current schema. 314 clearSchema(solrClient, collection); 315 316 SchemaRequest schemaRequest = new SchemaRequest(); 317 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 318 319 // The cleared schema contains only the basic fields which can't be deleted. 320 SchemaRepresentation clearedSchema = schemaResponse.getSchemaRepresentation(); 321 SchemaFields schemaFields = new SchemaFields(clearedSchema); 322 323 getLogger().debug("Schema after clear: \n{}", schemaFields.toString()); 324 325 // Add the static schema types and fields. 326 List<SchemaRequest.Update> updates = new ArrayList<>(); 327 328 // Set "add field" definitions from the static schema to the update list. 329 addStaticSchemaUpdates(updates, staticSchema, schemaFields); 330 331 getLogger().debug("Temporary schema after static add: \n{}", schemaFields.toString()); 332 333 // Set "add field" definitions from the static schema to the update list. 334 addCustomUpdates(updates, schemaFields); 335 336 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(updates); 337 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 338 339 getLogger().debug("Send schema response: {}", updateResponse.toString()); 340 Object errors = updateResponse.getResponse().get("errors"); 341 if (errors != null && errors instanceof List && !((List) errors).isEmpty()) 342 { 343 String msg = "An error occured with the sent schema to Solr, it contains errors:\n" + errors.toString(); 344 throw new SolrServerException(msg); 345 } 346 347 getLogger().info("Schema sent to the solr server."); 348 349 reloadCores(); 350 } 351 352 /** 353 * Compute the list of {@link Update} directives from the static schema. 354 * @param updates The list of {@link Update} directives to fill. 355 * @param staticSchema The static schema representation. 356 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 357 */ 358 protected void addStaticSchemaUpdates(List<SchemaRequest.Update> updates, SchemaRepresentation staticSchema, SchemaFields schemaFields) 359 { 360 List<FieldTypeDefinition> fieldTypes = staticSchema.getFieldTypes(); 361 for (FieldTypeDefinition fieldType : fieldTypes) 362 { 363 String name = (String) fieldType.getAttributes().get("name"); 364 if (!schemaFields.hasFieldType(name)) 365 { 366 updates.add(new SchemaRequest.AddFieldType(fieldType)); 367 schemaFields.addFieldType(name); 368 } 369 } 370 for (Map<String, Object> field : staticSchema.getFields()) 371 { 372 String name = (String) field.get("name"); 373 if (!schemaFields.hasField(name)) 374 { 375 updates.add(new SchemaRequest.AddField(field)); 376 schemaFields.addField(name); 377 } 378 } 379 for (Map<String, Object> field : staticSchema.getDynamicFields()) 380 { 381 String name = (String) field.get("name"); 382 if (!schemaFields.hasDynamicField(name)) 383 { 384 updates.add(new SchemaRequest.AddDynamicField(field)); 385 schemaFields.addDynamicField(name); 386 } 387 } 388 for (Map<String, Object> field : staticSchema.getCopyFields()) 389 { 390 String source = (String) field.get("source"); 391 String dest = (String) field.get("dest"); 392 if (!schemaFields.hasCopyField(source, dest)) 393 { 394 updates.add(new SchemaRequest.AddCopyField(source, Arrays.asList(dest))); 395 schemaFields.addCopyField(source, dest); 396 } 397 } 398 } 399 400 /** 401 * Compute the list of custom {@link Update} directives. 402 * @param updates The list of {@link Update} directives to fill. 403 * @param schemaFields The current schema fields, used to track the existing fields (to be filled). 404 */ 405 protected void addCustomUpdates(List<SchemaRequest.Update> updates, SchemaFields schemaFields) 406 { 407 // Add all our property-managed fields. 408 for (String providerId : _schemaDefProviderEP.getExtensionsIds()) 409 { 410 SchemaDefinitionProvider definitionProvider = _schemaDefProviderEP.getExtension(providerId); 411 412 for (SchemaDefinition definition : definitionProvider.getDefinitions()) 413 { 414 if (!definitionExists(definition, schemaFields)) 415 { 416 SchemaRequest.Update update = getSchemaUpdate(definition); 417 if (update != null) 418 { 419 updates.add(update); 420 } 421 } 422 } 423 } 424 425// for (String propId : _sysPropEP.getExtensionsIds()) 426// { 427// Collection<SchemaDefinition> definitions = _sysPropEP.getExtension(propId).getSchemaDefinitions(); 428// 429// for (SchemaDefinition definition : definitions) 430// { 431// if (!definitionExists(definition, schemaFields)) 432// { 433// SchemaRequest.Update update = getSchemaUpdate(definition); 434// if (update != null) 435// { 436// updates.add(update); 437// } 438// } 439// } 440// } 441 } 442 443 /** 444 * Test if the given schema definition exists in the given {@link SchemaFields} reference. 445 * @param definition The schema definition to test. 446 * @param schemaFields The current schema fields. 447 * @return true if the SchemaFields contain the schema definition. 448 */ 449 protected boolean definitionExists(SchemaDefinition definition, SchemaFields schemaFields) 450 { 451 if (definition instanceof FieldDefinition) 452 { 453 FieldDefinition fieldDef = (FieldDefinition) definition; 454 if (fieldDef.isDynamic()) 455 { 456 return schemaFields.hasField(fieldDef.getName()); 457 } 458 else 459 { 460 return schemaFields.hasDynamicField(fieldDef.getName()); 461 } 462 } 463 else if (definition instanceof CopyFieldDefinition) 464 { 465 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 466 return schemaFields.hasCopyField(fieldDef.getSource(), fieldDef.getDestination()); 467 } 468 469 return false; 470 } 471 472 /** 473 * Delete all the fields of the existing schema in the given collection. 474 * @param solrClient The Solr client 475 * @param collection The collection. 476 * @throws IOException If a communication error occurs. 477 * @throws SolrServerException If a solr error occurs. 478 */ 479 protected void clearSchema(SolrClient solrClient, String collection) throws IOException, SolrServerException 480 { 481 try 482 { 483 getLogger().info("Clearing the existing schema on the solr server."); 484 485 SchemaRequest schemaRequest = new SchemaRequest(); 486 SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection); 487 488 SchemaRepresentation schema = schemaResponse.getSchemaRepresentation(); 489 490 List<SchemaRequest.Update> deletions = new ArrayList<>(); 491 492 // First the copy fields, then dynamic and simple fields, and field types in the end. 493 for (Map<String, Object> field : schema.getCopyFields()) 494 { 495 String source = (String) field.get("source"); 496 String dest = (String) field.get("dest"); 497 deletions.add(new SchemaRequest.DeleteCopyField(source, Arrays.asList(dest))); 498 } 499 for (Map<String, Object> field : schema.getDynamicFields()) 500 { 501 String name = (String) field.get("name"); 502 deletions.add(new SchemaRequest.DeleteDynamicField(name)); 503 } 504 for (Map<String, Object> field : schema.getFields()) 505 { 506 String name = (String) field.get("name"); 507 if (!_READ_ONLY_FIELDS.contains(name)) 508 { 509 deletions.add(new SchemaRequest.DeleteField(name)); 510 } 511 } 512 for (FieldTypeDefinition fieldType : schema.getFieldTypes()) 513 { 514 String name = (String) fieldType.getAttributes().get("name"); 515 if (!_READ_ONLY_FIELDTYPES.contains(name)) 516 { 517 deletions.add(new SchemaRequest.DeleteFieldType(name)); 518 } 519 } 520 521 SchemaRequest.MultiUpdate multiUpdate = new SchemaRequest.MultiUpdate(deletions); 522 SchemaResponse.UpdateResponse updateResponse = multiUpdate.process(solrClient, collection); 523 524 getLogger().debug("Clear schema response: {}", updateResponse.toString()); 525 526 getLogger().info("Solr schema cleared."); 527 } 528 catch (SolrServerException | IOException e) 529 { 530 getLogger().error("Error clearing schema in collection " + collection, e); 531 throw e; 532 } 533 } 534 535 /** 536 * Get the schema {@link Update} directive from the given schema definition. 537 * @param definition The schema definition to add. 538 * @return The add {@link Update} directive. 539 */ 540 protected SchemaRequest.Update getSchemaUpdate(SchemaDefinition definition) 541 { 542 SchemaRequest.Update update = null; 543 544 if (definition instanceof FieldDefinition) 545 { 546 FieldDefinition fieldDef = (FieldDefinition) definition; 547 548 // Force indexed and stored attributes to true. 549 Map<String, Object> attributes = new HashMap<>(); 550 attributes.put("name", fieldDef.getName()); 551 attributes.put("type", fieldDef.getType()); 552 attributes.put("multiValued", fieldDef.isMultiValued()); 553 attributes.put("docValues", fieldDef.isDocValues()); 554 attributes.put("indexed", Boolean.TRUE); 555 attributes.put("stored", Boolean.TRUE); 556 557 update = new SchemaRequest.AddField(attributes); 558 } 559 else if (definition instanceof CopyFieldDefinition) 560 { 561 CopyFieldDefinition fieldDef = (CopyFieldDefinition) definition; 562 563 String source = fieldDef.getSource(); 564 String dest = fieldDef.getDestination(); 565 566 update = new SchemaRequest.AddCopyField(source, Arrays.asList(dest)); 567 } 568 569 return update; 570 } 571 572 /** 573 * Reload the solr cores. 574 * @throws IOException If a communication error occurs. 575 * @throws SolrServerException If a solr error occurs. 576 */ 577 protected void reloadCores() throws IOException, SolrServerException 578 { 579 getLogger().info("Reloading solr cores."); 580 581 for (String coreName : getCoreNames()) 582 { 583 String fullName = _solrCorePrefix + coreName; 584 585 CoreAdminResponse reloadResponse = CoreAdminRequest.reloadCore(fullName, _defaultSolrClient()); 586 587 getLogger().debug("Reload core response: {}", reloadResponse.toString()); 588 } 589 590 getLogger().info("All cores reloaded."); 591 } 592 593 /** 594 * Index all the contents in a given workspace. 595 * @param workspaceName the workspace where to index 596 * @param commit true to commit 597 * @return The indexation result as a Map. 598 * @throws Exception if an error occurs while indexing. 599 */ 600 public Map<String, Object> indexAllContents(String workspaceName, boolean commit) throws Exception 601 { 602 Request request = ContextHelper.getRequest(_context); 603 604 // Retrieve the current workspace. 605 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 606 607 try 608 { 609 // Force the workspace. 610 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 611 612 getLogger().info("Starting the indexation of all contents for workspace {}", workspaceName); 613 614 long start = System.currentTimeMillis(); 615 616 // Delete all contents 617 unindexAllContents(workspaceName, commit); 618 619 String query = ContentQueryHelper.getContentXPathQuery(null); 620 AmetysObjectIterable<Content> contents = _resolver.query(query); 621 622 IndexationResult result = doIndexContents(contents, workspaceName, commit); 623 624 long end = System.currentTimeMillis(); 625 626 if (!result.hasErrors()) 627 { 628 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 629 } 630 else 631 { 632 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 633 } 634 635 Map<String, Object> results = new HashMap<>(); 636 results.put("successCount", result.getSuccessCount()); 637 if (result.hasErrors()) 638 { 639 results.put("errorCount", result.getErrorCount()); 640 } 641 642 return results; 643 } 644 catch (Exception e) 645 { 646 String error = String.format("Failed to index all contents in workspace %s", workspaceName); 647 getLogger().error(error, e); 648 throw new IndexingException(error, e); 649 } 650 finally 651 { 652 // Restore context 653 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 654 } 655 } 656 657 /** 658 * Unindex all content documents. 659 * @param workspaceName The workspace name 660 * @param commit true to commit 661 * @throws Exception if an error occurs while unindexing. 662 */ 663 protected void unindexAllContents(String workspaceName, boolean commit) throws Exception 664 { 665 String collection = _solrClientProvider.getCollectionName(workspaceName); 666 _getSolrClient(workspaceName).deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_CONTENT); 667 668 if (commit) 669 { 670 commit(workspaceName); 671 } 672 } 673 674 /** 675 * Add or update the child contents of a {@link AmetysObjectCollection} into Solr index, for all workspaces and commit 676 * @param collectionId The id of collection 677 * @throws Exception if an error occurs while indexing. 678 */ 679 public void indexSubcontents(String collectionId) throws Exception 680 { 681 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 682 for (String workspaceName : workspaceNames) 683 { 684 indexSubcontents(collectionId, workspaceName, true); 685 } 686 } 687 688 /** 689 * Index the child contents of a {@link AmetysObjectCollection} 690 * @param collectionId The id of collection 691 * @param workspaceName the workspace where to index 692 * @param commit true to commit 693 * @throws Exception if an error occurs while unindexing. 694 */ 695 public void indexSubcontents(String collectionId, String workspaceName, boolean commit) throws Exception 696 { 697 Request request = ContextHelper.getRequest(_context); 698 699 // Retrieve the current workspace. 700 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 701 702 try 703 { 704 // Force the workspace. 705 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 706 707 if (_resolver.hasAmetysObjectForId(collectionId)) 708 { 709 AmetysObjectCollection collection = _resolver.resolveById(collectionId); 710 AmetysObjectIterable<AmetysObject> children = collection.getChildren(); 711 712 for (AmetysObject child : children) 713 { 714 if (child instanceof Content) 715 { 716 Content content = (Content) child; 717 718 _doIndexContent(content, workspaceName, false); 719 } 720 } 721 722 if (commit) 723 { 724 commit(workspaceName); 725 } 726 } 727 } 728 finally 729 { 730 // Restore context 731 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 732 } 733 } 734 735 /** 736 * Add or update a content into Solr index on all workspaces and commit 737 * @param contentId The id of the content to index 738 * @throws Exception if an error occurs while indexing. 739 */ 740 public void indexContent(String contentId) throws Exception 741 { 742 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 743 for (String workspaceName : workspaceNames) 744 { 745 indexContent(contentId, workspaceName, true); 746 } 747 } 748 749 /** 750 * Add or update a content into Solr index 751 * @param contentId The id of the content to index 752 * @param workspaceName the workspace where to index 753 * @param commit true to commit the indexation 754 * @throws Exception if an error occurs while indexing. 755 */ 756 public void indexContent(String contentId, String workspaceName, boolean commit) throws Exception 757 { 758 Request request = ContextHelper.getRequest(_context); 759 760 // Retrieve the current workspace. 761 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 762 763 try 764 { 765 // Force the workspace. 766 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 767 768 if (_resolver.hasAmetysObjectForId(contentId)) 769 { 770 Content content = _resolver.resolveById(contentId); 771 _doIndexContent(content, workspaceName, commit); 772 } 773 } 774 finally 775 { 776 // Restore context 777 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 778 } 779 } 780 781 private void _doIndexContent(Content content, String workspaceName, boolean commit) throws IndexingException 782 { 783 try 784 { 785 long time_0 = System.currentTimeMillis(); 786 787 getLogger().debug("Indexing content {} into Solr for workspace {}", content.getId(), workspaceName); 788 789 deleteRepeaterDocs(content.getId(), workspaceName); 790 doIndexContent(content, workspaceName); 791 doIndexContentWorkflow(content, workspaceName, false); 792 793 if (commit) 794 { 795 commit(workspaceName); 796 } 797 798 getLogger().debug("Successfully indexed content {} in Solr in {} ms", content.getId(), System.currentTimeMillis() - time_0); 799 } 800 catch (Exception e) 801 { 802 String error = String.format("Failed to index content %s in workspace %s", content.getId(), workspaceName); 803 getLogger().error(error, e); 804 throw new IndexingException(error, e); 805 } 806 } 807 808 /** 809 * Send a collection of contents for indexation in the solr server on all workspaces and commit 810 * @param contents the collection of contents to index. 811 * @return the indexation result. 812 * @throws Exception if an error occurs while indexing. 813 */ 814 public IndexationResult indexContents(Iterable<Content> contents) throws Exception 815 { 816 IndexationResult result = new IndexationResult(); 817 818 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 819 for (String workspaceName : workspaceNames) 820 { 821 IndexationResult wResult = indexContents(contents, workspaceName, true); 822 result = new IndexationResult(result.getSuccessCount() + wResult.getSuccessCount(), result.getErrorCount() + wResult.getErrorCount()); 823 } 824 825 return result; 826 } 827 828 /** 829 * Send a collection of contents for indexation in the solr server. 830 * @param contents the collection of contents to index. 831 * @param workspaceName the workspace where to index 832 * @param commit true to commit the indexation 833 * @return the indexation result. 834 * @throws Exception if an error occurs while indexing. 835 */ 836 public IndexationResult indexContents(Iterable<Content> contents, String workspaceName, boolean commit) throws Exception 837 { 838 Request request = ContextHelper.getRequest(_context); 839 840 // Retrieve the current workspace. 841 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 842 843 try 844 { 845 // Force the workspace. 846 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 847 848 getLogger().info("Starting indexation of several contents for workspace {}", workspaceName); 849 850 long start = System.currentTimeMillis(); 851 852 IndexationResult result = doIndexContents(contents, workspaceName, commit); 853 854 long end = System.currentTimeMillis(); 855 856 if (!result.hasErrors()) 857 { 858 getLogger().info("{} contents indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 859 } 860 else 861 { 862 getLogger().info("Content indexation ended, the process took {} milliseconds. {} contents were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 863 } 864 865 return result; 866 } 867 catch (Exception e) 868 { 869 String error = String.format("Failed to index several contents in workspace %s", workspaceName); 870 getLogger().error(error, e); 871 throw new IndexingException(error, e); 872 } 873 finally 874 { 875 // Restore context 876 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 877 } 878 879 } 880 881 /** 882 * Send some contents for indexation in the solr server. 883 * @param contents the contents to index. 884 * @param workspaceName The workspace name 885 * @param commit true to commit transaction 886 * @return the indexation result. 887 * @throws Exception if an error occurs committing the results. 888 */ 889 protected IndexationResult doIndexContents(Iterable<Content> contents, String workspaceName, boolean commit) throws Exception 890 { 891 int successCount = 0; 892 int errorCount = 0; 893 for (Content content : contents) 894 { 895 try 896 { 897 doIndexContent(content, workspaceName); 898 doIndexContentWorkflow(content, workspaceName, false); 899 successCount++; 900 } 901 catch (Exception e) 902 { 903 getLogger().error("Error indexing content '" + content.getId() + "' in the solr server.", e); 904 errorCount++; 905 } 906 } 907 908 if (commit) 909 { 910 commit(workspaceName); 911 } 912 913 return new IndexationResult(successCount, errorCount); 914 } 915 916 /** 917 * Update the value of a specific system property in a content document. 918 * @param content The content to update. 919 * @param propertyId The system property ID. 920 * @param workspaceName The workspace name 921 * @param commit true to commit update 922 * @throws Exception if an error occurs while indexing. 923 */ 924 public void updateSystemProperty(Content content, String propertyId, String workspaceName, boolean commit) throws Exception 925 { 926 getLogger().debug("Updating the property '{}' for content {} into Solr.", propertyId, content); 927 928 SolrInputDocument document = new SolrInputDocument(); 929 _solrContentIndexer.indexPartialSystemProperty(content, propertyId, document); 930 931 String collection = _solrClientProvider.getCollectionName(workspaceName); 932 UpdateResponse solrResponse = _getSolrClient(workspaceName).add(collection, document); 933 int status = solrResponse.getStatus(); 934 935 if (status != 0) 936 { 937 throw new IOException("Indexing of property '" + propertyId + "': got status code '" + status + "'."); 938 } 939 940 if (commit) 941 { 942 commit(workspaceName); 943 } 944 945 getLogger().debug("Succesfully indexed '{}' property for content {} in Solr.", propertyId, content); 946 } 947 948 /** 949 * Remove a content from Solr index for all workspaces and commit 950 * @param contentId The id of content to unindex 951 * @throws Exception if an error occurs while indexing. 952 */ 953 public void unindexContent(String contentId) throws Exception 954 { 955 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 956 for (String workspaceName : workspaceNames) 957 { 958 unindexContent(contentId, workspaceName, true); 959 } 960 } 961 962 /** 963 * Remove a content from Solr index 964 * @param contentId The id of content to unindex 965 * @param workspaceName The workspace where to work in 966 * @param commit true to commit operation 967 * @throws Exception if an error occurs while indexing. 968 */ 969 public void unindexContent(String contentId, String workspaceName, boolean commit) throws Exception 970 { 971 Request request = ContextHelper.getRequest(_context); 972 973 // Retrieve the current workspace. 974 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 975 976 try 977 { 978 // Force the workspace. 979 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 980 981 getLogger().debug("Unindexing content {} from Solr for workspace {}", contentId, workspaceName); 982 983 deleteRepeaterDocs(contentId, workspaceName); 984 doUnindexDocument(contentId, workspaceName); 985 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, false); 986 987 if (commit) 988 { 989 commit(workspaceName); 990 } 991 992 getLogger().debug("Succesfully deleted content {} from Solr.", contentId); 993 } 994 catch (Exception e) 995 { 996 String error = String.format("Failed to unindex content %s in workspace %s", contentId, workspaceName); 997 getLogger().error(error, e); 998 throw new IndexingException(error, e); 999 } 1000 finally 1001 { 1002 // Restore workspace 1003 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1004 } 1005 } 1006 1007 /** 1008 * Remove a content from Solr index for all workspaces and commit 1009 * @param contentIds The id of content to unindex 1010 * @throws Exception if an error occurs while indexing. 1011 */ 1012 public void unindexContents(Collection<String> contentIds) throws Exception 1013 { 1014 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1015 for (String workspaceName : workspaceNames) 1016 { 1017 unindexContents(contentIds, workspaceName, true); 1018 } 1019 } 1020 1021 /** 1022 * Remove a content from Solr index 1023 * @param contentIds The id of content to unindex 1024 * @param workspaceName The workspace where to work in 1025 * @param commit true to commit 1026 * @throws Exception if an error occurs while indexing. 1027 */ 1028 public void unindexContents(Collection<String> contentIds, String workspaceName, boolean commit) throws Exception 1029 { 1030 Request request = ContextHelper.getRequest(_context); 1031 1032 // Retrieve the current workspace. 1033 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1034 1035 try 1036 { 1037 // Force the workspace. 1038 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1039 1040 getLogger().debug("Unindexing several contents from Solr."); 1041 1042 for (String contentId : contentIds) 1043 { 1044 deleteRepeaterDocs(contentId, workspaceName); 1045 doUnindexDocument(contentId, workspaceName); 1046 _solrWorkflowIndexer.unindexAmetysObjectWorkflow(contentId, workspaceName, false); 1047 } 1048 1049 if (commit) 1050 { 1051 commit(workspaceName); 1052 } 1053 1054 getLogger().debug("Succesfully unindexed content from Solr."); 1055 } 1056 catch (Exception e) 1057 { 1058 String error = String.format("Failed to unindex several contents in workspace %s", workspaceName); 1059 getLogger().error(error, e); 1060 throw new IndexingException(error, e); 1061 } 1062 finally 1063 { 1064 // Restore workspace 1065 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1066 } 1067 } 1068 1069 /** 1070 * Add or update a content into Solr index 1071 * @param content The content to index 1072 * @param workspaceName The workspace where to index 1073 * @throws Exception if an error occurs while indexing. 1074 */ 1075 protected void doIndexContent(Content content, String workspaceName) throws Exception 1076 { 1077 long time_0 = System.currentTimeMillis(); 1078 1079 SolrInputDocument document = new SolrInputDocument(); 1080 List<SolrInputDocument> additionalDocuments = new ArrayList<>(); 1081 1082 _solrContentIndexer.indexContent(content, document, additionalDocuments); 1083 1084 long time_1 = System.currentTimeMillis(); 1085 getLogger().debug("Populate indexing fields for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_1 - time_0); 1086 1087 doIndexContentAcls(content, document); 1088 1089 long time_2 = System.currentTimeMillis(); 1090 getLogger().debug("Populate ACL for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_2 - time_1); 1091 1092 List<SolrInputDocument> documents = new ArrayList<>(additionalDocuments); 1093 documents.add(document); 1094 1095 UpdateResponse solrResponse = _getSolrClient(workspaceName).add(_solrClientProvider.getCollectionName(workspaceName), documents); 1096 int status = solrResponse.getStatus(); 1097 1098 long time_3 = System.currentTimeMillis(); 1099 getLogger().debug("Update document for content {} for workspace {} in {} ms", content.getId(), workspaceName, time_3 - time_2); 1100 1101 if (status != 0) 1102 { 1103 throw new IOException("Content indexation: got status code '" + status + "'."); 1104 } 1105 } 1106 1107 /** 1108 * Index the users and groups who are allowed to access the content. 1109 * @param content The content. 1110 * @param document The document. 1111 * @throws Exception If an error occurs while indexing. 1112 */ 1113 protected void doIndexContentAcls(Content content, SolrInputDocument document) throws Exception 1114 { 1115 _solrContentRightIndexer.indexContentAccess(content, document); 1116 } 1117 1118 /** 1119 * Index the whole workflow of a content. 1120 * @param content The content. 1121 * @param workspaceName The workspace name 1122 * @param commit true to commit, false otherwise. 1123 * @throws Exception if an error occurs while indexing. 1124 */ 1125 protected void doIndexContentWorkflow(Content content, String workspaceName, boolean commit) throws Exception 1126 { 1127 if (content instanceof WorkflowAwareContent) 1128 { 1129 _solrWorkflowIndexer.indexAmetysObjectWorkflow((WorkflowAwareContent) content, workspaceName, commit); 1130 } 1131 } 1132 1133 /** 1134 * Delete repeater documents of a specified content. 1135 * @param workspaceName The workspace name 1136 * @param contentId the content ID. 1137 * @throws Exception if an error occurs while indexing. 1138 */ 1139 protected void deleteRepeaterDocs(String contentId, String workspaceName) throws Exception 1140 { 1141 long time_0 = System.currentTimeMillis(); 1142 1143 // _documentType:repeater AND id:content\://xxx/* 1144 StringBuilder query = new StringBuilder(); 1145 query.append(SolrFieldNames.DOCUMENT_TYPE).append(':').append(SolrFieldNames.TYPE_REPEATER) 1146 .append(" AND id:").append(ClientUtils.escapeQueryChars(contentId)).append("/*"); 1147 1148 _getSolrClient(workspaceName).deleteByQuery(_solrClientProvider.getCollectionName(workspaceName), query.toString()); 1149 1150 getLogger().debug("Successfully delete repeaters documents for content {} in {} ms", contentId, System.currentTimeMillis() - time_0); 1151 } 1152 1153 /** 1154 * Index all the resources in a given workspace. 1155 * @param workspaceName The workspace where to index 1156 * @param commit true to commit indexation 1157 * @return The indexation result as a Map. 1158 * @throws Exception if an error occurs while indexing. 1159 */ 1160 public Map<String, Object> indexAllResources(String workspaceName, boolean commit) throws Exception 1161 { 1162 Request request = ContextHelper.getRequest(_context); 1163 1164 // Retrieve the current workspace. 1165 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1166 1167 try 1168 { 1169 // Force the workspace. 1170 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1171 1172 getLogger().info("Starting the indexation of all resources for workspace {}", workspaceName); 1173 1174 long start = System.currentTimeMillis(); 1175 1176 // Delete all resources 1177 unindexAllResources(workspaceName, commit); 1178 1179 Map<String, Object> results = new HashMap<>(); 1180 1181 try 1182 { 1183 TraversableAmetysObject resourceRoot = _resolver.resolveByPath(RepositoryConstants.NAMESPACE_PREFIX + ":resources"); 1184 AmetysObjectIterable<Resource> resources = resourceRoot.getChildren(); 1185 1186 IndexationResult result = doIndexResources(resources, SolrFieldNames.TYPE_RESOURCE, resourceRoot, workspaceName, commit); 1187 1188 long end = System.currentTimeMillis(); 1189 1190 if (!result.hasErrors()) 1191 { 1192 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1193 } 1194 else 1195 { 1196 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1197 } 1198 1199 results.put("successCount", result.getSuccessCount()); 1200 if (result.hasErrors()) 1201 { 1202 results.put("errorCount", result.getErrorCount()); 1203 } 1204 } 1205 catch (UnknownAmetysObjectException e) 1206 { 1207 getLogger().info("There is no root for resources in current workspace."); 1208 } 1209 1210 return results; 1211 } 1212 catch (Exception e) 1213 { 1214 String error = String.format("Failed to index all resources in workspace %s", workspaceName); 1215 getLogger().error(error, e); 1216 throw new IndexingException(error, e); 1217 } 1218 finally 1219 { 1220 // Restore context 1221 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1222 } 1223 } 1224 1225 /** 1226 * Send a collection of contents for indexation in the solr server. 1227 * @param resources the collection of contents to index. 1228 * @param documentType The document type of the resource 1229 * @param workspaceName The workspace where to index 1230 * @param commit true to commit indexation 1231 * @return the indexation result. 1232 * @throws Exception if an error occurs while indexing. 1233 */ 1234 public IndexationResult indexResources(Iterable<AmetysObject> resources, String documentType, String workspaceName, boolean commit) throws Exception 1235 { 1236 return indexResources(resources, documentType, null, workspaceName, commit); 1237 } 1238 1239 /** 1240 * Send a collection of contents for indexation in the solr server. 1241 * @param resources the collection of contents to index. 1242 * @param documentType The document type of the resource 1243 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1244 * @param workspaceName The workspace where to index 1245 * @param commit true to commit indexation 1246 * @return the indexation result. 1247 * @throws Exception if an error occurs while indexing. 1248 */ 1249 public IndexationResult indexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, boolean commit) throws Exception 1250 { 1251 Request request = ContextHelper.getRequest(_context); 1252 1253 // Retrieve the current workspace. 1254 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1255 1256 try 1257 { 1258 // Force the workspace. 1259 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1260 1261 getLogger().info("Starting indexation of several resources for workspace {}", workspaceName); 1262 1263 long start = System.currentTimeMillis(); 1264 1265 IndexationResult result = doIndexResources(resources, documentType, resourceRoot, workspaceName, commit); 1266 1267 long end = System.currentTimeMillis(); 1268 1269 if (!result.hasErrors()) 1270 { 1271 getLogger().info("{} resources indexed without error in {} milliseconds.", result.getSuccessCount(), end - start); 1272 } 1273 else 1274 { 1275 getLogger().info("Resource indexation ended, the process took {} milliseconds. {} resources were not indexed successfully, please review the error logs above for more details.", end - start, result.getErrorCount()); 1276 } 1277 1278 return result; 1279 } 1280 catch (Exception e) 1281 { 1282 String error = String.format("Failed to index several resources in workspace %s", workspaceName); 1283 getLogger().error(error, e); 1284 throw new IndexingException(error, e); 1285 } 1286 finally 1287 { 1288 // Restore context 1289 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1290 } 1291 } 1292 1293 /** 1294 * Send some resources for indexation in the solr server. 1295 * @param resources the resources to index. 1296 * @param documentType The document type of the resource 1297 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1298 * @param workspaceName The workspace where to index 1299 * @param commit true to commit indexation 1300 * @return the indexation result. 1301 * @throws Exception if an error occurs committing the results. 1302 */ 1303 protected IndexationResult doIndexResources(Iterable<? extends AmetysObject> resources, String documentType, TraversableAmetysObject resourceRoot, String workspaceName, boolean commit) throws Exception 1304 { 1305 int successCount = 0; 1306 int errorCount = 0; 1307 for (AmetysObject resource : resources) 1308 { 1309 try 1310 { 1311 doIndexExplorerItem(resource, documentType, resourceRoot); 1312 successCount++; 1313 } 1314 catch (Exception e) 1315 { 1316 getLogger().error("Error indexing resource '" + resource.getId() + "' in the solr server.", e); 1317 errorCount++; 1318 } 1319 } 1320 1321 if (commit) 1322 { 1323 commit(workspaceName); 1324 } 1325 1326 return new IndexationResult(successCount, errorCount); 1327 } 1328 1329 /** 1330 * Add or update a resource into Solr index 1331 * @param resource The resource to index 1332 * @param documentType The document type of the resource 1333 * @param workspaceName The workspace where to index 1334 * @param commit true to commit indexation 1335 * @throws Exception if an error occurs while indexing. 1336 */ 1337 public void indexResource(Resource resource, String documentType, String workspaceName, boolean commit) throws Exception 1338 { 1339 Request request = ContextHelper.getRequest(_context); 1340 1341 // Retrieve the current workspace. 1342 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1343 1344 try 1345 { 1346 // Force the workspace. 1347 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1348 1349 getLogger().debug("Indexing resource {} into Solr.", resource.getId()); 1350 1351 doIndexResource(resource, documentType, null); 1352 1353 if (commit) 1354 { 1355 commit(workspaceName); 1356 } 1357 1358 getLogger().debug("Succesfully indexed resource {} in Solr.", resource.getId()); 1359 } 1360 catch (Exception e) 1361 { 1362 String error = String.format("Failed to index resource %s in workspace %s", resource.getId(), workspaceName); 1363 getLogger().error(error, e); 1364 throw new IndexingException(error, e); 1365 } 1366 finally 1367 { 1368 // Restore context 1369 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1370 } 1371 } 1372 1373 /** 1374 * Delete all resource documents from the solr index. 1375 * @param workspaceName The workspace name 1376 * @param commit true to commit deletion 1377 * @throws Exception If an error occurs while unindexing. 1378 */ 1379 public void unindexAllResources(String workspaceName, boolean commit) throws Exception 1380 { 1381 getLogger().debug("Unindexing all resources from Solr."); 1382 1383 String collection = _solrClientProvider.getCollectionName(workspaceName); 1384 _getSolrClient(workspaceName).deleteByQuery(collection, SolrFieldNames.DOCUMENT_TYPE + ':' + SolrFieldNames.TYPE_RESOURCE); 1385 1386 if (commit) 1387 { 1388 commit(workspaceName); 1389 } 1390 1391 getLogger().debug("Succesfully deleted all resource documents from Solr."); 1392 } 1393 1394 /** 1395 * Delete all resource documents at a given path for all workspaces and commit 1396 * @param rootId The resource root ID, must not be null. 1397 * @param path The resource path relative to the given root, must start with a slash. 1398 * @throws Exception If an error occurs while unindexing. 1399 */ 1400 public void unindexResourcesByPath(String rootId, String path) throws Exception 1401 { 1402 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1403 for (String workspaceName : workspaceNames) 1404 { 1405 unindexResourcesByPath(rootId, path, workspaceName, true); 1406 } 1407 } 1408 1409 /** 1410 * Delete all resource documents at a given path. 1411 * @param rootId The resource root ID, must not be null. 1412 * @param path The resource path relative to the given root, must start with a slash. 1413 * @param workspaceName The workspace where to work in 1414 * @param commit true to commit operation 1415 * @throws Exception If an error occurs while unindexing. 1416 */ 1417 public void unindexResourcesByPath(String rootId, String path, String workspaceName, boolean commit) throws Exception 1418 { 1419 Request request = ContextHelper.getRequest(_context); 1420 1421 // Retrieve the current workspace. 1422 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1423 1424 try 1425 { 1426 // Force the workspace. 1427 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1428 1429 getLogger().debug("Unindexing all resources at path {} in root {}", path, rootId); 1430 1431 Query query = new ResourceLocationQuery(rootId, path); 1432 1433 String collection = _solrClientProvider.getCollectionName(workspaceName); 1434 _getSolrClient(workspaceName).deleteByQuery(collection, query.build()); 1435 1436 if (commit) 1437 { 1438 commit(workspaceName); 1439 } 1440 1441 getLogger().debug("Succesfully deleted resource document from Solr."); 1442 } 1443 catch (Exception e) 1444 { 1445 String error = String.format("Failed to unindex resource %s in workspace %s", path, workspaceName); 1446 getLogger().error(error, e); 1447 throw new IndexingException(error, e); 1448 } 1449 finally 1450 { 1451 // Restore workspace 1452 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1453 } 1454 } 1455 1456 /** 1457 * Remove a resource from Solr index for all workspaces and commit 1458 * @param resourceId The id of resource to unindex 1459 * @throws Exception if an error occurs while indexing. 1460 */ 1461 public void unindexResource(String resourceId) throws Exception 1462 { 1463 String[] workspaceNames = _workspaceSelector.getWorkspaces(); 1464 for (String workspaceName : workspaceNames) 1465 { 1466 unindexResource(resourceId, workspaceName, true); 1467 } 1468 } 1469 1470 /** 1471 * Remove a resource from the Solr index. 1472 * @param resourceId The id of resource to unindex 1473 * @param workspaceName The workspace where to work in 1474 * @param commit true to commit operation 1475 * @throws Exception if an error occurs while unindexing. 1476 */ 1477 public void unindexResource(String resourceId, String workspaceName, boolean commit) throws Exception 1478 { 1479 Request request = ContextHelper.getRequest(_context); 1480 1481 // Retrieve the current workspace. 1482 String currentWsp = RequestAttributeWorkspaceSelector.getForcedWorkspace(request); 1483 1484 try 1485 { 1486 // Force the workspace. 1487 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, workspaceName); 1488 1489 getLogger().debug("Unindexing resource {} from Solr.", resourceId); 1490 1491 doUnindexDocument(resourceId, workspaceName); 1492 1493 if (commit) 1494 { 1495 commit(workspaceName); 1496 } 1497 1498 getLogger().debug("Succesfully deleted resource {} from Solr.", resourceId); 1499 } 1500 catch (Exception e) 1501 { 1502 String error = String.format("Failed to unindex resource %s in workspace %s", resourceId, workspaceName); 1503 getLogger().error(error, e); 1504 throw new IndexingException(error, e); 1505 } 1506 finally 1507 { 1508 // Restore workspace 1509 RequestAttributeWorkspaceSelector.setForcedWorkspace(request, currentWsp); 1510 } 1511 } 1512 1513 /** 1514 * Add or update a resource into Solr index 1515 * @param node The resource to index 1516 * @param documentType The document type of the resource 1517 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1518 * @throws Exception if an error occurs while indexing. 1519 */ 1520 protected void doIndexExplorerItem(AmetysObject node, String documentType, TraversableAmetysObject resourceRoot) throws Exception 1521 { 1522 if (node instanceof ResourceCollection) 1523 { 1524 for (AmetysObject child : ((ResourceCollection) node).getChildren()) 1525 { 1526 doIndexExplorerItem(child, documentType, resourceRoot); 1527 } 1528 } 1529 else if (node instanceof Resource) 1530 { 1531 doIndexResource((Resource) node, documentType, resourceRoot); 1532 } 1533 } 1534 1535 /** 1536 * Add or update a resource into Solr index 1537 * @param resource The resource to index 1538 * @param documentType The document type of the resource 1539 * @param resourceRoot The resource root, can be null. In that case, it will be computed for each resource. 1540 * @throws Exception if an error occurs while indexing. 1541 */ 1542 protected void doIndexResource(Resource resource, String documentType, TraversableAmetysObject resourceRoot) throws Exception 1543 { 1544 SolrInputDocument document = new SolrInputDocument(); 1545 1546 _solrResourceIndexer.indexResource(resource, document, documentType, resourceRoot); 1547 1548 String workspaceName = _workspaceSelector.getWorkspace(); 1549 UpdateResponse solrResponse = _getSolrClient(workspaceName).add(_solrClientProvider.getCollectionName(workspaceName), document); 1550 int status = solrResponse.getStatus(); 1551 1552 if (status != 0) 1553 { 1554 throw new IOException("Resource indexation: got status code '" + status + "'."); 1555 } 1556 } 1557 1558 /** 1559 * Process a Solr commit operation in given workspace. 1560 * @param workspaceName The workspace's name 1561 * @throws SolrServerException if there is an error on the server 1562 * @throws IOException if there is a communication error with the server 1563 */ 1564 public void commit(String workspaceName) throws SolrServerException, IOException 1565 { 1566 long time_0 = System.currentTimeMillis(); 1567 1568 // Commit 1569 UpdateResponse solrResponse = _getSolrClient(workspaceName).commit(_solrClientProvider.getCollectionName(workspaceName)); 1570 int status = solrResponse.getStatus(); 1571 1572 if (status != 0) 1573 { 1574 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 1575 } 1576 1577 getLogger().debug("Successful Solr commit operation during an Ametys indexing process in {} ms", System.currentTimeMillis() - time_0); 1578 } 1579 1580 /** 1581 * Process a Solr rollback operation. 1582 * @param workspaceName The workspace's name 1583 * @throws SolrServerException if there is an error on the server 1584 * @throws IOException if there is a communication error with the server 1585 */ 1586 public void rollback(String workspaceName) throws SolrServerException, IOException 1587 { 1588 UpdateResponse solrResponse = _getSolrClient(workspaceName).rollback(_solrClientProvider.getCollectionName(workspaceName)); 1589 int status = solrResponse.getStatus(); 1590 1591 if (status != 0) 1592 { 1593 throw new IOException("Ametys indexing: Solr commit operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 1594 } 1595 1596 getLogger().debug("Successful Solr commit operation during an Ametys indexing process."); 1597 } 1598 1599 /** 1600 * Launch a solr index optimization. 1601 * @param workspaceName The workspace's name 1602 * @throws SolrServerException if there is an error on the server 1603 * @throws IOException if there is a communication error with the server 1604 */ 1605 public void optimize(String workspaceName) throws SolrServerException, IOException 1606 { 1607 UpdateResponse solrResponse = _getSolrClient(workspaceName).optimize(_solrClientProvider.getCollectionName(workspaceName)); 1608 int status = solrResponse.getStatus(); 1609 1610 if (status != 0) 1611 { 1612 throw new IOException("Ametys indexing: Solr optimize operation - Expecting status code of '0' in the Solr response but got : '" + status + "'."); 1613 } 1614 1615 getLogger().debug("Successful Solr optimize operation during an Ametys indexing process."); 1616 } 1617 1618 /** 1619 * Delete all documents from the solr index. 1620 * @param workspaceName The workspace name 1621 * @param commit true to commit 1622 * @throws Exception if an error occurs while unindexing. 1623 */ 1624 public void unindexAllDocuments(String workspaceName, boolean commit) throws Exception 1625 { 1626 getLogger().debug("Deleting all documents from Solr."); 1627 1628 String collection = _solrClientProvider.getCollectionName(workspaceName); 1629 _getSolrClient(workspaceName).deleteByQuery(collection, "*:*"); 1630 1631 if (commit) 1632 { 1633 commit(workspaceName); 1634 } 1635 1636 getLogger().debug("Successfully deleted all documents from Solr."); 1637 } 1638 1639 /** 1640 * Delete a document from the Solr server. 1641 * @param id The id of the document to delete from Solr 1642 * @param workspaceName The workspace name 1643 * @throws Exception if an error occurs while indexing. 1644 */ 1645 protected void doUnindexDocument(String id, String workspaceName) throws Exception 1646 { 1647 UpdateResponse solrResponse = _getSolrClient(workspaceName).deleteById(_solrClientProvider.getCollectionName(workspaceName), id); 1648 int status = solrResponse.getStatus(); 1649 1650 if (status != 0) 1651 { 1652 throw new IOException("Deletion of document " + id + ": got status code '" + status + "'."); 1653 } 1654 } 1655 1656// /** 1657// * Get the collection to use. 1658// * @return The name of the collection to index into. 1659// */ 1660// protected String getCollection() 1661// { 1662// return _solrCorePrefix + _workspaceSelector.getWorkspace(); 1663// } 1664 1665 class IndexationResult 1666 { 1667 protected int _successCount; 1668 1669 protected int _errorCount; 1670 1671 /** 1672 * Constructor 1673 */ 1674 public IndexationResult() 1675 { 1676 this(0, 0); 1677 } 1678 1679 /** 1680 * Constructor 1681 * @param successCount The success count. 1682 * @param errorCount The error count. 1683 */ 1684 public IndexationResult(int successCount, int errorCount) 1685 { 1686 this._successCount = successCount; 1687 this._errorCount = errorCount; 1688 } 1689 1690 /** 1691 * Get the successCount. 1692 * @return the successCount 1693 */ 1694 public int getSuccessCount() 1695 { 1696 return _successCount; 1697 } 1698 1699 /** 1700 * Set the successCount. 1701 * @param successCount the successCount to set 1702 */ 1703 public void setSuccessCount(int successCount) 1704 { 1705 this._successCount = successCount; 1706 } 1707 1708 /** 1709 * Test if the indexation had errors. 1710 * @return true if the indexation had errors, false otherwise. 1711 */ 1712 public boolean hasErrors() 1713 { 1714 return _errorCount > 0; 1715 } 1716 1717 /** 1718 * Get the errorCount. 1719 * @return the errorCount 1720 */ 1721 public int getErrorCount() 1722 { 1723 return _errorCount; 1724 } 1725 1726 /** 1727 * Set the errorCount. 1728 * @param errorCount the errorCount to set 1729 */ 1730 public void setErrorCount(int errorCount) 1731 { 1732 this._errorCount = errorCount; 1733 } 1734 } 1735 1736}