Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
SolrSearchService.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2015-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.Reader;
24 import java.net.InetAddress;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.MissingResourceException;
28 import java.util.logging.Level;
29 import org.apache.solr.client.solrj.SolrServerException;
30 import org.openide.util.NbBundle;
31 import org.openide.util.lookup.ServiceProvider;
32 import org.openide.util.lookup.ServiceProviders;
44 import org.sleuthkit.datamodel.BlackboardArtifact;
45 import org.sleuthkit.datamodel.Content;
46 import org.sleuthkit.datamodel.TskCoreException;
47 
56 @ServiceProviders(value = {
57  @ServiceProvider(service = KeywordSearchService.class),
58  @ServiceProvider(service = AutopsyService.class)
59 })
61 
62  private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
63  private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
64  private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
65  private static final Logger logger = Logger.getLogger(SolrSearchService.class.getName());
66 
79  @Override
80  public void index(Content content) throws TskCoreException {
81  if (content == null) {
82  return;
83  }
84  final Ingester ingester = Ingester.getDefault();
85  if (content instanceof BlackboardArtifact) {
86  BlackboardArtifact artifact = (BlackboardArtifact) content;
87  if (artifact.getArtifactID() > 0) {
88  /*
89  * Artifact indexing is only supported for artifacts that use
90  * negative artifact ids to avoid overlapping with the object
91  * ids of other types of Content.
92  */
93  return;
94  }
95  try {
96  Reader blackboardExtractedTextReader = KeywordSearchUtil.getReader(content);
97  String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
98  ingester.indexMetaDataOnly(artifact, sourceName);
99  // Will not cause an inline search becauce the keyword list is null
100  ingester.search(blackboardExtractedTextReader, artifact.getArtifactID(), sourceName, content, null, true, true, null);
101  } catch (Exception ex) {
102  throw new TskCoreException("Error indexing artifact", ex);
103  }
104  } else {
105  try {
106 
107  Reader reader = KeywordSearchUtil.getReader(content);
108  // Will not cause an inline search becauce the keyword list is null
109  ingester.search(reader, content.getId(), content.getName(), content, null, true, true, null);
110  } catch (Exception ex) {
111  throw new TskCoreException("Error indexing content", ex);
112  }
113  // only do a Solr commit if ingest is not running. If ingest is running, the changes will
114  // be committed via a periodic commit or via final commit after the ingest job has finished.
116  ingester.commit();
117  }
118  }
119  }
120 
129  @Override
130  public void tryConnect(String host, int port) throws KeywordSearchServiceException {
131  if (host == null || host.isEmpty()) {
132  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.MissingHostname")); //NON-NLS
133  }
134  try {
135  KeywordSearch.getServer().connectToSolrServer(host, Integer.toString(port));
136  } catch (SolrServerException ex) {
137  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
138  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort")); //NON-NLS*/
139  } catch (IOException ex) {
140  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
141  String result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
142  String message = ex.getCause().getMessage().toLowerCase();
143  if (message.startsWith(SERVER_REFUSED_CONNECTION)) {
144  try {
145  if (InetAddress.getByName(host).isReachable(IS_REACHABLE_TIMEOUT_MS)) {
146  // if we can reach the host, then it's probably port problem
147  result = Bundle.SolrConnectionCheck_Port();
148  } else {
149  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
150  }
151  } catch (IOException | MissingResourceException any) {
152  // it may be anything
153  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
154  }
155  } else if (message.startsWith(BAD_IP_ADDRESS_FORMAT)) {
156  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.Hostname"); //NON-NLS
157  }
158  throw new KeywordSearchServiceException(result);
159  } catch (NumberFormatException ex) {
160  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
161  throw new KeywordSearchServiceException(Bundle.SolrConnectionCheck_Port());
162  } catch (IllegalArgumentException ex) {
163  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
164  throw new KeywordSearchServiceException(ex.getMessage());
165  }
166  }
167 
176  @Override
177  public void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
178 
179  try {
180  Server ddsServer = KeywordSearch.getServer();
181  ddsServer.deleteDataSource(dataSourceId);
182  } catch (IOException | KeywordSearchModuleException | NoOpenCoreException | SolrServerException ex) {
183  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
184  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
185  }
186  }
187 
193  @NbBundle.Messages({
194  "# {0} - case directory", "SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0}",
195  "SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case",
196  "# {0} - collection name", "SolrSearchService.exceptionMessage.unableToDeleteCollection=Unable to delete collection {0}",
197  "# {0} - index folder path", "SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0}"
198  })
199  @Override
201  String caseDirectory = metadata.getCaseDirectory();
202  IndexMetadata indexMetadata;
203  try {
204  indexMetadata = new IndexMetadata(caseDirectory);
205  } catch (IndexMetadata.TextIndexMetadataException ex) {
206  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
207  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
208  }
209 
210  if (indexMetadata.getIndexes().isEmpty()) {
211  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class,
212  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
213  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class,
214  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
215  }
216 
217  // delete index(es) for this case
218  for (Index index : indexMetadata.getIndexes()) {
219  try {
220  // Unload/delete the collection on the server and then delete the text index files.
221  KeywordSearch.getServer().deleteCollection(index.getIndexName(), metadata);
222  } catch (KeywordSearchModuleException ex) {
223  throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_unableToDeleteCollection(index.getIndexName()), ex);
224  }
225  File indexDir = new File(index.getIndexPath()).getParentFile();
226  if (indexDir.exists()) {
227  if (!FileUtil.deleteDir(indexDir)) {
228  throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
229  }
230  }
231  }
232  }
233 
234  @Override
235  public String getServiceName() {
236  return NbBundle.getMessage(this.getClass(), "SolrSearchService.ServiceName");
237  }
238 
247  @Override
248  @NbBundle.Messages({
249  "SolrSearch.lookingForMetadata.msg=Looking for text index metadata file",
250  "SolrSearch.readingIndexes.msg=Reading text index metadata file",
251  "SolrSearch.findingIndexes.msg=Looking for existing text index directories",
252  "SolrSearch.creatingNewIndex.msg=Creating new text index",
253  "SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr and schema version",
254  "SolrSearch.indentifyingIndex.msg=Identifying text index to use",
255  "SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes.",
256  "# {0} - futureVersion", "# {1} - currentVersion",
257  "SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}.",
258  "SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case",
259  "SolrSearch.complete.msg=Text index successfully opened"})
261  if (context.cancelRequested()) {
262  return;
263  }
264 
265  ProgressIndicator progress = context.getProgressIndicator();
266  int totalNumProgressUnits = 7;
267  int progressUnitsCompleted = 0;
268 
269  String caseDirPath = context.getCase().getCaseDirectory();
270  Case theCase = context.getCase();
271  List<Index> indexes = new ArrayList<>();
272  progress.progress(Bundle.SolrSearch_lookingForMetadata_msg(), totalNumProgressUnits);
273  if (IndexMetadata.isMetadataFilePresent(caseDirPath)) {
274  try {
275  // metadata file exists, get list of existing Solr cores for this case
276  progressUnitsCompleted++;
277  progress.progress(Bundle.SolrSearch_findingIndexes_msg(), progressUnitsCompleted);
278  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath);
279  indexes = indexMetadata.getIndexes();
280  } catch (IndexMetadata.TextIndexMetadataException ex) {
281  logger.log(Level.SEVERE, String.format("Unable to read text index metadata file"), ex);
282  throw new AutopsyServiceException("Unable to read text index metadata file", ex);
283  }
284  }
285 
286  if (context.cancelRequested()) {
287  return;
288  }
289 
290  // check if we found any existing indexes
291  Index currentVersionIndex = null;
292  if (indexes.isEmpty()) {
293  // new case that doesn't have an existing index. create new index folder
294  progressUnitsCompleted++;
295  progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted);
296  currentVersionIndex = IndexFinder.createLatestVersionIndex(theCase);
297  // add current index to the list of indexes that exist for this case
298  indexes.add(currentVersionIndex);
299  } else {
300  // check if one of the existing indexes is for latest Solr version and schema
301  progressUnitsCompleted++;
302  progress.progress(Bundle.SolrSearch_checkingForLatestIndex_msg(), progressUnitsCompleted);
303  currentVersionIndex = IndexFinder.findLatestVersionIndex(indexes);
304  if (currentVersionIndex == null) {
305  // found existing index(es) but none were for latest Solr version and schema version
306  progressUnitsCompleted++;
307  progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted);
308  Index indexToUse = IndexFinder.identifyIndexToUse(indexes);
309  if (indexToUse == null) {
310  // unable to find index that can be used. check if the available index is for a "future" version of Solr,
311  // i.e. the user is using an "old/legacy" version of Autopsy to open cases created by later versions of Autopsy.
312  String futureIndexVersion = IndexFinder.isFutureIndexPresent(indexes);
313  if (!futureIndexVersion.isEmpty()) {
314  throw new AutopsyServiceException(Bundle.SolrSearch_futureIndexVersion_msg(futureIndexVersion, IndexFinder.getCurrentSolrVersion()));
315  }
316  throw new AutopsyServiceException(Bundle.SolrSearch_unableToFindIndex_msg());
317  }
318 
319  if (context.cancelRequested()) {
320  return;
321  }
322 
323  // check if schema is compatible
324  if (!indexToUse.isCompatible(IndexFinder.getCurrentSchemaVersion())) {
325  String msg = "Text index schema version " + indexToUse.getSchemaVersion() + " is not compatible with current schema";
326  logger.log(Level.WARNING, msg);
327  throw new AutopsyServiceException(msg);
328  }
329  // proceed with case open
330  currentVersionIndex = indexToUse;
331  }
332  }
333 
334  try {
335  // update text index metadata file
336  if (!indexes.isEmpty()) {
337  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath, indexes);
338  }
339  } catch (IndexMetadata.TextIndexMetadataException ex) {
340  throw new AutopsyServiceException("Failed to save Solr core info in text index metadata file", ex);
341  }
342 
343  // open core
344  try {
345  progress.progress(Bundle.SolrSearch_openCore_msg(), totalNumProgressUnits - 1);
346  KeywordSearch.getServer().openCoreForCase(theCase, currentVersionIndex);
347  } catch (KeywordSearchModuleException ex) {
348  throw new AutopsyServiceException(String.format("Failed to open or create core for %s", caseDirPath), ex);
349  }
350  if (context.cancelRequested()) {
351  return;
352  }
353 
354  theCase.getSleuthkitCase().registerForEvents(this);
355 
356  progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);
357  }
358 
367  @Override
369  /*
370  * TODO (JIRA 2525): The following code KeywordSearch.CaseChangeListener
371  * gambles that any BlackboardResultWriters (SwingWorkers) will complete
372  * in less than roughly two seconds. This stuff should be reworked using
373  * an ExecutorService and tasks with Futures.
374  */
375  AdHocSearchChildFactory.BlackboardResultWriter.stopAllWriters();
376  try {
377  Thread.sleep(2000);
378  } catch (InterruptedException ex) {
379  logger.log(Level.SEVERE, "Unexpected interrupt while waiting for BlackboardResultWriters to terminate", ex);
380  }
381 
382  try {
383  KeywordSearch.getServer().closeCore();
384  } catch (KeywordSearchModuleException ex) {
385  throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex);
386  }
387 
388  if (context.getCase().getSleuthkitCase() != null) {
389  context.getCase().getSleuthkitCase().unregisterForEvents(this);
390  }
391  }
392 
402  @Deprecated
403  @Override
404  public void indexArtifact(BlackboardArtifact artifact) throws TskCoreException {
405  if (artifact == null) {
406  return;
407  }
408 
409  // We only support artifact indexing for Autopsy versions that use
410  // the negative range for artifact ids.
411  if (artifact.getArtifactID() > 0) {
412  return;
413  }
414  final Ingester ingester = Ingester.getDefault();
415 
416  try {
417  String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
418  TextExtractor blackboardExtractor = TextExtractorFactory.getExtractor(artifact, null);
419  Reader blackboardExtractedTextReader = blackboardExtractor.getReader();
420  ingester.indexMetaDataOnly(artifact, sourceName);
421  ingester.search(blackboardExtractedTextReader, artifact.getId(), sourceName, artifact, null, true, true, null);
422  } catch (Exception ex) {
423  throw new TskCoreException(ex.getCause().getMessage(), ex);
424  }
425  }
426 }
static synchronized IngestManager getInstance()
static TextExtractor getExtractor(Content content, Lookup context)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static boolean deleteDir(File dirPath)
Definition: FileUtil.java:47

Copyright © 2012-2022 Basis Technology. Generated on: Tue Jun 27 2023
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.