Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
SolrSearchService.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2015-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.Reader;
24 import java.net.InetAddress;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.MissingResourceException;
28 import java.util.logging.Level;
29 import org.apache.solr.client.solrj.SolrServerException;
30 import org.openide.util.NbBundle;
31 import org.openide.util.lookup.ServiceProvider;
32 import org.openide.util.lookup.ServiceProviders;
44 import org.sleuthkit.datamodel.BlackboardArtifact;
45 import org.sleuthkit.datamodel.Content;
46 import org.sleuthkit.datamodel.TskCoreException;
47 
56 @ServiceProviders(value = {
57  @ServiceProvider(service = KeywordSearchService.class),
58  @ServiceProvider(service = AutopsyService.class)
59 })
61 
62  private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
63  private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
64  private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
65  private static final Logger logger = Logger.getLogger(SolrSearchService.class.getName());
66 
79  @Override
80  public void index(Content content) throws TskCoreException {
81  if (content == null) {
82  return;
83  }
84  final Ingester ingester = Ingester.getDefault();
85  if (content instanceof BlackboardArtifact) {
86  BlackboardArtifact artifact = (BlackboardArtifact) content;
87  if (artifact.getArtifactID() > 0) {
88  /*
89  * Artifact indexing is only supported for artifacts that use
90  * negative artifact ids to avoid overlapping with the object
91  * ids of other types of Content.
92  */
93  return;
94  }
95  try {
96  TextExtractor blackboardExtractor = TextExtractorFactory.getExtractor(content, null);
97  Reader blackboardExtractedTextReader = blackboardExtractor.getReader();
98  String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
99  ingester.indexMetaDataOnly(artifact, sourceName);
100  ingester.indexText(blackboardExtractedTextReader, artifact.getArtifactID(), sourceName, content, null);
101  } catch (Ingester.IngesterException | TextExtractorFactory.NoTextExtractorFound | TextExtractor.InitReaderException ex) {
102  throw new TskCoreException("Error indexing artifact", ex);
103  }
104  } else {
105  try {
106  TextExtractor contentExtractor = TextExtractorFactory.getExtractor(content, null);
107  Reader contentExtractedTextReader = contentExtractor.getReader();
108  ingester.indexText(contentExtractedTextReader, content.getId(), content.getName(), content, null);
109  } catch (TextExtractorFactory.NoTextExtractorFound | Ingester.IngesterException | TextExtractor.InitReaderException ex) {
110  try {
111  // Try the StringsTextExtractor if Tika extractions fails.
112  TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(content, null);
113  Reader stringsExtractedTextReader = stringsExtractor.getReader();
114  ingester.indexStrings(stringsExtractedTextReader, content.getId(), content.getName(), content, null);
115  } catch (Ingester.IngesterException | TextExtractor.InitReaderException ex1) {
116  throw new TskCoreException("Error indexing content", ex1);
117  }
118  }
119  // only do a Solr commit if ingest is not running. If ingest is running, the changes will
120  // be committed via a periodic commit or via final commit after the ingest job has finished.
122  ingester.commit();
123  }
124  }
125  }
126 
135  @Override
136  public void tryConnect(String host, int port) throws KeywordSearchServiceException {
137  if (host == null || host.isEmpty()) {
138  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.MissingHostname")); //NON-NLS
139  }
140  try {
141  KeywordSearch.getServer().connectToSolrServer(host, Integer.toString(port));
142  } catch (SolrServerException ex) {
143  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
144  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort")); //NON-NLS*/
145  } catch (IOException ex) {
146  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
147  String result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
148  String message = ex.getCause().getMessage().toLowerCase();
149  if (message.startsWith(SERVER_REFUSED_CONNECTION)) {
150  try {
151  if (InetAddress.getByName(host).isReachable(IS_REACHABLE_TIMEOUT_MS)) {
152  // if we can reach the host, then it's probably port problem
153  result = Bundle.SolrConnectionCheck_Port();
154  } else {
155  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
156  }
157  } catch (IOException | MissingResourceException any) {
158  // it may be anything
159  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
160  }
161  } else if (message.startsWith(BAD_IP_ADDRESS_FORMAT)) {
162  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.Hostname"); //NON-NLS
163  }
164  throw new KeywordSearchServiceException(result);
165  } catch (NumberFormatException ex) {
166  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
167  throw new KeywordSearchServiceException(Bundle.SolrConnectionCheck_Port());
168  } catch (IllegalArgumentException ex) {
169  logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex);
170  throw new KeywordSearchServiceException(ex.getMessage());
171  }
172  }
173 
182  @Override
183  public void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
184 
185  try {
186  Server ddsServer = KeywordSearch.getServer();
187  ddsServer.deleteDataSource(dataSourceId);
188  } catch (IOException | KeywordSearchModuleException | NoOpenCoreException | SolrServerException ex) {
189  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
190  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
191  }
192  }
193 
199  @NbBundle.Messages({
200  "# {0} - case directory", "SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0}",
201  "SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case",
202  "# {0} - collection name", "SolrSearchService.exceptionMessage.unableToDeleteCollection=Unable to delete collection {0}",
203  "# {0} - index folder path", "SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0}"
204  })
205  @Override
207  String caseDirectory = metadata.getCaseDirectory();
208  IndexMetadata indexMetadata;
209  try {
210  indexMetadata = new IndexMetadata(caseDirectory);
211  } catch (IndexMetadata.TextIndexMetadataException ex) {
212  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
213  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
214  }
215 
216  if (indexMetadata.getIndexes().isEmpty()) {
217  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class,
218  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
219  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class,
220  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
221  }
222 
223  // delete index(es) for this case
224  for (Index index : indexMetadata.getIndexes()) {
225  try {
226  // Unload/delete the collection on the server and then delete the text index files.
227  KeywordSearch.getServer().deleteCollection(index.getIndexName(), metadata);
228  } catch (KeywordSearchModuleException ex) {
229  throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_unableToDeleteCollection(index.getIndexName()), ex);
230  }
231  File indexDir = new File(index.getIndexPath()).getParentFile();
232  if (indexDir.exists()) {
233  if (!FileUtil.deleteDir(indexDir)) {
234  throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
235  }
236  }
237  }
238  }
239 
240  @Override
241  public String getServiceName() {
242  return NbBundle.getMessage(this.getClass(), "SolrSearchService.ServiceName");
243  }
244 
253  @Override
254  @NbBundle.Messages({
255  "SolrSearch.lookingForMetadata.msg=Looking for text index metadata file",
256  "SolrSearch.readingIndexes.msg=Reading text index metadata file",
257  "SolrSearch.findingIndexes.msg=Looking for existing text index directories",
258  "SolrSearch.creatingNewIndex.msg=Creating new text index",
259  "SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr and schema version",
260  "SolrSearch.indentifyingIndex.msg=Identifying text index to use",
261  "SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes.",
262  "# {0} - futureVersion", "# {1} - currentVersion",
263  "SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}.",
264  "SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case",
265  "SolrSearch.complete.msg=Text index successfully opened"})
267  if (context.cancelRequested()) {
268  return;
269  }
270 
271  ProgressIndicator progress = context.getProgressIndicator();
272  int totalNumProgressUnits = 7;
273  int progressUnitsCompleted = 0;
274 
275  String caseDirPath = context.getCase().getCaseDirectory();
276  Case theCase = context.getCase();
277  List<Index> indexes = new ArrayList<>();
278  progress.progress(Bundle.SolrSearch_lookingForMetadata_msg(), totalNumProgressUnits);
279  if (IndexMetadata.isMetadataFilePresent(caseDirPath)) {
280  try {
281  // metadata file exists, get list of existing Solr cores for this case
282  progressUnitsCompleted++;
283  progress.progress(Bundle.SolrSearch_findingIndexes_msg(), progressUnitsCompleted);
284  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath);
285  indexes = indexMetadata.getIndexes();
286  } catch (IndexMetadata.TextIndexMetadataException ex) {
287  logger.log(Level.SEVERE, String.format("Unable to read text index metadata file"), ex);
288  throw new AutopsyServiceException("Unable to read text index metadata file", ex);
289  }
290  }
291 
292  if (context.cancelRequested()) {
293  return;
294  }
295 
296  // check if we found any existing indexes
297  Index currentVersionIndex = null;
298  if (indexes.isEmpty()) {
299  // new case that doesn't have an existing index. create new index folder
300  progressUnitsCompleted++;
301  progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted);
302  currentVersionIndex = IndexFinder.createLatestVersionIndex(theCase);
303  // add current index to the list of indexes that exist for this case
304  indexes.add(currentVersionIndex);
305  } else {
306  // check if one of the existing indexes is for latest Solr version and schema
307  progressUnitsCompleted++;
308  progress.progress(Bundle.SolrSearch_checkingForLatestIndex_msg(), progressUnitsCompleted);
309  currentVersionIndex = IndexFinder.findLatestVersionIndex(indexes);
310  if (currentVersionIndex == null) {
311  // found existing index(es) but none were for latest Solr version and schema version
312  progressUnitsCompleted++;
313  progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted);
314  Index indexToUse = IndexFinder.identifyIndexToUse(indexes);
315  if (indexToUse == null) {
316  // unable to find index that can be used. check if the available index is for a "future" version of Solr,
317  // i.e. the user is using an "old/legacy" version of Autopsy to open cases created by later versions of Autopsy.
318  String futureIndexVersion = IndexFinder.isFutureIndexPresent(indexes);
319  if (!futureIndexVersion.isEmpty()) {
320  throw new AutopsyServiceException(Bundle.SolrSearch_futureIndexVersion_msg(futureIndexVersion, IndexFinder.getCurrentSolrVersion()));
321  }
322  throw new AutopsyServiceException(Bundle.SolrSearch_unableToFindIndex_msg());
323  }
324 
325  if (context.cancelRequested()) {
326  return;
327  }
328 
329  // check if schema is compatible
330  if (!indexToUse.isCompatible(IndexFinder.getCurrentSchemaVersion())) {
331  String msg = "Text index schema version " + indexToUse.getSchemaVersion() + " is not compatible with current schema";
332  logger.log(Level.WARNING, msg);
333  throw new AutopsyServiceException(msg);
334  }
335  // proceed with case open
336  currentVersionIndex = indexToUse;
337  }
338  }
339 
340  try {
341  // update text index metadata file
342  if (!indexes.isEmpty()) {
343  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath, indexes);
344  }
345  } catch (IndexMetadata.TextIndexMetadataException ex) {
346  throw new AutopsyServiceException("Failed to save Solr core info in text index metadata file", ex);
347  }
348 
349  // open core
350  try {
351  progress.progress(Bundle.SolrSearch_openCore_msg(), totalNumProgressUnits - 1);
352  KeywordSearch.getServer().openCoreForCase(theCase, currentVersionIndex);
353  } catch (KeywordSearchModuleException ex) {
354  throw new AutopsyServiceException(String.format("Failed to open or create core for %s", caseDirPath), ex);
355  }
356  if (context.cancelRequested()) {
357  return;
358  }
359 
360  theCase.getSleuthkitCase().registerForEvents(this);
361 
362  progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);
363  }
364 
373  @Override
375  /*
376  * TODO (JIRA 2525): The following code KeywordSearch.CaseChangeListener
377  * gambles that any BlackboardResultWriters (SwingWorkers) will complete
378  * in less than roughly two seconds. This stuff should be reworked using
379  * an ExecutorService and tasks with Futures.
380  */
381  AdHocSearchChildFactory.BlackboardResultWriter.stopAllWriters();
382  try {
383  Thread.sleep(2000);
384  } catch (InterruptedException ex) {
385  logger.log(Level.SEVERE, "Unexpected interrupt while waiting for BlackboardResultWriters to terminate", ex);
386  }
387 
388  try {
389  KeywordSearch.getServer().closeCore();
390  } catch (KeywordSearchModuleException ex) {
391  throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex);
392  }
393 
394  if (context.getCase().getSleuthkitCase() != null) {
395  context.getCase().getSleuthkitCase().unregisterForEvents(this);
396  }
397  }
398 
408  @Deprecated
409  @Override
410  public void indexArtifact(BlackboardArtifact artifact) throws TskCoreException {
411  if (artifact == null) {
412  return;
413  }
414 
415  // We only support artifact indexing for Autopsy versions that use
416  // the negative range for artifact ids.
417  if (artifact.getArtifactID() > 0) {
418  return;
419  }
420  final Ingester ingester = Ingester.getDefault();
421 
422  try {
423  String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
424  TextExtractor blackboardExtractor = TextExtractorFactory.getExtractor((Content) artifact, null);
425  Reader blackboardExtractedTextReader = blackboardExtractor.getReader();
426  ingester.indexMetaDataOnly(artifact, sourceName);
427  ingester.indexText(blackboardExtractedTextReader, artifact.getId(), sourceName, artifact, null);
428  } catch (Ingester.IngesterException | TextExtractorFactory.NoTextExtractorFound | TextExtractor.InitReaderException ex) {
429  throw new TskCoreException(ex.getCause().getMessage(), ex);
430  }
431  }
432 }
static synchronized IngestManager getInstance()
static TextExtractor getStringsExtractor(Content content, Lookup context)
static TextExtractor getExtractor(Content content, Lookup context)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static boolean deleteDir(File dirPath)
Definition: FileUtil.java:47

Copyright © 2012-2022 Basis Technology. Generated on: Thu Sep 29 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.