Autopsy  3.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
SearchRunner.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011 - 2014 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 
20 package org.sleuthkit.autopsy.keywordsearch;
21 
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Map.Entry;
28 import java.util.concurrent.CancellationException;
29 import java.util.concurrent.ExecutionException;
30 import java.util.concurrent.atomic.AtomicLong;
31 import java.util.logging.Level;
32 import javax.swing.SwingUtilities;
33 import javax.swing.SwingWorker;
34 import java.util.Timer;
35 import java.util.TimerTask;
36 import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
37 import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
38 import org.netbeans.api.progress.aggregate.ProgressContributor;
39 import org.openide.util.Cancellable;
40 import org.openide.util.NbBundle;
46 
52 public final class SearchRunner {
53  private static final Logger logger = Logger.getLogger(SearchRunner.class.getName());
54  private static SearchRunner instance = null;
56  private Ingester ingester = null;
57  private volatile boolean updateTimerRunning = false;
58  private Timer updateTimer;
59 
60  // maps a jobID to the search
61  private Map<Long, SearchJobInfo> jobs = new HashMap<>(); //guarded by "this"
62 
63  SearchRunner() {
64  ingester = Server.getIngester();
65  updateTimer = new Timer(NbBundle.getMessage(this.getClass(), "SearchRunner.updateTimer.title.text"), true); // run as a daemon
66  }
67 
72  public static synchronized SearchRunner getInstance() {
73  if (instance == null) {
74  instance = new SearchRunner();
75  }
76  return instance;
77  }
78 
85  public synchronized void startJob(long jobId, long dataSourceId, List<String> keywordListNames) {
86  if (jobs.containsKey(jobId) == false) {
87  logger.log(Level.INFO, "Adding job {0}", jobId); //NON-NLS
88  SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames);
89  jobs.put(jobId, jobData);
90  }
91 
92  // keep track of how many threads / module instances from this job have asked for this
93  jobs.get(jobId).incrementModuleReferenceCount();
94 
95  // start the timer, if needed
96  if ((jobs.size() > 0) && (updateTimerRunning == false)) {
97  final long updateIntervalMs = ((long)KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000;
98  updateTimer.scheduleAtFixedRate(new UpdateTimerTask(), updateIntervalMs, updateIntervalMs);
99  updateTimerRunning = true;
100  }
101  }
102 
108  public void endJob(long jobId) {
109  SearchJobInfo job;
110  boolean readyForFinalSearch = false;
111  synchronized(this) {
112  job = jobs.get(jobId);
113  if (job == null) {
114  return;
115  }
116 
117  // Only do final search if this is the last module/thread in this job to call endJob()
118  if(job.decrementModuleReferenceCount() == 0) {
119  jobs.remove(jobId);
120  readyForFinalSearch = true;
121  }
122  }
123 
124  if (readyForFinalSearch) {
125  commit();
126  doFinalSearch(job); //this will block until it's done
127  }
128  }
129 
130 
136  public void stopJob(long jobId) {
137  logger.log(Level.INFO, "Stopping job {0}", jobId); //NON-NLS
138  commit();
139 
140  SearchJobInfo job;
141  synchronized(this) {
142  job = jobs.get(jobId);
143  if (job == null) {
144  return;
145  }
146 
147  //stop currentSearcher
148  SearchRunner.Searcher currentSearcher = job.getCurrentSearcher();
149  if ((currentSearcher != null) && (!currentSearcher.isDone())) {
150  currentSearcher.cancel(true);
151  }
152 
153  jobs.remove(jobId);
154  }
155  }
156 
161  public synchronized void addKeywordListsToAllJobs(List<String> keywordListNames) {
162  for(String listName : keywordListNames) {
163  logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); //NON-NLS
164  for(SearchJobInfo j : jobs.values()) {
165  j.addKeywordListName(listName);
166  }
167  }
168  }
169 
173  private void commit() {
174  ingester.commit();
175 
176  // Signal a potential change in number of text_ingested files
177  try {
178  final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles();
179  KeywordSearch.fireNumIndexedFilesChange(null, numIndexedFiles);
181  logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); //NON-NLS
182  }
183  }
184 
190  private void doFinalSearch(SearchJobInfo job) {
191  // Run one last search as there are probably some new files committed
192  logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); //NON-NLS
193  if (!job.getKeywordListNames().isEmpty()) {
194  try {
195  // In case this job still has a worker running, wait for it to finish
196  job.waitForCurrentWorker();
197 
198  SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true);
199  job.setCurrentSearcher(finalSearcher); //save the ref
200  finalSearcher.execute(); //start thread
201 
202  // block until the search is complete
203  finalSearcher.get();
204 
205  } catch (InterruptedException | ExecutionException ex) {
206  logger.log(Level.WARNING, "Job {1} final search thread failed: {2}", new Object[]{job.getJobId(), ex}); //NON-NLS
207  }
208  }
209  }
210 
211 
215  private class UpdateTimerTask extends TimerTask {
216  private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerTask.class.getName());
217 
218  @Override
219  public void run() {
220  // If no jobs then cancel the task. If more job(s) come along, a new task will start up.
221  if (jobs.isEmpty()) {
222  this.cancel(); //terminate this timer task
223  updateTimerRunning = false;
224  return;
225  }
226 
227  commit();
228 
229  synchronized(SearchRunner.this) {
230  // Spawn a search thread for each job
231  for(Entry<Long, SearchJobInfo> j : jobs.entrySet()) {
232  SearchJobInfo job = j.getValue();
233  // If no lists or the worker is already running then skip it
234  if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) {
235  Searcher searcher = new Searcher(job);
236  job.setCurrentSearcher(searcher); //save the ref
237  searcher.execute(); //start thread
238  job.setWorkerRunning(true);
239  }
240  }
241  }
242  }
243  }
244 
249  private class SearchJobInfo {
250  private final long jobId;
251  private final long dataSourceId;
252  // mutable state:
253  private volatile boolean workerRunning;
254  private List<String> keywordListNames; //guarded by SearchJobInfo.this
255  private Map<Keyword, List<Long>> currentResults; //guarded by SearchJobInfo.this
257  private AtomicLong moduleReferenceCount = new AtomicLong(0);
258  private final Object finalSearchLock = new Object(); //used for a condition wait
259 
260  public SearchJobInfo(long jobId, long dataSourceId, List<String> keywordListNames) {
261  this.jobId = jobId;
262  this.dataSourceId = dataSourceId;
263  this.keywordListNames = new ArrayList<>(keywordListNames);
264  currentResults = new HashMap<>();
265  workerRunning = false;
266  currentSearcher = null;
267  }
268 
269  public long getJobId() {
270  return jobId;
271  }
272 
273  public long getDataSourceId() {
274  return dataSourceId;
275  }
276 
277  public synchronized List<String> getKeywordListNames() {
278  return new ArrayList<>(keywordListNames);
279  }
280 
281  public synchronized void addKeywordListName(String keywordListName) {
282  if (!keywordListNames.contains(keywordListName)) {
283  keywordListNames.add(keywordListName);
284  }
285  }
286 
287  public synchronized List<Long> currentKeywordResults(Keyword k) {
288  return currentResults.get(k);
289  }
290 
291  public synchronized void addKeywordResults(Keyword k, List<Long> resultsIDs) {
292  currentResults.put(k, resultsIDs);
293  }
294 
295  public boolean isWorkerRunning() {
296  return workerRunning;
297  }
298 
299  public void setWorkerRunning(boolean flag) {
300  workerRunning = flag;
301  }
302 
303  public synchronized SearchRunner.Searcher getCurrentSearcher() {
304  return currentSearcher;
305  }
306 
307  public synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) {
308  currentSearcher = searchRunner;
309  }
310 
312  moduleReferenceCount.incrementAndGet();
313  }
314 
316  return moduleReferenceCount.decrementAndGet();
317  }
318 
323  public void waitForCurrentWorker() throws InterruptedException {
324  synchronized(finalSearchLock) {
325  while(workerRunning) {
326  finalSearchLock.wait(); //wait() releases the lock
327  }
328  }
329  }
330 
334  public void searchNotify() {
335  synchronized(finalSearchLock) {
336  workerRunning = false;
337  finalSearchLock.notify();
338  }
339  }
340  }
341 
348  private final class Searcher extends SwingWorker<Object, Void> {
349 
354  private List<Keyword> keywords; //keywords to search
355  private List<String> keywordListNames; // lists currently being searched
356  private List<KeywordList> keywordLists;
357  private Map<String, KeywordList> keywordToList; //keyword to list name mapping
358  private AggregateProgressHandle progressGroup;
359  private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName());
360  private boolean finalRun = false;
361 
362  Searcher(SearchJobInfo job) {
363  this.job = job;
364  keywordListNames = job.getKeywordListNames();
365  keywords = new ArrayList<>();
366  keywordToList = new HashMap<>();
367  keywordLists = new ArrayList<>();
368  //keywords are populated as searcher runs
369  }
370 
371  Searcher(SearchJobInfo job, boolean finalRun) {
372  this(job);
373  this.finalRun = finalRun;
374  }
375 
376  @Override
377  protected Object doInBackground() throws Exception {
378  final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName")
379  + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : "");
380  final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")");
381  progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() {
382  @Override
383  public boolean cancel() {
384  logger.log(Level.INFO, "Cancelling the searcher by user."); //NON-NLS
385  if (progressGroup != null) {
386  progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg"));
387  }
388  return SearchRunner.Searcher.this.cancel(true);
389  }
390  }, null);
391 
392  updateKeywords();
393 
394  ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()];
395  int i = 0;
396  for (Keyword keywordQuery : keywords) {
397  subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery());
398  progressGroup.addContributor(subProgresses[i]);
399  i++;
400  }
401 
402  progressGroup.start();
403 
404  final StopWatch stopWatch = new StopWatch();
405  stopWatch.start();
406  try {
407  progressGroup.setDisplayName(displayName);
408 
409  int keywordsSearched = 0;
410 
411  for (Keyword keywordQuery : keywords) {
412  if (this.isCancelled()) {
413  logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); //NON-NLS
414  return null;
415  }
416 
417  final String queryStr = keywordQuery.getQuery();
418  final KeywordList list = keywordToList.get(queryStr);
419 
420  //new subProgress will be active after the initial query
421  //when we know number of hits to start() with
422  if (keywordsSearched > 0) {
423  subProgresses[keywordsSearched - 1].finish();
424  }
425 
426  KeywordSearchQuery keywordSearchQuery = null;
427 
428  boolean isRegex = !keywordQuery.isLiteral();
429  if (isRegex) {
430  keywordSearchQuery = new TermComponentQuery(list, keywordQuery);
431  } else {
432  keywordSearchQuery = new LuceneQuery(list, keywordQuery);
433  keywordSearchQuery.escape();
434  }
435 
436  // Filtering
437  //limit search to currently ingested data sources
438  //set up a filter with 1 or more image ids OR'ed
439  final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId());
440  keywordSearchQuery.addFilter(dataSourceFilter);
441 
442  QueryResults queryResults;
443 
444  // Do the actual search
445  try {
446  queryResults = keywordSearchQuery.performQuery();
447  } catch (NoOpenCoreException ex) {
448  logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); //NON-NLS
449  //no reason to continue with next query if recovery failed
450  //or wait for recovery to kick in and run again later
451  //likely case has closed and threads are being interrupted
452  return null;
453  } catch (CancellationException e) {
454  logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery()); //NON-NLS
455  return null;
456  } catch (Exception e) {
457  logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); //NON-NLS
458  continue;
459  }
460 
461  // calculate new results by substracting results already obtained in this ingest
462  // this creates a map of each keyword to the list of unique files that have that hit.
463  QueryResults newResults = filterResults(queryResults);
464 
465  if (!newResults.getKeywords().isEmpty()) {
466 
467  // Write results to BB
468 
469  //new artifacts created, to report to listeners
470  Collection<BlackboardArtifact> newArtifacts = new ArrayList<>();
471 
472  //scale progress bar more more granular, per result sub-progress, within per keyword
473  int totalUnits = newResults.getKeywords().size();
474  subProgresses[keywordsSearched].start(totalUnits);
475  int unitProgress = 0;
476  String queryDisplayStr = keywordQuery.getQuery();
477  if (queryDisplayStr.length() > 50) {
478  queryDisplayStr = queryDisplayStr.substring(0, 49) + "...";
479  }
480  subProgresses[keywordsSearched].progress(list.getName() + ": " + queryDisplayStr, unitProgress);
481 
482  // Create blackboard artifacts
483  newArtifacts = newResults.writeAllHitsToBlackBoard(null, subProgresses[keywordsSearched], this, list.getIngestMessages());
484 
485  } //if has results
486 
487  //reset the status text before it goes away
488  subProgresses[keywordsSearched].progress("");
489 
490  ++keywordsSearched;
491 
492  } //for each keyword
493 
494  } //end try block
495  catch (Exception ex) {
496  logger.log(Level.WARNING, "searcher exception occurred", ex); //NON-NLS
497  } finally {
498  try {
500  stopWatch.stop();
501 
502  logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); //NON-NLS
503  } finally {
504  // In case a thread is waiting on this worker to be done
505  job.searchNotify();
506  }
507  }
508 
509  return null;
510  }
511 
512  @Override
513  protected void done() {
514  // call get to see if there were any errors
515  try {
516  get();
517  } catch (InterruptedException | ExecutionException e) {
518  logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); //NON-NLS
520  NbBundle.getMessage(this.getClass(),
521  "SearchRunner.Searcher.done.err.msg"), e.getMessage()));
522  } // catch and ignore if we were cancelled
523  catch (java.util.concurrent.CancellationException ex) {
524  }
525  }
526 
530  private void updateKeywords() {
531  XmlKeywordSearchList loader = XmlKeywordSearchList.getCurrent();
532 
533  keywords.clear();
534  keywordToList.clear();
535  keywordLists.clear();
536 
537  for (String name : keywordListNames) {
538  KeywordList list = loader.getList(name);
539  keywordLists.add(list);
540  for (Keyword k : list.getKeywords()) {
541  keywords.add(k);
542  keywordToList.put(k.getQuery(), list);
543  }
544  }
545  }
546 
552  private void finalizeSearcher() {
553  SwingUtilities.invokeLater(new Runnable() {
554  @Override
555  public void run() {
556  progressGroup.finish();
557  }
558  });
559  }
560 
561  //calculate new results but substracting results already obtained in this ingest
562  //update currentResults map with the new results
563  private QueryResults filterResults(QueryResults queryResult) {
564 
565  QueryResults newResults = new QueryResults(queryResult.getQuery(), queryResult.getKeywordList());
566 
567  for (Keyword keyword : queryResult.getKeywords()) {
568  List<KeywordHit> queryTermResults = queryResult.getResults(keyword);
569 
570  //translate to list of IDs that we keep track of
571  List<Long> queryTermResultsIDs = new ArrayList<>();
572  for (KeywordHit ch : queryTermResults) {
573  queryTermResultsIDs.add(ch.getSolrObjectId());
574  }
575 
576  List<Long> curTermResults = job.currentKeywordResults(keyword);
577  if (curTermResults == null) {
578  job.addKeywordResults(keyword, queryTermResultsIDs);
579  newResults.addResult(keyword, queryTermResults);
580  } else {
581  //some AbstractFile hits already exist for this keyword
582  for (KeywordHit res : queryTermResults) {
583  if (!curTermResults.contains(res.getSolrObjectId())) {
584  //add to new results
585  List<KeywordHit> newResultsFs = newResults.getResults(keyword);
586  if (newResultsFs == null) {
587  newResultsFs = new ArrayList<>();
588  newResults.addResult(keyword, newResultsFs);
589  }
590  newResultsFs.add(res);
591  curTermResults.add(res.getSolrObjectId());
592  }
593  }
594  }
595  }
596 
597  return newResults;
598  }
599  }
600 }
SearchJobInfo(long jobId, long dataSourceId, List< String > keywordListNames)
synchronized void addKeywordResults(Keyword k, List< Long > resultsIDs)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
static void fireNumIndexedFilesChange(Integer oldNum, Integer newNum)
synchronized void addKeywordListName(String keywordListName)
synchronized void startJob(long jobId, long dataSourceId, List< String > keywordListNames)
synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner)
static synchronized SearchRunner getInstance()
synchronized void addKeywordListsToAllJobs(List< String > keywordListNames)
void postMessage(final IngestMessage message)
QueryResults filterResults(QueryResults queryResult)
static Logger getLogger(String name)
Definition: Logger.java:131
static synchronized IngestServices getInstance()

Copyright © 2012-2015 Basis Technology. Generated on: Mon Oct 19 2015
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.