Autopsy  4.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
KeywordSearchResultFactory.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2015 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this content except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.awt.EventQueue;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.LinkedHashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ExecutionException;
31 import java.util.concurrent.locks.ReentrantReadWriteLock;
32 import java.util.logging.Level;
33 import org.openide.util.NbBundle;
35 import javax.swing.SwingWorker;
36 import org.netbeans.api.progress.ProgressHandle;
37 import org.netbeans.api.progress.ProgressHandleFactory;
38 import org.openide.nodes.ChildFactory;
39 import org.openide.nodes.Children;
40 import org.openide.nodes.Node;
41 import org.openide.util.Cancellable;
42 import org.openide.util.lookup.Lookups;
49 import org.sleuthkit.datamodel.AbstractFile;
50 import org.sleuthkit.datamodel.BlackboardArtifact;
51 import org.sleuthkit.datamodel.BlackboardAttribute;
52 import org.sleuthkit.datamodel.Content;
53 
61 class KeywordSearchResultFactory extends ChildFactory<KeyValueQueryContent> {
62 
63  //common properties (superset of all Node properties) to be displayed as columns
64  //these are merged with FsContentPropertyType defined properties
65  public static enum CommonPropertyTypes {
66 
67  KEYWORD {
68  @Override
69  public String toString() {
70  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getDisplayName();
71  }
72  },
73  REGEX {
74  @Override
75  public String toString() {
76  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getDisplayName();
77  }
78  },
79  CONTEXT {
80  @Override
81  public String toString() {
82  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getDisplayName();
83  }
84  },
85  }
86  private Collection<QueryRequest> queryRequests;
87  private final DataResultTopComponent viewer; //viewer driving this child node factory
88  private static final Logger logger = Logger.getLogger(KeywordSearchResultFactory.class.getName());
89 
90  KeywordSearchResultFactory(Collection<QueryRequest> queryRequests, DataResultTopComponent viewer) {
91  this.queryRequests = queryRequests;
92  this.viewer = viewer;
93  }
94 
102  public static void initCommonProperties(Map<String, Object> toSet) {
103  CommonPropertyTypes[] commonTypes = CommonPropertyTypes.values();
104  final int COMMON_PROPS_LEN = commonTypes.length;
105  for (int i = 0; i < COMMON_PROPS_LEN; ++i) {
106  toSet.put(commonTypes[i].toString(), "");
107  }
108 
109  AbstractAbstractFileNode.AbstractFilePropertyType[] fsTypes = AbstractAbstractFileNode.AbstractFilePropertyType.values();
110  final int FS_PROPS_LEN = fsTypes.length;
111  for (int i = 0; i < FS_PROPS_LEN; ++i) {
112  toSet.put(fsTypes[i].toString(), "");
113  }
114  }
115 
116  public static void setCommonProperty(Map<String, Object> toSet, CommonPropertyTypes type, String value) {
117  final String typeStr = type.toString();
118  toSet.put(typeStr, value);
119  }
120 
121  public static void setCommonProperty(Map<String, Object> toSet, CommonPropertyTypes type, Boolean value) {
122  final String typeStr = type.toString();
123  toSet.put(typeStr, value);
124  }
125 
126  @Override
127  protected boolean createKeys(List<KeyValueQueryContent> toPopulate) {
128 
129  for (QueryRequest queryRequest : queryRequests) {
130  Map<String, Object> map = queryRequest.getProperties();
131  initCommonProperties(map);
132  final String query = queryRequest.getQueryString();
133  setCommonProperty(map, CommonPropertyTypes.KEYWORD, query);
134  setCommonProperty(map, CommonPropertyTypes.REGEX, !queryRequest.getQuery().isLiteral());
135  createFlatKeys(queryRequest, toPopulate);
136  }
137 
138  return true;
139  }
140 
148  private boolean createFlatKeys(QueryRequest queryRequest, List<KeyValueQueryContent> toPopulate) {
152  final KeywordSearchQuery keywordSearchQuery = queryRequest.getQuery();
153  if (!keywordSearchQuery.validate()) {
154  //TODO mark the particular query node RED
155  return false;
156  }
157 
161  QueryResults queryResults;
162  try {
163  queryResults = keywordSearchQuery.performQuery();
164  } catch (NoOpenCoreException ex) {
165  logger.log(Level.SEVERE, "Could not perform the query " + keywordSearchQuery.getQueryString(), ex); //NON-NLS
166  return false;
167  }
168 
169  int id = 0;
170  List<KeyValueQueryContent> tempList = new ArrayList<>();
171  for (KeywordHit hit : getOneHitPerObject(queryResults)) {
175  Map<String, Object> properties = new LinkedHashMap<>();
176  Content content = hit.getContent();
177  if (content instanceof AbstractFile) {
178  AbstractFsContentNode.fillPropertyMap(properties, (AbstractFile) content);
179  } else {
180  properties.put(AbstractAbstractFileNode.AbstractFilePropertyType.LOCATION.toString(), content.getName());
181  }
182 
186  if (hit.hasSnippet()) {
187  setCommonProperty(properties, CommonPropertyTypes.CONTEXT, hit.getSnippet());
188  }
189 
190  //@@@ USE ConentHit in UniqueFileMap instead of the below search
191  //get unique match result files
192  // BC: @@@ THis is really ineffecient. We should keep track of this when
193  // we flattened the list of files to the unique files.
194  final String highlightQueryEscaped = getHighlightQuery(keywordSearchQuery, keywordSearchQuery.isLiteral(), queryResults, content);
195 
196  String name = content.getName();
197  if (hit.isArtifactHit()) {
198  name = hit.getArtifact().getDisplayName() + " Artifact"; // NON-NLS
199  }
200  tempList.add(new KeyValueQueryContent(name, properties, ++id, hit.getSolrObjectId(), content, highlightQueryEscaped, keywordSearchQuery, queryResults));
201  }
202 
203  // Add all the nodes to toPopulate at once. Minimizes node creation
204  // EDT threads, which can slow and/or hang the UI on large queries.
205  toPopulate.addAll(tempList);
206 
207  //write to bb
208  //cannot reuse snippet in BlackboardResultWriter
209  //because for regex searches in UI we compress results by showing a content per regex once (even if multiple term hits)
210  //whereas in bb we write every hit per content separately
211  new BlackboardResultWriter(queryResults, queryRequest.getQuery().getKeywordList().getName()).execute();
212 
213  return true;
214  }
215 
225  Collection<KeywordHit> getOneHitPerObject(QueryResults queryResults) {
226  HashMap<Long, KeywordHit> hits = new HashMap<Long, KeywordHit>();
227  for (Keyword keyWord : queryResults.getKeywords()) {
228  for (KeywordHit hit : queryResults.getResults(keyWord)) {
229  // add hit with lowest SolrObjectID-Chunk-ID combination.
230  if (!hits.containsKey(hit.getSolrObjectId())) {
231  hits.put(hit.getSolrObjectId(), hit);
232  } else {
233  if (hit.getChunkId() < hits.get(hit.getSolrObjectId()).getChunkId()) {
234  hits.put(hit.getSolrObjectId(), hit);
235  }
236  }
237  }
238  }
239  return hits.values();
240  }
241 
252  private String getHighlightQuery(KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) {
253  String highlightQueryEscaped;
254  if (literal_query) {
255  //literal, treat as non-regex, non-term component query
256  highlightQueryEscaped = query.getQueryString();
257  } else {
258  //construct a Solr query using aggregated terms to get highlighting
259  //the query is executed later on demand
260  StringBuilder highlightQuery = new StringBuilder();
261 
262  if (queryResults.getKeywords().size() == 1) {
263  //simple case, no need to process subqueries and do special escaping
264  Keyword term = queryResults.getKeywords().iterator().next();
265  highlightQuery.append(term.toString());
266  } else {
267  //find terms for this content hit
268  List<String> hitTerms = new ArrayList<>();
269  for (Keyword keyword : queryResults.getKeywords()) {
270  for (KeywordHit hit : queryResults.getResults(keyword)) {
271  if (hit.getContent().equals(content)) {
272  hitTerms.add(keyword.toString());
273  break; //go to next term
274  }
275  }
276  }
277 
278  final int lastTerm = hitTerms.size() - 1;
279  int curTerm = 0;
280  for (String term : hitTerms) {
281  //escape subqueries, they shouldn't be escaped again later
282  final String termS = KeywordSearchUtil.escapeLuceneQuery(term);
283  highlightQuery.append("\"");
284  highlightQuery.append(termS);
285  highlightQuery.append("\"");
286  if (lastTerm != curTerm) {
287  highlightQuery.append(" "); //acts as OR ||
288  //force HIGHLIGHT_FIELD_REGEX index and stored content
289  //in each term after first. First term taken care by HighlightedMatchesSource
290  highlightQuery.append(LuceneQuery.HIGHLIGHT_FIELD_REGEX).append(":");
291  }
292 
293  ++curTerm;
294  }
295  }
296  //String highlightQueryEscaped = KeywordSearchUtil.escapeLuceneQuery(highlightQuery.toString());
297  highlightQueryEscaped = highlightQuery.toString();
298  }
299 
300  return highlightQueryEscaped;
301  }
302 
303  @Override
304  protected Node createNodeForKey(KeyValueQueryContent key) {
305  final Content content = key.getContent();
306  final String queryStr = key.getQueryStr();;
307  QueryResults hits = key.getHits();
308 
309  Node kvNode = new KeyValueNode(key, Children.LEAF, Lookups.singleton(content));
310 
311  //wrap in KeywordSearchFilterNode for the markup content, might need to override FilterNode for more customization
312  // store the data in HighlightedMatchesSource so that it can be looked up (in content viewer)
313  HighlightedText highlights = new HighlightedText(key.solrObjectId, queryStr, !key.getQuery().isLiteral(), false, hits);
314  return new KeywordSearchFilterNode(highlights, kvNode);
315  }
316 
321  class KeyValueQueryContent extends KeyValue {
322 
323  private long solrObjectId;
324  private Content content;
325  private String queryStr;
326  private QueryResults hits;
327  private KeywordSearchQuery query;
328 
344  public KeyValueQueryContent(String name, Map<String, Object> map, int id, long solrObjectId, Content content, String queryStr, KeywordSearchQuery query, QueryResults hits) {
345  super(name, map, id);
346  this.solrObjectId = solrObjectId;
347  this.content = content;
348  this.queryStr = queryStr;
349  this.hits = hits;
350  this.query = query;
351  }
352 
353  Content getContent() {
354  return content;
355  }
356 
357  String getQueryStr() {
358  return queryStr;
359  }
360 
361  QueryResults getHits() {
362  return hits;
363  }
364 
365  KeywordSearchQuery getQuery() {
366  return query;
367  }
368  }
369 
374  static class BlackboardResultWriter extends SwingWorker<Object, Void> {
375 
376  private static List<BlackboardResultWriter> writers = new ArrayList<>();
377  //lock utilized to enqueue writers and limit execution to 1 at a time
378  private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy
379  //private static final Lock writerLock = rwLock.writeLock();
380  private ProgressHandle progress;
381  private KeywordSearchQuery query;
382  private String listName;
383  private QueryResults hits;
384  private Collection<BlackboardArtifact> newArtifacts = new ArrayList<>();
385  private static final int QUERY_DISPLAY_LEN = 40;
386 
387  BlackboardResultWriter(QueryResults hits, String listName) {
388  this.hits = hits;
389  this.query = hits.getQuery();
390  this.listName = listName;
391  }
392 
393  protected void finalizeWorker() {
394  deregisterWriter(this);
395 
396  EventQueue.invokeLater(new Runnable() {
397  @Override
398  public void run() {
399  progress.finish();
400  }
401  });
402  }
403 
404  @Override
405  protected Object doInBackground() throws Exception {
406  registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock
407  final String queryStr = query.getQueryString();
408  final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr;
409  //block until previous writer is done
410  //writerLock.lock();
411 
412  try {
413  progress = ProgressHandleFactory.createHandle(
414  NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), new Cancellable() {
415  @Override
416  public boolean cancel() {
417  return BlackboardResultWriter.this.cancel(true);
418  }
419  });
420 
421  // Create blackboard artifacts
422  newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false);
423  } finally {
424  finalizeWorker();
425  }
426 
427  return null;
428  }
429 
430  @Override
431  protected void done() {
432  try {
433  // test if any exceptions were thrown
434  get();
435  } catch (InterruptedException | ExecutionException ex) {
436  logger.log(Level.SEVERE, "Error querying ", ex); //NON-NLS
437  }
438  }
439 
440  private static synchronized void registerWriter(BlackboardResultWriter writer) {
441  writers.add(writer);
442  }
443 
444  private static synchronized void deregisterWriter(BlackboardResultWriter writer) {
445  writers.remove(writer);
446  }
447 
448  static synchronized void stopAllWriters() {
449  for (BlackboardResultWriter w : writers) {
450  w.cancel(true);
451  writers.remove(w);
452  }
453  }
454  }
455 }
synchronized static Logger getLogger(String name)
Definition: Logger.java:166

Copyright © 2012-2015 Basis Technology. Generated on: Wed Apr 6 2016
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.