Autopsy  3.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
TermComponentQuery.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2014 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 //
20 package org.sleuthkit.autopsy.keywordsearch;
21 
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashSet;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.logging.Level;
29 import java.util.regex.Pattern;
30 import java.util.regex.PatternSyntaxException;
31 import org.apache.solr.client.solrj.SolrQuery;
32 import org.apache.solr.client.solrj.response.TermsResponse;
33 import org.apache.solr.client.solrj.response.TermsResponse.Term;
41 
45 class TermComponentQuery implements KeywordSearchQuery {
46 
47  private static final int TERMS_UNLIMITED = -1;
48  //corresponds to field in Solr schema, analyzed with white-space tokenizer only
49  private static final String TERMS_SEARCH_FIELD = Server.Schema.CONTENT_WS.toString();
50  private static final String TERMS_HANDLER = "/terms"; //NON-NLS
51  private static final int TERMS_TIMEOUT = 90 * 1000; //in ms
52  private static final Logger logger = Logger.getLogger(TermComponentQuery.class.getName());
53  private String queryEscaped;
54  private final KeywordList keywordList;
55  private final Keyword keyword;
56  private boolean isEscaped;
57  private List<Term> terms;
58  private final List<KeywordQueryFilter> filters = new ArrayList<>();
59  private String field;
60  private static final int MAX_TERMS_RESULTS = 20000;
61 
62  private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
63 
64  public TermComponentQuery(KeywordList keywordList, Keyword keyword) {
65  this.field = null;
66  this.keyword = keyword;
67  this.keywordList = keywordList;
68  this.queryEscaped = keyword.getQuery();
69  isEscaped = false;
70  terms = null;
71  }
72 
73  @Override
74  public void addFilter(KeywordQueryFilter filter) {
75  this.filters.add(filter);
76  }
77 
78  @Override
79  public void setField(String field) {
80  this.field = field;
81  }
82 
83  @Override
84  public void setSubstringQuery() {
85  queryEscaped = ".*" + queryEscaped + ".*";
86  }
87 
88  @Override
89  public void escape() {
90  queryEscaped = Pattern.quote(keyword.getQuery());
91  isEscaped = true;
92  }
93 
94  @Override
95  public boolean validate() {
96  if (queryEscaped.equals("")) {
97  return false;
98  }
99 
100  boolean valid = true;
101  try {
102  Pattern.compile(queryEscaped);
103  } catch (PatternSyntaxException ex1) {
104  valid = false;
105  } catch (IllegalArgumentException ex2) {
106  valid = false;
107  }
108  return valid;
109  }
110 
111  @Override
112  public boolean isEscaped() {
113  return isEscaped;
114  }
115 
116  @Override
117  public boolean isLiteral() {
118  return false;
119  }
120 
121  /*
122  * helper method to create a Solr terms component query
123  */
124  protected SolrQuery createQuery() {
125  final SolrQuery q = new SolrQuery();
126  q.setRequestHandler(TERMS_HANDLER);
127  q.setTerms(true);
128  q.setTermsLimit(TERMS_UNLIMITED);
129  q.setTermsRegexFlag("case_insensitive"); //NON-NLS
130  //q.setTermsLimit(200);
131  //q.setTermsRegexFlag(regexFlag);
132  //q.setTermsRaw(true);
133  q.setTermsRegex(queryEscaped);
134  q.addTermsField(TERMS_SEARCH_FIELD);
135  q.setTimeAllowed(TERMS_TIMEOUT);
136 
137  return q;
138 
139  }
140 
141  /*
142  * execute query and return terms, helper method
143  */
144  protected List<Term> executeQuery(SolrQuery q) throws NoOpenCoreException {
145  try {
146  Server solrServer = KeywordSearch.getServer();
147  TermsResponse tr = solrServer.queryTerms(q);
148  List<Term> termsCol = tr.getTerms(TERMS_SEARCH_FIELD);
149  return termsCol;
150  } catch (KeywordSearchModuleException ex) {
151  logger.log(Level.WARNING, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS
152  return null; //no need to create result view, just display error dialog
153  }
154  }
155 
156  @Override
157  public String getEscapedQueryString() {
158  return this.queryEscaped;
159  }
160 
161  @Override
162  public String getQueryString() {
163  return keyword.getQuery();
164  }
165 
166 
167 
168  @Override
169  public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) {
170  final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
171 
172  //there is match actually in this file, create artifact only then
173  BlackboardArtifact bba;
174  KeywordCachedArtifact writeResult;
175  Collection<BlackboardAttribute> attributes = new ArrayList<>();
176  try {
177  bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
178  writeResult = new KeywordCachedArtifact(bba);
179  } catch (Exception e) {
180  logger.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS
181  return null;
182  }
183 
184  //regex match
185  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID(), MODULE_NAME, termHit));
186 
187  if ((listName != null) && (listName.equals("") == false)) {
188  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, listName));
189  }
190 
191  //preview
192  if (snippet != null) {
193  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID(), MODULE_NAME, snippet));
194  }
195  //regex keyword
196  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID(), MODULE_NAME, keyword.getQuery()));
197 
198  if (hit.isArtifactHit()) {
199  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT.getTypeID(), MODULE_NAME, hit.getArtifact().getArtifactID()));
200  }
201 
202  try {
203  bba.addAttributes(attributes);
204  writeResult.add(attributes);
205  return writeResult;
206  } catch (TskException e) {
207  logger.log(Level.WARNING, "Error adding bb attributes for terms search artifact", e); //NON-NLS
208  }
209 
210  return null;
211  }
212 
213  @Override
214  public QueryResults performQuery() throws NoOpenCoreException {
215 
216  final SolrQuery q = createQuery();
217  q.setShowDebugInfo(DEBUG);
218  q.setTermsLimit(MAX_TERMS_RESULTS);
219  logger.log(Level.INFO, "Query: {0}", q.toString()); //NON-NLS
220  terms = executeQuery(q);
221 
222  QueryResults results = new QueryResults(this, keywordList);
223  int resultSize = 0;
224 
225  for (Term term : terms) {
226  final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
227 
228  LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(termStr, true));
229 
230  //filesQuery.setField(TERMS_SEARCH_FIELD);
231  for (KeywordQueryFilter filter : filters) {
232  //set filter
233  //note: we can't set filter query on terms query
234  //but setting filter query on terms results query will yield the same result
235  filesQuery.addFilter(filter);
236  }
237  try {
238  QueryResults subResults = filesQuery.performQuery();
239  Set<KeywordHit> filesResults = new HashSet<>();
240  for (Keyword key : subResults.getKeywords()) {
241  List<KeywordHit> keyRes = subResults.getResults(key);
242  resultSize += keyRes.size();
243  filesResults.addAll(keyRes);
244  }
245  results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(filesResults));
246  } catch (NoOpenCoreException e) {
247  logger.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS
248  throw e;
249  } catch (RuntimeException e) {
250  logger.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS
251  }
252 
253  }
254 
255  //TODO limit how many results we store, not to hit memory limits
256  logger.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS
257 
258  return results;
259  }
260 
261  @Override
262  public KeywordList getKeywordList() {
263  return keywordList;
264  }
265 }

Copyright © 2012-2015 Basis Technology. Generated on: Mon Oct 19 2015
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.