19 package org.sleuthkit.autopsy.keywordsearch;
21 import com.google.common.base.CharMatcher;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.List;
29 import java.util.logging.Level;
30 import java.util.regex.Matcher;
31 import java.util.regex.Pattern;
32 import org.apache.commons.lang.StringUtils;
33 import org.apache.solr.client.solrj.SolrQuery;
34 import org.apache.solr.client.solrj.response.TermsResponse.Term;
41 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
43 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
54 final class TermsComponentQuery
implements KeywordSearchQuery {
56 private static final Logger LOGGER = Logger.getLogger(TermsComponentQuery.class.getName());
57 private static final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
58 private static final String SEARCH_HANDLER =
"/terms";
59 private static final String SEARCH_FIELD = Server.Schema.TEXT.toString();
60 private static final int TERMS_SEARCH_TIMEOUT = 90 * 1000;
61 private static final String CASE_INSENSITIVE =
"case_insensitive";
62 private static final boolean DEBUG_FLAG = Version.Type.DEVELOPMENT.equals(Version.getBuildType());
63 private static final int MAX_TERMS_QUERY_RESULTS = 20000;
65 private final KeywordList keywordList;
66 private final Keyword originalKeyword;
67 private final List<KeywordQueryFilter> filters =
new ArrayList<>();
69 private String searchTerm;
70 private boolean searchTermIsEscaped;
82 static final Pattern CREDIT_CARD_NUM_PATTERN =
83 Pattern.compile(
"(?<ccn>[2-6]([ -]?[0-9]){11,18})");
84 static final Pattern CREDIT_CARD_TRACK1_PATTERN = Pattern.compile(
96 +
"(?<accountNumber>[2-6]([ -]?[0-9]){11,18})"
98 +
"(?<name>[^^]{2,26})"
100 +
"(?:(?:\\^|(?<expiration>\\d{4}))"
101 +
"(?:(?:\\^|(?<serviceCode>\\d{3}))"
102 +
"(?:(?<discretionary>[^?]*)"
106 static final Pattern CREDIT_CARD_TRACK2_PATTERN = Pattern.compile(
117 +
"(?<accountNumber>[2-6]([ -]?[0-9]){11,18})"
119 +
"(?:(?<expiration>\\d{4})"
120 +
"(?:(?<serviceCode>\\d{3})"
121 +
"(?:(?<discretionary>[^:;<=>?]*)"
125 static final BlackboardAttribute.Type KEYWORD_SEARCH_DOCUMENT_ID =
new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_DOCUMENT_ID);
144 TermsComponentQuery(KeywordList keywordList, Keyword keyword) {
145 this.keywordList = keywordList;
146 this.originalKeyword = keyword;
147 this.searchTerm = keyword.getSearchTerm();
157 public KeywordList getKeywordList() {
168 public String getQueryString() {
169 return originalKeyword.getSearchTerm();
180 public boolean isLiteral() {
189 public void setSubstringQuery() {
190 searchTerm =
".*" + searchTerm +
".*";
197 public void escape() {
198 searchTerm = Pattern.quote(originalKeyword.getSearchTerm());
199 searchTermIsEscaped =
true;
208 public boolean isEscaped() {
209 return searchTermIsEscaped;
219 public String getEscapedQueryString() {
220 return this.searchTerm;
229 public boolean validate() {
230 if (searchTerm.isEmpty()) {
234 Pattern.compile(searchTerm);
236 }
catch (IllegalArgumentException ex) {
248 public void setField(String field) {
258 public void addFilter(KeywordQueryFilter filter) {
259 this.filters.add(filter);
273 public QueryResults performQuery() throws KeywordSearchModuleException, NoOpenCoreException {
278 final SolrQuery termsQuery =
new SolrQuery();
279 termsQuery.setRequestHandler(SEARCH_HANDLER);
280 termsQuery.setTerms(
true);
281 termsQuery.setTermsRegexFlag(CASE_INSENSITIVE);
282 termsQuery.setTermsRegex(searchTerm);
283 termsQuery.addTermsField(SEARCH_FIELD);
284 termsQuery.setTimeAllowed(TERMS_SEARCH_TIMEOUT);
285 termsQuery.setShowDebugInfo(DEBUG_FLAG);
286 termsQuery.setTermsLimit(MAX_TERMS_QUERY_RESULTS);
287 List<Term> terms = KeywordSearch.getServer().queryTerms(termsQuery).getTerms(SEARCH_FIELD);
291 QueryResults results =
new QueryResults(
this);
292 for (Term term : terms) {
297 if (originalKeyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
298 Matcher matcher = CREDIT_CARD_NUM_PATTERN.matcher(term.getTerm());
299 if (
false == matcher.find()
300 ||
false == CreditCardValidator.isValidCCN(matcher.group(
"ccn"))) {
315 String escapedTerm = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
316 LuceneQuery termQuery =
new LuceneQuery(keywordList,
new Keyword(escapedTerm,
true,
true));
317 filters.forEach(termQuery::addFilter);
318 QueryResults termQueryResult = termQuery.performQuery();
319 Set<KeywordHit> termHits =
new HashSet<>();
320 for (Keyword word : termQueryResult.getKeywords()) {
321 termHits.addAll(termQueryResult.getResults(word));
323 results.addResult(
new Keyword(term.getTerm(),
false,
true, originalKeyword.getListName(), originalKeyword.getOriginalTerm()),
new ArrayList<>(termHits));
329 public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
336 BlackboardArtifact newArtifact;
337 Collection<BlackboardAttribute> attributes =
new ArrayList<>();
338 if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
339 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
340 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, originalKeyword.getSearchTerm()));
343 newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
345 }
catch (TskCoreException ex) {
346 LOGGER.log(Level.SEVERE,
"Error adding artifact for keyword hit to blackboard", ex);
354 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, MODULE_NAME, Account.Type.CREDIT_CARD.name()));
355 Map<BlackboardAttribute.Type, BlackboardAttribute> parsedTrackAttributeMap =
new HashMap<>();
356 Matcher matcher = CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet());
357 if (matcher.find()) {
358 parseTrack1Data(parsedTrackAttributeMap, matcher);
360 matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet());
361 if (matcher.find()) {
362 parseTrack2Data(parsedTrackAttributeMap, matcher);
364 final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(
new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER));
365 if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) {
366 if (hit.isArtifactHit()) {
367 LOGGER.log(Level.SEVERE, String.format(
"Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", searchTerm, hit.getSnippet(), hit.getArtifactID().get()));
369 LOGGER.log(Level.SEVERE, String.format(
"Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", searchTerm, hit.getSnippet(), hit.getContentID()));
373 attributes.addAll(parsedTrackAttributeMap.values());
379 final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8));
380 CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin);
381 if (binInfo != null) {
382 binInfo.getScheme().ifPresent(scheme
383 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme)));
384 binInfo.getCardType().ifPresent(cardType
385 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType)));
386 binInfo.getBrand().ifPresent(brand
387 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand)));
388 binInfo.getBankName().ifPresent(bankName
389 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName)));
390 binInfo.getBankPhoneNumber().ifPresent(phoneNumber
391 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber)));
392 binInfo.getBankURL().ifPresent(url
393 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url)));
394 binInfo.getCountry().ifPresent(country
395 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country)));
396 binInfo.getBankCity().ifPresent(city
397 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city)));
405 if (content instanceof AbstractFile) {
406 AbstractFile file = (AbstractFile) content;
407 if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS
408 || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) {
409 attributes.add(
new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId()));
417 newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_ACCOUNT);
418 }
catch (TskCoreException ex) {
419 LOGGER.log(Level.SEVERE,
"Error adding artifact for account to blackboard", ex);
424 if (StringUtils.isNotBlank(listName)) {
425 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
427 if (snippet != null) {
428 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
431 hit.getArtifactID().ifPresent(
432 artifactID -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
436 attributes.add(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal()));
439 newArtifact.addAttributes(attributes);
441 }
catch (TskCoreException e) {
442 LOGGER.log(Level.SEVERE,
"Error adding bb attributes for terms search artifact", e);
455 static private void parseTrack2Data(Map<BlackboardAttribute.Type, BlackboardAttribute> attributesMap, Matcher matcher) {
456 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_NUMBER,
"accountNumber", matcher);
457 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_EXPIRATION,
"expiration", matcher);
458 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_SERVICE_CODE,
"serviceCode", matcher);
459 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_DISCRETIONARY,
"discretionary", matcher);
460 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_LRC,
"LRC", matcher);
472 static private void parseTrack1Data(Map<BlackboardAttribute.Type, BlackboardAttribute> attributeMap, Matcher matcher) {
473 parseTrack2Data(attributeMap, matcher);
474 addAttributeIfNotAlreadyCaptured(attributeMap, ATTRIBUTE_TYPE.TSK_NAME_PERSON,
"name", matcher);
488 static private void addAttributeIfNotAlreadyCaptured(Map<BlackboardAttribute.Type, BlackboardAttribute> attributeMap, ATTRIBUTE_TYPE attrType, String groupName, Matcher matcher) {
489 BlackboardAttribute.Type type =
new BlackboardAttribute.Type(attrType);
490 attributeMap.computeIfAbsent(type, (BlackboardAttribute.Type t) -> {
491 String value = matcher.group(groupName);
492 if (attrType.equals(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)) {
493 value = CharMatcher.anyOf(
" -").removeFrom(value);
495 if (StringUtils.isNotBlank(value)) {
496 return new BlackboardAttribute(attrType, MODULE_NAME, value);