Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
DiscoveryAttributes.java
Go to the documentation of this file.
1 /*
2  * Autopsy
3  *
4  * Copyright 2020-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.discovery.search;
20 
21 import java.sql.ResultSet;
22 import java.sql.SQLException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.logging.Level;
32 import org.openide.util.NbBundle;
40 import org.sleuthkit.datamodel.BlackboardArtifact;
41 import org.sleuthkit.datamodel.BlackboardAttribute;
42 import org.sleuthkit.datamodel.CaseDbAccessManager;
43 import org.sleuthkit.datamodel.ContentTag;
44 import org.sleuthkit.datamodel.SleuthkitCase;
45 import org.sleuthkit.datamodel.TskCoreException;
46 import org.sleuthkit.datamodel.TskData;
47 import java.util.StringJoiner;
49 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_CATEGORIZATION;
50 
55 public class DiscoveryAttributes {
56 
57  private final static Logger logger = Logger.getLogger(DiscoveryAttributes.class.getName());
58 
62  public abstract static class AttributeType {
63 
72  public abstract DiscoveryKeyUtils.GroupKey getGroupKey(Result result);
73 
88  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb, CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
89  // Default is to do nothing
90  }
91  }
92 
96  public static class FileSizeAttribute extends AttributeType {
97 
98  @Override
100  return new DiscoveryKeyUtils.FileSizeGroupKey(result);
101  }
102  }
103 
107  public static class ParentPathAttribute extends AttributeType {
108 
109  @Override
112  }
113  }
114 
118  static class NoGroupingAttribute extends AttributeType {
119 
120  @Override
121  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
123  }
124  }
125 
129  static class DataSourceAttribute extends AttributeType {
130 
131  @Override
132  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
133  return new DiscoveryKeyUtils.DataSourceGroupKey(result);
134  }
135  }
136 
140  static class FileTypeAttribute extends AttributeType {
141 
142  @Override
143  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
144  return new DiscoveryKeyUtils.FileTypeGroupKey(file);
145  }
146  }
147 
152  static class DomainCategoryAttribute extends AttributeType {
153 
154  @Override
155  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
156  return new DiscoveryKeyUtils.DomainCategoryGroupKey(result);
157  }
158 
159  @Override
160  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
161  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
162  try {
163  Map<String, Set<String>> domainsToCategories = getDomainsWithWebCategories(caseDb, context);
164  for (Result result : results) {
165  if (context.searchIsCancelled()) {
166  throw new SearchCancellationException("The search was cancelled while Domain Category Attribute was being added.");
167  }
168  if (result instanceof ResultDomain) {
169  ResultDomain domain = (ResultDomain) result;
170  domain.addWebCategories(domainsToCategories.get(domain.getDomain()));
171  }
172  }
173  } catch (TskCoreException | InterruptedException ex) {
174  throw new DiscoveryException("Error fetching TSK_WEB_CATEGORY artifacts from the database", ex);
175  }
176  }
177 
195  private Map<String, Set<String>> getDomainsWithWebCategories(SleuthkitCase caseDb, SearchContext context) throws TskCoreException, InterruptedException, SearchCancellationException {
196  Map<String, Set<String>> domainToCategory = new HashMap<>();
197 
198  for (BlackboardArtifact artifact : caseDb.getBlackboardArtifacts(TSK_WEB_CATEGORIZATION)) {
199  if (Thread.currentThread().isInterrupted()) {
200  throw new InterruptedException();
201  }
202  if (context.searchIsCancelled()) {
203  throw new SearchCancellationException("Search was cancelled while getting domains for artifact type: " + artifact.getDisplayName());
204  }
205  BlackboardAttribute webCategory = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME));
206  BlackboardAttribute domain = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN));
207  if (webCategory != null && domain != null) {
208  String domainDisplayName = domain.getValueString().trim().toLowerCase();
209  if (!domainToCategory.containsKey(domainDisplayName)) {
210  domainToCategory.put(domainDisplayName, new HashSet<>());
211  }
212  domainToCategory.get(domainDisplayName).add(webCategory.getValueString());
213  }
214  }
215  return domainToCategory;
216  }
217  }
218 
222  static class KeywordListAttribute extends AttributeType {
223 
224  @Override
225  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
226  return new DiscoveryKeyUtils.KeywordListGroupKey((ResultFile) file);
227  }
228 
229  @Override
230  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
231  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
232 
233  // Get pairs of (object ID, keyword list name) for all files in the list of files that have
234  // keyword list hits.
235  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID(),
236  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
237  SetKeywordListNamesCallback callback = new SetKeywordListNamesCallback(results);
238  if (context.searchIsCancelled()) {
239  throw new SearchCancellationException("The search was cancelled while Keyword List Attribute was being added.");
240  }
241  try {
242  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
243  } catch (TskCoreException ex) {
244  throw new DiscoveryException("Error looking up keyword list attributes", ex); // NON-NLS
245  }
246  }
247 
253  private static class SetKeywordListNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
254 
255  List<Result> resultFiles;
256 
262  SetKeywordListNamesCallback(List<Result> resultFiles) {
263  this.resultFiles = resultFiles;
264  }
265 
266  @Override
267  public void process(ResultSet rs) {
268  try {
269  // Create a temporary map of object ID to ResultFile
270  Map<Long, ResultFile> tempMap = new HashMap<>();
271  for (Result result : resultFiles) {
272  if (result.getType() == SearchData.Type.DOMAIN) {
273  break;
274  }
275  ResultFile file = (ResultFile) result;
276  tempMap.put(file.getFirstInstance().getId(), file);
277  }
278 
279  while (rs.next()) {
280  try {
281  Long objId = rs.getLong("object_id"); // NON-NLS
282  String keywordListName = rs.getString("set_name"); // NON-NLS
283 
284  tempMap.get(objId).addKeywordListName(keywordListName);
285 
286  } catch (SQLException ex) {
287  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
288  }
289  }
290  } catch (SQLException ex) {
291  logger.log(Level.SEVERE, "Failed to get keyword list names", ex); // NON-NLS
292  }
293  }
294  }
295  }
296 
318  private static Map<String, List<ResultDomain>> organizeByValue(List<ResultDomain> domainsBatch, CorrelationAttributeInstance.Type attributeType, SearchContext context) throws SearchCancellationException {
319  final Map<String, List<ResultDomain>> resultDomainTable = new HashMap<>();
320  for (ResultDomain domainInstance : domainsBatch) {
321  try {
322  final String domainValue = domainInstance.getDomain();
323  final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue);
324  final List<ResultDomain> bucket = resultDomainTable.getOrDefault(normalizedDomain, new ArrayList<>());
325  bucket.add(domainInstance);
326  resultDomainTable.put(normalizedDomain, bucket);
327  if (context.searchIsCancelled()) {
328  throw new SearchCancellationException("Search was cancelled while orgainizing domains by their normalized value.");
329  }
331  logger.log(Level.INFO, String.format("Domain [%s] failed normalization, skipping...", domainInstance.getDomain()));
332  }
333  }
334  return resultDomainTable;
335  }
336 
342  private static String createCSV(Set<String> values) {
343  StringJoiner joiner = new StringJoiner(", ");
344  for (String value : values) {
345  joiner.add("'" + value + "'");
346  }
347  return joiner.toString();
348  }
349 
353  static class PreviouslyNotableAttribute extends AttributeType {
354 
355  static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time
356 
357  @Override
358  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
359  return new DiscoveryKeyUtils.PreviouslyNotableGroupKey(result);
360  }
361 
362  @Override
363  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
364  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
365 
366  if (centralRepoDb != null) {
367  processFilesWithCr(results, centralRepoDb, context);
368  }
369  }
370 
385  private void processFilesWithCr(List<Result> results, CentralRepository centralRepo, SearchContext context) throws DiscoveryException, SearchCancellationException {
386 
387  List<ResultDomain> domainsBatch = new ArrayList<>();
388  for (Result result : results) {
389  if (context.searchIsCancelled()) {
390  throw new SearchCancellationException("The search was cancelled while Previously Notable attribute was being calculated with the CR.");
391  }
392  if (result.getType() == SearchData.Type.DOMAIN) {
393  domainsBatch.add((ResultDomain) result);
394  if (domainsBatch.size() == DOMAIN_BATCH_SIZE) {
395  queryPreviouslyNotable(domainsBatch, centralRepo, context);
396  domainsBatch.clear();
397  }
398  }
399  }
400 
401  queryPreviouslyNotable(domainsBatch, centralRepo, context);
402  }
403 
419  private void queryPreviouslyNotable(List<ResultDomain> domainsBatch, CentralRepository centralRepo, SearchContext context) throws DiscoveryException, SearchCancellationException {
420  if (domainsBatch.isEmpty()) {
421  return;
422  }
423 
424  try {
425  final CorrelationAttributeInstance.Type attributeType = centralRepo.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID);
426  final Map<String, List<ResultDomain>> resultDomainTable = organizeByValue(domainsBatch, attributeType, context);
427  final String values = createCSV(resultDomainTable.keySet());
428  if (context.searchIsCancelled()) {
429  throw new SearchCancellationException("Search was cancelled while checking for previously notable domains.");
430  }
431  final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
432  final String domainFrequencyQuery = " value AS domain_name "
433  + "FROM " + tableName + " "
434  + "WHERE value IN (" + values + ") "
435  + "AND known_status = " + TskData.FileKnown.BAD.getFileKnownValue();
436 
437  final DomainPreviouslyNotableCallback previouslyNotableCallback = new DomainPreviouslyNotableCallback(resultDomainTable);
438  centralRepo.processSelectClause(domainFrequencyQuery, previouslyNotableCallback);
439 
440  if (previouslyNotableCallback.getCause() != null) {
441  throw previouslyNotableCallback.getCause();
442  }
443  } catch (CentralRepoException | SQLException ex) {
444  throw new DiscoveryException("Fatal exception encountered querying the CR.", ex);
445  }
446  }
447 
448  private static class DomainPreviouslyNotableCallback implements InstanceTableCallback {
449 
450  private final Map<String, List<ResultDomain>> domainLookup;
451  private SQLException sqlCause;
452 
453  private DomainPreviouslyNotableCallback(Map<String, List<ResultDomain>> domainLookup) {
454  this.domainLookup = domainLookup;
455  }
456 
457  @Override
458  public void process(ResultSet resultSet) {
459  try {
460  while (resultSet.next()) {
461  String domain = resultSet.getString("domain_name");
462  List<ResultDomain> domainInstances = domainLookup.get(domain);
463  for (ResultDomain domainInstance : domainInstances) {
464  domainInstance.markAsPreviouslyNotableInCR();
465  }
466  }
467  } catch (SQLException ex) {
468  this.sqlCause = ex;
469  }
470  }
471 
475  SQLException getCause() {
476  return this.sqlCause;
477  }
478  }
479  }
480 
484  static class FrequencyAttribute extends AttributeType {
485 
486  static final int BATCH_SIZE = 50; // Number of hashes to look up at one time
487 
488  static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time
489 
490  @Override
491  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
492  return new DiscoveryKeyUtils.FrequencyGroupKey(file);
493  }
494 
495  @Override
496  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
497  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
498  if (centralRepoDb == null) {
499  for (Result result : results) {
500  if (result.getFrequency() == SearchData.Frequency.UNKNOWN && result.getKnown() == TskData.FileKnown.KNOWN) {
501  result.setFrequency(SearchData.Frequency.KNOWN);
502  }
503  }
504  } else {
505  processResultFilesForCR(results, centralRepoDb, context);
506  }
507  }
508 
524  private void processResultFilesForCR(List<Result> results,
525  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
526  List<ResultFile> currentFiles = new ArrayList<>();
527  Set<String> hashesToLookUp = new HashSet<>();
528  List<ResultDomain> domainsToQuery = new ArrayList<>();
529  for (Result result : results) {
530  if (context.searchIsCancelled()) {
531  throw new SearchCancellationException("The search was cancelled while Frequency attribute was being calculated with the CR.");
532  }
533  // If frequency was already calculated, skip...
534  if (result.getFrequency() == SearchData.Frequency.UNKNOWN) {
535  if (result.getKnown() == TskData.FileKnown.KNOWN) {
536  result.setFrequency(SearchData.Frequency.KNOWN);
537  }
538 
539  if (result.getType() != SearchData.Type.DOMAIN) {
540  ResultFile file = (ResultFile) result;
541  if (file.getFirstInstance().getMd5Hash() != null
542  && !file.getFirstInstance().getMd5Hash().isEmpty()) {
543  hashesToLookUp.add(file.getFirstInstance().getMd5Hash());
544  currentFiles.add(file);
545  }
546 
547  if (hashesToLookUp.size() >= BATCH_SIZE) {
548  computeFrequency(hashesToLookUp, currentFiles, centralRepoDb, context);
549 
550  hashesToLookUp.clear();
551  currentFiles.clear();
552  }
553  } else {
554  domainsToQuery.add((ResultDomain) result);
555  if (domainsToQuery.size() == DOMAIN_BATCH_SIZE) {
556  queryDomainFrequency(domainsToQuery, centralRepoDb, context);
557  domainsToQuery.clear();
558  }
559  }
560  }
561  }
562 
563  queryDomainFrequency(domainsToQuery, centralRepoDb, context);
564  computeFrequency(hashesToLookUp, currentFiles, centralRepoDb, context);
565  }
566  }
567 
581  private static void queryDomainFrequency(List<ResultDomain> domainsToQuery, CentralRepository centralRepository, SearchContext context) throws DiscoveryException, SearchCancellationException {
582  if (domainsToQuery.isEmpty()) {
583  return;
584  }
585  try {
586  final CorrelationAttributeInstance.Type attributeType = centralRepository.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID);
587  final Map<String, List<ResultDomain>> resultDomainTable = organizeByValue(domainsToQuery, attributeType, context);
588  final String values = createCSV(resultDomainTable.keySet());
589  final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
590  final String domainFrequencyQuery = " value AS domain_name, COUNT(value) AS frequency FROM"
591  + "(SELECT DISTINCT case_id, value FROM "
592  + tableName
593  + " WHERE value IN ("
594  + values
595  + ")) AS foo GROUP BY value";
596 
597  final DomainFrequencyCallback frequencyCallback = new DomainFrequencyCallback(resultDomainTable);
598 
599  centralRepository.processSelectClause(domainFrequencyQuery, frequencyCallback);
600  if (context.searchIsCancelled()) {
601  throw new SearchCancellationException("The search was cancelled while Domain frequency was being queried with the CR.");
602  }
603  if (frequencyCallback.getCause() != null) {
604  throw frequencyCallback.getCause();
605  }
606  } catch (CentralRepoException | SQLException ex) {
607  throw new DiscoveryException("Fatal exception encountered querying the CR.", ex);
608  }
609  }
610 
614  private static class DomainFrequencyCallback implements InstanceTableCallback {
615 
616  private final Map<String, List<ResultDomain>> domainLookup;
617  private SQLException sqlCause;
618 
624  private DomainFrequencyCallback(Map<String, List<ResultDomain>> domainLookup) {
625  this.domainLookup = domainLookup;
626  }
627 
628  @Override
629  public void process(ResultSet resultSet) {
630  try {
631  while (resultSet.next()) {
632  String domain = resultSet.getString("domain_name");
633  Long frequency = resultSet.getLong("frequency");
634 
635  List<ResultDomain> domainInstances = domainLookup.get(domain);
636  for (ResultDomain domainInstance : domainInstances) {
637  domainInstance.setFrequency(SearchData.Frequency.fromCount(frequency));
638  }
639  }
640  } catch (SQLException ex) {
641  this.sqlCause = ex;
642  }
643  }
644 
650  SQLException getCause() {
651  return this.sqlCause;
652  }
653  }
654 
659  private static class FrequencyCallback implements InstanceTableCallback {
660 
661  private final List<ResultFile> files;
662 
668  private FrequencyCallback(List<ResultFile> files) {
669  this.files = new ArrayList<>(files);
670  }
671 
672  @Override
673  public void process(ResultSet resultSet) {
674  try {
675 
676  while (resultSet.next()) {
677  String hash = resultSet.getString(1);
678  int count = resultSet.getInt(2);
679  for (Iterator<ResultFile> iterator = files.iterator(); iterator.hasNext();) {
680  ResultFile file = iterator.next();
681  if (file.getFirstInstance().getMd5Hash().equalsIgnoreCase(hash)) {
683  iterator.remove();
684  }
685  }
686  }
687 
688  // The files left had no matching entries in the CR, so mark them as unique
689  for (ResultFile file : files) {
690  file.setFrequency(SearchData.Frequency.UNIQUE);
691  }
692  } catch (SQLException ex) {
693  logger.log(Level.WARNING, "Error getting frequency counts from Central Repository", ex); // NON-NLS
694  }
695  }
696  }
697 
701  static class HashHitsAttribute extends AttributeType {
702 
703  @Override
704  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
705  if (result.getType() == SearchData.Type.DOMAIN) {
706  return null;
707  }
708  return new DiscoveryKeyUtils.HashHitsGroupKey((ResultFile) result);
709  }
710 
711  @Override
712  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
713  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
714 
715  // Get pairs of (object ID, hash set name) for all files in the list of files that have
716  // hash set hits.
717  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID(),
718  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
719 
720  HashSetNamesCallback callback = new HashSetNamesCallback(results);
721  if (context.searchIsCancelled()) {
722  throw new SearchCancellationException("The search was cancelled while Hash Hit attribute was being added.");
723  }
724  try {
725  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
726  } catch (TskCoreException ex) {
727  throw new DiscoveryException("Error looking up hash set attributes", ex); // NON-NLS
728  }
729  }
730 
735  private static class HashSetNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
736 
737  List<Result> results;
738 
744  HashSetNamesCallback(List<Result> results) {
745  this.results = results;
746  }
747 
748  @Override
749  public void process(ResultSet rs) {
750  try {
751  // Create a temporary map of object ID to ResultFile
752  Map<Long, ResultFile> tempMap = new HashMap<>();
753  for (Result result : results) {
754  if (result.getType() == SearchData.Type.DOMAIN) {
755  return;
756  }
757  ResultFile file = (ResultFile) result;
758  tempMap.put(file.getFirstInstance().getId(), file);
759  }
760 
761  while (rs.next()) {
762  try {
763  Long objId = rs.getLong("object_id"); // NON-NLS
764  String hashSetName = rs.getString("set_name"); // NON-NLS
765 
766  tempMap.get(objId).addHashSetName(hashSetName);
767 
768  } catch (SQLException ex) {
769  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
770  }
771  }
772  } catch (SQLException ex) {
773  logger.log(Level.SEVERE, "Failed to get hash set names", ex); // NON-NLS
774  }
775  }
776  }
777  }
778 
782  static class InterestingItemAttribute extends AttributeType {
783 
784  @Override
785  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
787  }
788 
789  @Override
790  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
791  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
792 
793  // Get pairs of (object ID, interesting item set name) for all files in the list of files that have
794  // interesting item set hits.
795  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ITEM.getTypeID(),
796  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
797 
798  InterestingFileSetNamesCallback callback = new InterestingFileSetNamesCallback(results);
799  if (context.searchIsCancelled()) {
800  throw new SearchCancellationException("The search was cancelled while Interesting Item attribute was being added.");
801  }
802  try {
803  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
804  } catch (TskCoreException ex) {
805  throw new DiscoveryException("Error looking up interesting file set attributes", ex); // NON-NLS
806  }
807  }
808 
814  private static class InterestingFileSetNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
815 
816  List<Result> results;
817 
824  InterestingFileSetNamesCallback(List<Result> results) {
825  this.results = results;
826  }
827 
828  @Override
829  public void process(ResultSet rs) {
830  try {
831  // Create a temporary map of object ID to ResultFile
832  Map<Long, ResultFile> tempMap = new HashMap<>();
833  for (Result result : results) {
834  if (result.getType() == SearchData.Type.DOMAIN) {
835  return;
836  }
837  ResultFile file = (ResultFile) result;
838  tempMap.put(file.getFirstInstance().getId(), file);
839  }
840 
841  while (rs.next()) {
842  try {
843  Long objId = rs.getLong("object_id"); // NON-NLS
844  String setName = rs.getString("set_name"); // NON-NLS
845 
846  tempMap.get(objId).addInterestingSetName(setName);
847 
848  } catch (SQLException ex) {
849  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
850  }
851  }
852  } catch (SQLException ex) {
853  logger.log(Level.SEVERE, "Failed to get interesting file set names", ex); // NON-NLS
854  }
855  }
856  }
857  }
858 
862  static class LastActivityDateAttribute extends AttributeType {
863 
864  @Override
865  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
866  return new DiscoveryKeyUtils.LastActivityDateGroupKey(result);
867  }
868 
869  }
870 
874  static class FirstActivityDateAttribute extends AttributeType {
875 
876  @Override
877  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
878  return new DiscoveryKeyUtils.FirstActivityDateGroupKey(result);
879  }
880 
881  }
882 
887  static class PageViewsAttribute extends AttributeType {
888 
889  @Override
890  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
891  return new DiscoveryKeyUtils.PageViewsGroupKey(result);
892  }
893  }
894 
898  static class ObjectDetectedAttribute extends AttributeType {
899 
900  @Override
901  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
902  return new DiscoveryKeyUtils.ObjectDetectedGroupKey((ResultFile) file);
903  }
904 
905  @Override
906  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
907  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
908 
909  // Get pairs of (object ID, object type name) for all files in the list of files that have
910  // objects detected
911  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_OBJECT_DETECTED.getTypeID(),
912  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION.getTypeID());
913 
914  ObjectDetectedNamesCallback callback = new ObjectDetectedNamesCallback(results);
915  if (context.searchIsCancelled()) {
916  throw new SearchCancellationException("The search was cancelled while Object Detected attribute was being added.");
917  }
918  try {
919  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
920  } catch (TskCoreException ex) {
921  throw new DiscoveryException("Error looking up object detected attributes", ex); // NON-NLS
922  }
923  }
924 
930  private static class ObjectDetectedNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
931 
932  List<Result> results;
933 
939  ObjectDetectedNamesCallback(List<Result> results) {
940  this.results = results;
941  }
942 
943  @Override
944  public void process(ResultSet rs) {
945  try {
946  // Create a temporary map of object ID to ResultFile
947  Map<Long, ResultFile> tempMap = new HashMap<>();
948  for (Result result : results) {
949  if (result.getType() == SearchData.Type.DOMAIN) {
950  return;
951  }
952  ResultFile file = (ResultFile) result;
953  tempMap.put(file.getFirstInstance().getId(), file);
954  }
955 
956  while (rs.next()) {
957  try {
958  Long objId = rs.getLong("object_id"); // NON-NLS
959  String setName = rs.getString("set_name"); // NON-NLS
960 
961  tempMap.get(objId).addObjectDetectedName(setName);
962 
963  } catch (SQLException ex) {
964  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
965  }
966  }
967  } catch (SQLException ex) {
968  logger.log(Level.SEVERE, "Failed to get object detected names", ex); // NON-NLS
969  }
970  }
971  }
972  }
973 
977  static class FileTagAttribute extends AttributeType {
978 
979  @Override
980  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
981  return new DiscoveryKeyUtils.FileTagGroupKey((ResultFile) file);
982  }
983 
984  @Override
985  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
986  CentralRepository centralRepoDb, SearchContext context) throws DiscoveryException, SearchCancellationException {
987 
988  try {
989  for (Result result : results) {
990  if (context.searchIsCancelled()) {
991  throw new SearchCancellationException("The search was cancelled while File Tag attribute was being added.");
992  }
993  if (result.getType() == SearchData.Type.DOMAIN) {
994  return;
995  }
996  ResultFile file = (ResultFile) result;
997  List<ContentTag> contentTags = caseDb.getContentTagsByContent(file.getFirstInstance());
998 
999  for (ContentTag tag : contentTags) {
1000  result.addTagName(tag.getName().getDisplayName());
1001  }
1002  }
1003  } catch (TskCoreException ex) {
1004  throw new DiscoveryException("Error looking up file tag attributes", ex); // NON-NLS
1005  }
1006  }
1007  }
1008 
1012  @NbBundle.Messages({
1013  "DiscoveryAttributes.GroupingAttributeType.fileType.displayName=File Type",
1014  "DiscoveryAttributes.GroupingAttributeType.frequency.displayName=Past Occurrences",
1015  "DiscoveryAttributes.GroupingAttributeType.keywordList.displayName=Keyword",
1016  "DiscoveryAttributes.GroupingAttributeType.size.displayName=File Size",
1017  "DiscoveryAttributes.GroupingAttributeType.datasource.displayName=Data Source",
1018  "DiscoveryAttributes.GroupingAttributeType.parent.displayName=Parent Folder",
1019  "DiscoveryAttributes.GroupingAttributeType.hash.displayName=Hash Set",
1020  "DiscoveryAttributes.GroupingAttributeType.interestingItem.displayName=Interesting Item",
1021  "DiscoveryAttributes.GroupingAttributeType.tag.displayName=Tag",
1022  "DiscoveryAttributes.GroupingAttributeType.object.displayName=Object Detected",
1023  "DiscoveryAttributes.GroupingAttributeType.lastDate.displayName=Final Activity Date",
1024  "DiscoveryAttributes.GroupingAttributeType.firstDate.displayName=First Activity Date",
1025  "DiscoveryAttributes.GroupingAttributeType.pageViews.displayName=Page Views",
1026  "DiscoveryAttributes.GroupingAttributeType.none.displayName=None",
1027  "DiscoveryAttributes.GroupingAttributeType.previouslyNotable.displayName=Previous Notability",
1028  "DiscoveryAttributes.GroupingAttributeType.webCategory.displayName=Domain Category"})
1030  FILE_SIZE(new FileSizeAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_size_displayName()),
1031  FREQUENCY(new FrequencyAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_frequency_displayName()),
1032  KEYWORD_LIST_NAME(new KeywordListAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_keywordList_displayName()),
1033  DATA_SOURCE(new DataSourceAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_datasource_displayName()),
1034  PARENT_PATH(new ParentPathAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_parent_displayName()),
1035  HASH_LIST_NAME(new HashHitsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_hash_displayName()),
1036  INTERESTING_ITEM_SET(new InterestingItemAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_interestingItem_displayName()),
1037  FILE_TAG(new FileTagAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_tag_displayName()),
1038  OBJECT_DETECTED(new ObjectDetectedAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_object_displayName()),
1039  LAST_ACTIVITY_DATE(new LastActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_lastDate_displayName()),
1040  FIRST_ACTIVITY_DATE(new FirstActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_firstDate_displayName()),
1041  PAGE_VIEWS(new PageViewsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_pageViews_displayName()),
1042  NO_GROUPING(new NoGroupingAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_none_displayName()),
1043  PREVIOUSLY_NOTABLE(new PreviouslyNotableAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_previouslyNotable_displayName()),
1044  DOMAIN_CATEGORY(new DomainCategoryAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_webCategory_displayName());
1045 
1047  private final String displayName;
1048 
1057  GroupingAttributeType(AttributeType attributeType, String displayName) {
1058  this.attributeType = attributeType;
1059  this.displayName = displayName;
1060  }
1061 
1062  @Override
1063  public String toString() {
1064  return displayName;
1065  }
1066 
1073  return attributeType;
1074  }
1075 
1081  public static List<GroupingAttributeType> getOptionsForGroupingForFiles() {
1082  return Arrays.asList(FILE_SIZE, FREQUENCY, PARENT_PATH, OBJECT_DETECTED, HASH_LIST_NAME, INTERESTING_ITEM_SET);
1083  }
1084 
1090  public static List<GroupingAttributeType> getOptionsForGroupingForDomains() {
1091  if (CentralRepository.isEnabled()) {
1092  return Arrays.asList(PAGE_VIEWS, FREQUENCY, LAST_ACTIVITY_DATE, FIRST_ACTIVITY_DATE, PREVIOUSLY_NOTABLE, DOMAIN_CATEGORY);
1093  } else {
1094  return Arrays.asList(PAGE_VIEWS, LAST_ACTIVITY_DATE, FIRST_ACTIVITY_DATE, DOMAIN_CATEGORY);
1095  }
1096  }
1097  }
1098 
1113  private static void computeFrequency(Set<String> hashesToLookUp, List<ResultFile> currentFiles, CentralRepository centralRepoDb, SearchContext context) throws SearchCancellationException {
1114 
1115  if (hashesToLookUp.isEmpty()) {
1116  return;
1117  }
1118 
1119  String hashes = String.join("','", hashesToLookUp);
1120  hashes = "'" + hashes + "'";
1121  try {
1122  CorrelationAttributeInstance.Type attributeType = centralRepoDb.getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID);
1123  String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
1124 
1125  String selectClause = " value, COUNT(value) FROM "
1126  + "(SELECT DISTINCT case_id, value FROM " + tableName
1127  + " WHERE value IN ("
1128  + hashes
1129  + ")) AS foo GROUP BY value";
1130 
1131  FrequencyCallback callback = new FrequencyCallback(currentFiles);
1132  centralRepoDb.processSelectClause(selectClause, callback);
1133  if (context.searchIsCancelled()) {
1134  throw new SearchCancellationException("The search was cancelled while Domain frequency was being queried with the CR.");
1135  }
1136  } catch (CentralRepoException ex) {
1137  logger.log(Level.WARNING, "Error getting frequency counts from Central Repository", ex); // NON-NLS
1138  }
1139 
1140  }
1141 
1155  private static String createSetNameClause(List<Result> results,
1156  int artifactTypeID, int setNameAttrID) throws DiscoveryException {
1157 
1158  // Concatenate the object IDs in the list of files
1159  String objIdList = ""; // NON-NLS
1160  for (Result result : results) {
1161  if (result.getType() == SearchData.Type.DOMAIN) {
1162  break;
1163  }
1164  ResultFile file = (ResultFile) result;
1165  if (!objIdList.isEmpty()) {
1166  objIdList += ","; // NON-NLS
1167  }
1168  objIdList += "\'" + file.getFirstInstance().getId() + "\'"; // NON-NLS
1169  }
1170 
1171  // Get pairs of (object ID, set name) for all files in the list of files that have
1172  // the given artifact type.
1173  return "blackboard_artifacts.obj_id AS object_id, blackboard_attributes.value_text AS set_name "
1174  + "FROM blackboard_artifacts "
1175  + "INNER JOIN blackboard_attributes ON blackboard_artifacts.artifact_id=blackboard_attributes.artifact_id "
1176  + "WHERE blackboard_attributes.artifact_type_id=\'" + artifactTypeID + "\' "
1177  + "AND blackboard_attributes.attribute_type_id=\'" + setNameAttrID + "\' "
1178  + "AND blackboard_artifacts.obj_id IN (" + objIdList
1179  + ") "; // NON-NLS
1180  }
1181 
1186  // Class should not be instantiated
1187  }
1188 }
static String createSetNameClause(List< Result > results, int artifactTypeID, int setNameAttrID)
abstract TskData.FileKnown getKnown()
static void queryDomainFrequency(List< ResultDomain > domainsToQuery, CentralRepository centralRepository, SearchContext context)
final void setFrequency(SearchData.Frequency frequency)
Definition: Result.java:81
static String correlationTypeToInstanceTableName(CorrelationAttributeInstance.Type type)
static String normalize(CorrelationAttributeInstance.Type attributeType, String data)
void addAttributeToResults(List< Result > results, SleuthkitCase caseDb, CentralRepository centralRepoDb, SearchContext context)
abstract DiscoveryKeyUtils.GroupKey getGroupKey(Result result)
void addKeywordListName(String keywordListName)
static Map< String, List< ResultDomain > > organizeByValue(List< ResultDomain > domainsBatch, CorrelationAttributeInstance.Type attributeType, SearchContext context)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static void computeFrequency(Set< String > hashesToLookUp, List< ResultFile > currentFiles, CentralRepository centralRepoDb, SearchContext context)

Copyright © 2012-2022 Basis Technology. Generated on: Wed Oct 5 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.