Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
HashDbIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2012-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.hashdatabase;
20 
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.concurrent.atomic.AtomicLong;
26 import java.util.function.Function;
27 import java.util.logging.Level;
28 import java.util.stream.Stream;
29 import org.openide.util.NbBundle;
30 import org.openide.util.NbBundle.Messages;
42 import org.sleuthkit.datamodel.AbstractFile;
43 import org.sleuthkit.datamodel.Blackboard;
44 import org.sleuthkit.datamodel.BlackboardArtifact;
45 import org.sleuthkit.datamodel.BlackboardAttribute;
46 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
47 import org.sleuthkit.datamodel.HashHitInfo;
48 import org.sleuthkit.datamodel.HashUtility;
49 import org.sleuthkit.datamodel.Score;
50 import org.sleuthkit.datamodel.SleuthkitCase;
51 import org.sleuthkit.datamodel.TskCoreException;
52 import org.sleuthkit.datamodel.TskData;
53 import org.sleuthkit.datamodel.TskException;
54 
58 @Messages({
59  "HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
60  "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
61  "HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
62  "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed.",
63  "# {0} - fileName", "HashDbIngestModule.lookingUpKnownBadHashValueErr=Error encountered while looking up notable hash value for {0}.",
64  "# {0} - fileName", "HashDbIngestModule.lookingUpNoChangeHashValueErr=Error encountered while looking up no change hash value for {0}.",
65  "# {0} - fileName", "HashDbIngestModule.lookingUpKnownHashValueErr=Error encountered while looking up known hash value for {0}.",})
66 public class HashDbIngestModule implements FileIngestModule {
67 
68  private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
69 
70  private final Function<AbstractFile, String> knownBadLookupError
71  = (file) -> Bundle.HashDbIngestModule_lookingUpKnownBadHashValueErr(file.getName());
72 
73  private final Function<AbstractFile, String> noChangeLookupError
74  = (file) -> Bundle.HashDbIngestModule_lookingUpNoChangeHashValueErr(file.getName());
75 
76  private final Function<AbstractFile, String> knownLookupError
77  = (file) -> Bundle.HashDbIngestModule_lookingUpKnownHashValueErr(file.getName());
78 
79  private static final int MAX_COMMENT_SIZE = 500;
80  private final IngestServices services = IngestServices.getInstance();
81  private final SleuthkitCase skCase;
82  private final HashDbManager hashDbManager = HashDbManager.getInstance();
83  private final HashLookupModuleSettings settings;
84  private final List<HashDb> knownBadHashSets = new ArrayList<>();
85  private final List<HashDb> knownHashSets = new ArrayList<>();
86  private final List<HashDb> noChangeHashSets = new ArrayList<>();
87  private long jobId;
88  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
89  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
90  private Blackboard blackboard;
91 
95  private static class IngestJobTotals {
96 
97  private final AtomicLong totalKnownBadCount = new AtomicLong(0);
98  private final AtomicLong totalNoChangeCount = new AtomicLong(0);
99  private final AtomicLong totalCalctime = new AtomicLong(0);
100  private final AtomicLong totalLookuptime = new AtomicLong(0);
101  }
102 
103  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
104  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
105  if (totals == null) {
106  totals = new HashDbIngestModule.IngestJobTotals();
107  totalsForIngestJobs.put(ingestJobId, totals);
108  }
109  return totals;
110  }
111 
121  HashDbIngestModule(HashLookupModuleSettings settings) throws NoCurrentCaseException {
122  this.settings = settings;
124  }
125 
126  @Override
128  jobId = context.getJobId();
129  if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
130  throw new IngestModuleException("Could not load all hash sets");
131  }
132 
133  initializeHashsets(hashDbManager.getAllHashSets());
134 
135  if (refCounter.incrementAndGet(jobId) == 1) {
136  // initialize job totals
137  getTotalsForIngestJobs(jobId);
138 
139  // if first module for this job then post error msgs if needed
140  if (knownBadHashSets.isEmpty()) {
143  Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
144  Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
145  }
146 
147  if (knownHashSets.isEmpty()) {
150  Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
151  Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
152  }
153  }
154  }
155 
162  private void initializeHashsets(List<HashDb> allHashSets) {
163  for (HashDb db : allHashSets) {
164  if (settings.isHashSetEnabled(db)) {
165  try {
166  if (db.isValid()) {
167  switch (db.getKnownFilesType()) {
168  case KNOWN:
169  knownHashSets.add(db);
170  break;
171  case KNOWN_BAD:
172  knownBadHashSets.add(db);
173  break;
174  case NO_CHANGE:
175  noChangeHashSets.add(db);
176  break;
177  default:
178  throw new TskCoreException("Unknown KnownFilesType: " + db.getKnownFilesType());
179  }
180  }
181  } catch (TskCoreException ex) {
182  logger.log(Level.WARNING, "Error getting index status for " + db.getDisplayName() + " hash set", ex); //NON-NLS
183  }
184  }
185  }
186  }
187 
188  @Messages({
189  "# {0} - File name",
190  "HashDbIngestModule.dialogTitle.errorFindingArtifacts=Error Finding Artifacts: {0}",
191  "# {0} - File name",
192  "HashDbIngestModule.errorMessage.lookingForFileArtifacts=Error encountered while looking for existing artifacts for {0}."
193  })
194  @Override
195  public ProcessResult process(AbstractFile file) {
196  try {
197  blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
198  } catch (NoCurrentCaseException ex) {
199  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
200  return ProcessResult.ERROR;
201  }
202 
203  if (shouldSkip(file)) {
204  return ProcessResult.OK;
205  }
206 
207  // Safely get a reference to the totalsForIngestJobs object
208  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
209 
210  // calc hash values
211  try {
212  calculateHashes(file, totals);
213  } catch (TskCoreException ex) {
214  logger.log(Level.WARNING, String.format("Error calculating hash of file '%s' (id=%d).", file.getName(), file.getId()), ex); //NON-NLS
217  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", file.getName()),
218  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr",
219  file.getParentPath() + file.getName(),
220  file.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC) ? "Allocated File" : "Deleted File")));
221  }
222 
223  // the processing result of handling this file
225 
226  // look up in notable first
227  FindInHashsetsResult knownBadResult = findInHashsets(file, totals.totalKnownBadCount,
228  totals.totalLookuptime, knownBadHashSets, TskData.FileKnown.BAD, knownBadLookupError);
229 
230  boolean foundBad = knownBadResult.isFound();
231  if (knownBadResult.isError()) {
232  ret = ProcessResult.ERROR;
233  }
234 
235  // look up no change items next
236  FindInHashsetsResult noChangeResult = findInHashsets(file, totals.totalNoChangeCount,
237  totals.totalLookuptime, noChangeHashSets, TskData.FileKnown.UNKNOWN, noChangeLookupError);
238 
239  if (noChangeResult.isError()) {
240  ret = ProcessResult.ERROR;
241  }
242 
243  // If the file is not in the notable sets, search for it in the known sets.
244  // Any hit is sufficient to classify it as known, and there is no need to create
245  // a hit artifact or send a message to the application inbox.
246  if (!foundBad) {
247  for (HashDb db : knownHashSets) {
248  try {
249  long lookupstart = System.currentTimeMillis();
250  if (db.lookupMD5Quick(file)) {
251  file.setKnown(TskData.FileKnown.KNOWN);
252  break;
253  }
254  long delta = (System.currentTimeMillis() - lookupstart);
255  totals.totalLookuptime.addAndGet(delta);
256 
257  } catch (TskException ex) {
258  reportLookupError(ex, file, knownLookupError);
259  ret = ProcessResult.ERROR;
260  }
261  }
262  }
263 
264  return ret;
265  }
266 
274  private boolean shouldSkip(AbstractFile file) {
275  // Skip unallocated space files.
276  if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
277  || file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
278  return true;
279  }
280 
281  /*
282  * Skip directories. One reason for this is because we won't accurately
283  * calculate hashes of NTFS directories that have content that spans the
284  * IDX_ROOT and IDX_ALLOC artifacts. So we disable that until a solution
285  * for it is developed.
286  */
287  if (file.isDir()) {
288  return true;
289  }
290 
291  // bail out if we have no hashes set
292  if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
293  return true;
294  }
295 
296  return false;
297  }
298 
308  private void reportLookupError(TskException ex, AbstractFile file, Function<AbstractFile, String> lookupErrorMessage) {
309  logger.log(Level.WARNING, String.format(
310  "Couldn't lookup notable hash for file '%s' (id=%d) - see sleuthkit log for details", file.getName(), file.getId()), ex); //NON-NLS
313  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", file.getName()),
314  lookupErrorMessage.apply(file)));
315  }
316 
320  private static class FindInHashsetsResult {
321 
322  private final boolean found;
323  private final boolean error;
324 
325  FindInHashsetsResult(boolean found, boolean error) {
326  this.found = found;
327  this.error = error;
328  }
329 
335  boolean isFound() {
336  return found;
337  }
338 
346  boolean isError() {
347  return error;
348  }
349  }
350 
369  private FindInHashsetsResult findInHashsets(AbstractFile file, AtomicLong totalCount, AtomicLong totalLookupTime,
370  List<HashDb> hashSets, TskData.FileKnown statusIfFound, Function<AbstractFile, String> lookupErrorMessage) {
371 
372  boolean found = false;
373  boolean wasError = false;
374  for (HashDb db : hashSets) {
375  try {
376  long lookupstart = System.currentTimeMillis();
377  HashHitInfo hashInfo = db.lookupMD5(file);
378  if (null != hashInfo) {
379  found = true;
380 
381  totalCount.incrementAndGet();
382  file.setKnown(statusIfFound);
383  String comment = generateComment(hashInfo);
384  if (!createArtifactIfNotExists(file, comment, db)) {
385  wasError = true;
386  }
387  }
388  long delta = (System.currentTimeMillis() - lookupstart);
389  totalLookupTime.addAndGet(delta);
390 
391  } catch (TskException ex) {
392  reportLookupError(ex, file, lookupErrorMessage);
393  wasError = true;
394  }
395  }
396 
397  return new FindInHashsetsResult(found, wasError);
398  }
399 
407  private String generateComment(HashHitInfo hashInfo) {
408  String comment = "";
409  ArrayList<String> comments = hashInfo.getComments();
410  int i = 0;
411  for (String c : comments) {
412  if (++i > 1) {
413  comment += " ";
414  }
415  comment += c;
416  if (comment.length() > MAX_COMMENT_SIZE) {
417  comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
418  break;
419  }
420  }
421  return comment;
422  }
423 
433  private boolean createArtifactIfNotExists(AbstractFile file, String comment, HashDb db) {
434  /*
435  * We have a match. Now create an artifact if it is determined that one
436  * hasn't been created yet.
437  */
438  List<BlackboardAttribute> attributesList = new ArrayList<>();
439  attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, HashLookupModuleFactory.getModuleName(), db.getDisplayName()));
440  try {
441  Blackboard tskBlackboard = skCase.getBlackboard();
442  if (tskBlackboard.artifactExists(file, BlackboardArtifact.Type.TSK_HASHSET_HIT, attributesList) == false) {
443  postHashSetHitToBlackboard(file, file.getMd5Hash(), db, comment);
444  }
445  } catch (TskCoreException ex) {
446  logger.log(Level.SEVERE, String.format(
447  "A problem occurred while checking for existing artifacts for file '%s' (id=%d).", file.getName(), file.getId()), ex); //NON-NLS
450  Bundle.HashDbIngestModule_dialogTitle_errorFindingArtifacts(file.getName()),
451  Bundle.HashDbIngestModule_errorMessage_lookingForFileArtifacts(file.getName())));
452  return false;
453  }
454  return true;
455  }
456 
464  private void calculateHashes(AbstractFile file, IngestJobTotals totals) throws TskCoreException {
465 
466  // First check if we've already calculated the hashes.
467  String md5Hash = file.getMd5Hash();
468  String sha256Hash = file.getSha256Hash();
469  if ((md5Hash != null && ! md5Hash.isEmpty())
470  && (sha256Hash != null && ! sha256Hash.isEmpty())) {
471  return;
472  }
473 
474  TimingMetric metric = HealthMonitor.getTimingMetric("Disk Reads: Hash calculation");
475  long calcstart = System.currentTimeMillis();
476  List<HashUtility.HashResult> newHashResults =
477  HashUtility.calculateHashes(file, Arrays.asList(HashUtility.HashType.MD5,HashUtility.HashType.SHA256 ));
478  if (file.getSize() > 0) {
479  // Surprisingly, the hash calculation does not seem to be correlated that
480  // strongly with file size until the files get large.
481  // Only normalize if the file size is greater than ~1MB.
482  if (file.getSize() < 1000000) {
484  } else {
485  // In testing, this normalization gave reasonable resuls
486  HealthMonitor.submitNormalizedTimingMetric(metric, file.getSize() / 500000);
487  }
488  }
489  for (HashUtility.HashResult hash : newHashResults) {
490  if (hash.getType().equals(HashUtility.HashType.MD5)) {
491  file.setMd5Hash(hash.getValue());
492  } else if (hash.getType().equals(HashUtility.HashType.SHA256)) {
493  file.setSha256Hash(hash.getValue());
494  }
495  }
496  long delta = (System.currentTimeMillis() - calcstart);
497  totals.totalCalctime.addAndGet(delta);
498  }
499 
505  private Score getScore(HashDb.KnownFilesType knownFilesType) {
506  if (knownFilesType == null) {
507  return Score.SCORE_UNKNOWN;
508  }
509  switch (knownFilesType) {
510  case KNOWN:
511  return Score.SCORE_NONE;
512  case KNOWN_BAD:
513  return Score.SCORE_NOTABLE;
514  default:
515  case NO_CHANGE:
516  return Score.SCORE_UNKNOWN;
517  }
518  }
527  @Messages({
528  "HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."
529  })
530  private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, HashDb db, String comment) {
531  try {
532  String moduleName = HashLookupModuleFactory.getModuleName();
533 
534  List<BlackboardAttribute> attributes = Arrays.asList(
535  new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, moduleName, db.getDisplayName()),
536  new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash),
537  new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment)
538  );
539 
540  // BlackboardArtifact.Type artifactType, Score score, String conclusion, String configuration, String justification, Collection<BlackboardAttribute> attributesList
541  BlackboardArtifact badFile = abstractFile.newAnalysisResult(
542  BlackboardArtifact.Type.TSK_HASHSET_HIT, getScore(db.getKnownFilesType()),
543  null, db.getDisplayName(), null,
544  attributes
545  ).getAnalysisResult();
546 
547  try {
548  /*
549  * post the artifact which will index the artifact for keyword
550  * search, and fire an event to notify UI of this new artifact
551  */
552  blackboard.postArtifact(badFile, moduleName, jobId);
553  } catch (Blackboard.BlackboardException ex) {
554  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS
556  Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
557  }
558 
559  if (db.getSendIngestMessages()) {
560  StringBuilder detailsSb = new StringBuilder();
561  //details
562  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
563  //hit
564  detailsSb.append("<tr>"); //NON-NLS
565  detailsSb.append("<th>") //NON-NLS
566  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.fileName"))
567  .append("</th>"); //NON-NLS
568  detailsSb.append("<td>") //NON-NLS
569  .append(abstractFile.getName())
570  .append("</td>"); //NON-NLS
571  detailsSb.append("</tr>"); //NON-NLS
572 
573  detailsSb.append("<tr>"); //NON-NLS
574  detailsSb.append("<th>") //NON-NLS
575  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.md5Hash"))
576  .append("</th>"); //NON-NLS
577  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
578  detailsSb.append("</tr>"); //NON-NLS
579 
580  detailsSb.append("<tr>"); //NON-NLS
581  detailsSb.append("<th>") //NON-NLS
582  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.hashsetName"))
583  .append("</th>"); //NON-NLS
584  detailsSb.append("<td>").append(db.getDisplayName()).append("</td>"); //NON-NLS
585  detailsSb.append("</tr>"); //NON-NLS
586 
587  detailsSb.append("</table>"); //NON-NLS
588 
590  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()),
591  detailsSb.toString(),
592  abstractFile.getName() + md5Hash,
593  badFile));
594  }
595  } catch (TskException ex) {
596  logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
597  }
598  }
599 
608  @Messages("HashDbIngestModule.complete.noChangesFound=No Change items found:")
609  private static synchronized void postSummary(long jobId, List<HashDb> knownBadHashSets,
610  List<HashDb> noChangeHashSets, List<HashDb> knownHashSets) {
611 
612  IngestJobTotals jobTotals = getTotalsForIngestJobs(jobId);
613  totalsForIngestJobs.remove(jobId);
614 
615  if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty()) || (!noChangeHashSets.isEmpty())) {
616  StringBuilder detailsSb = new StringBuilder();
617  //details
618  detailsSb.append(
619  "<table border='0' cellpadding='4' width='280'>" +
620  "<tr><td>" + NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.knownBadsFound") + "</td>" +
621  "<td>" + jobTotals.totalKnownBadCount.get() + "</td></tr>" +
622 
623  "<tr><td>" + Bundle.HashDbIngestModule_complete_noChangesFound() + "</td>" +
624  "<td>" + jobTotals.totalNoChangeCount.get() + "</td></tr>" +
625 
626  "<tr><td>" + NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalCalcTime") +
627  "</td><td>" + jobTotals.totalCalctime.get() + "</td></tr>\n" +
628 
629  "<tr><td>" + NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalLookupTime") +
630  "</td><td>" + jobTotals.totalLookuptime.get() + "</td></tr>\n</table>" +
631 
632  "<p>" + NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.databasesUsed") + "</p>\n<ul>"); //NON-NLS
633 
634  Stream.concat(knownBadHashSets.stream(), noChangeHashSets.stream()).forEach((db) -> {
635  detailsSb.append("<li>" + db.getHashSetName() + "</li>\n"); //NON-NLS
636  });
637 
638  detailsSb.append("</ul>"); //NON-NLS
639 
643  NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.hashLookupResults"),
644  detailsSb.toString()));
645  }
646  }
647 
648  @Override
649  public void shutDown() {
650  if (refCounter.decrementAndGet(jobId) == 0) {
651  postSummary(jobId, knownBadHashSets, noChangeHashSets, knownHashSets);
652  }
653  }
654 }
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void calculateHashes(AbstractFile file, IngestJobTotals totals)
static TimingMetric getTimingMetric(String name)
FindInHashsetsResult findInHashsets(AbstractFile file, AtomicLong totalCount, AtomicLong totalLookupTime, List< HashDb > hashSets, TskData.FileKnown statusIfFound, Function< AbstractFile, String > lookupErrorMessage)
void postMessage(final IngestMessage message)
void reportLookupError(TskException ex, AbstractFile file, Function< AbstractFile, String > lookupErrorMessage)
static void submitTimingMetric(TimingMetric metric)
static void error(String title, String message)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
boolean createArtifactIfNotExists(AbstractFile file, String comment, HashDb db)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, HashDb db, String comment)
static void submitNormalizedTimingMetric(TimingMetric metric, long normalization)
static synchronized IngestServices getInstance()

Copyright © 2012-2022 Basis Technology. Generated on: Thu Dec 8 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.