Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
HashDbIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.hashdatabase;
20 
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.logging.Level;
29 import org.openide.util.NbBundle;
30 import org.openide.util.NbBundle.Messages;
44 import org.sleuthkit.datamodel.AbstractFile;
45 import org.sleuthkit.datamodel.BlackboardArtifact;
46 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
47 import org.sleuthkit.datamodel.BlackboardAttribute;
48 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
49 import org.sleuthkit.datamodel.HashHitInfo;
50 import org.sleuthkit.datamodel.HashUtility;
51 import org.sleuthkit.datamodel.SleuthkitCase;
52 import org.sleuthkit.datamodel.TskCoreException;
53 import org.sleuthkit.datamodel.TskData;
54 import org.sleuthkit.datamodel.TskException;
55 
59 @Messages({
60  "HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
61  "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
62  "HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
63  "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
64 })
65 public class HashDbIngestModule implements FileIngestModule {
66 
67  private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
68  private static final int MAX_COMMENT_SIZE = 500;
69  private final IngestServices services = IngestServices.getInstance();
70  private final SleuthkitCase skCase;
71  private final HashDbManager hashDbManager = HashDbManager.getInstance();
72  private final HashLookupModuleSettings settings;
73  private final List<HashDb> knownBadHashSets = new ArrayList<>();
74  private final List<HashDb> knownHashSets = new ArrayList<>();
75  private long jobId;
76  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
77  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
79 
83  private static class IngestJobTotals {
84 
85  private final AtomicLong totalKnownBadCount = new AtomicLong(0);
86  private final AtomicLong totalCalctime = new AtomicLong(0);
87  private final AtomicLong totalLookuptime = new AtomicLong(0);
88  }
89 
90  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
91  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
92  if (totals == null) {
93  totals = new HashDbIngestModule.IngestJobTotals();
94  totalsForIngestJobs.put(ingestJobId, totals);
95  }
96  return totals;
97  }
98 
108  HashDbIngestModule(HashLookupModuleSettings settings) throws NoCurrentCaseException {
109  this.settings = settings;
111  }
112 
113  @Override
115  jobId = context.getJobId();
116  if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
117  throw new IngestModuleException("Could not load all hash sets");
118  }
119  updateEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets);
120  updateEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets);
121 
122  if (refCounter.incrementAndGet(jobId) == 1) {
123  // initialize job totals
124  getTotalsForIngestJobs(jobId);
125 
126  // if first module for this job then post error msgs if needed
127  if (knownBadHashSets.isEmpty()) {
129  HashLookupModuleFactory.getModuleName(),
130  Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
131  Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
132  }
133 
134  if (knownHashSets.isEmpty()) {
136  HashLookupModuleFactory.getModuleName(),
137  Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
138  Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
139  }
140  }
141  }
142 
149  private void updateEnabledHashSets(List<HashDb> allHashSets, List<HashDb> enabledHashSets) {
150  enabledHashSets.clear();
151  for (HashDb db : allHashSets) {
152  if (settings.isHashSetEnabled(db)) {
153  try {
154  if (db.isValid()) {
155  enabledHashSets.add(db);
156  }
157  } catch (TskCoreException ex) {
158  logger.log(Level.WARNING, "Error getting index status for " + db.getDisplayName() + " hash set", ex); //NON-NLS
159  }
160  }
161  }
162  }
163 
164  @Messages({
165  "# {0} - File name",
166  "HashDbIngestModule.dialogTitle.errorFindingArtifacts=Error Finding Artifacts: {0}",
167  "# {0} - File name",
168  "HashDbIngestModule.errorMessage.lookingForFileArtifacts=Error encountered while looking for existing artifacts for {0}."
169  })
170  @Override
171  public ProcessResult process(AbstractFile file) {
172  try {
174  } catch (NoCurrentCaseException ex) {
175  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
176  return ProcessResult.ERROR;
177  }
178 
179  // Skip unallocated space files.
180  if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
181  || file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
182  return ProcessResult.OK;
183  }
184 
185  /*
186  * Skip directories. One reason for this is because we won't accurately
187  * calculate hashes of NTFS directories that have content that spans the
188  * IDX_ROOT and IDX_ALLOC artifacts. So we disable that until a solution
189  * for it is developed.
190  */
191  if (file.isDir()) {
192  return ProcessResult.OK;
193  }
194 
195  // bail out if we have no hashes set
196  if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
197  return ProcessResult.OK;
198  }
199 
200  // Safely get a reference to the totalsForIngestJobs object
201  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
202 
203  // calc hash value
204  String name = file.getName();
205  long fileId = file.getId();
206  String md5Hash = file.getMd5Hash();
207  if (md5Hash == null || md5Hash.isEmpty()) {
208  try {
209  TimingMetric metric = HealthMonitor.getTimingMetric("Disk Reads: Hash calculation");
210  long calcstart = System.currentTimeMillis();
211  md5Hash = HashUtility.calculateMd5Hash(file);
212  if (file.getSize() > 0) {
213  // Surprisingly, the hash calculation does not seem to be correlated that
214  // strongly with file size until the files get large.
215  // Only normalize if the file size is greater than ~1MB.
216  if (file.getSize() < 1000000) {
218  } else {
219  // In testing, this normalization gave reasonable resuls
220  HealthMonitor.submitNormalizedTimingMetric(metric, file.getSize() / 500000);
221  }
222  }
223  file.setMd5Hash(md5Hash);
224  long delta = (System.currentTimeMillis() - calcstart);
225  totals.totalCalctime.addAndGet(delta);
226 
227  } catch (IOException ex) {
228  logger.log(Level.WARNING, String.format("Error calculating hash of file '%s' (id=%d).", name, fileId), ex); //NON-NLS
230  HashLookupModuleFactory.getModuleName(),
231  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", name),
232  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr",
233  file.getParentPath() + file.getName(),
234  file.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)?"Allocated File" : "Deleted File")));
235  return ProcessResult.ERROR;
236  }
237  }
238 
239  // look up in notable first
240  boolean foundBad = false;
242  for (HashDb db : knownBadHashSets) {
243  try {
244  long lookupstart = System.currentTimeMillis();
245  HashHitInfo hashInfo = db.lookupMD5(file);
246  if (null != hashInfo) {
247  foundBad = true;
248  totals.totalKnownBadCount.incrementAndGet();
249 
250  file.setKnown(TskData.FileKnown.BAD);
251 
252  String hashSetName = db.getDisplayName();
253 
254  String comment = "";
255  ArrayList<String> comments = hashInfo.getComments();
256  int i = 0;
257  for (String c : comments) {
258  if (++i > 1) {
259  comment += " ";
260  }
261  comment += c;
262  if (comment.length() > MAX_COMMENT_SIZE) {
263  comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
264  break;
265  }
266  }
267 
268  /*
269  * We have a match. Now create an artifact if it is
270  * determined that one hasn't been created yet.
271  */
272  List<BlackboardAttribute> attributesList = new ArrayList<>();
273  attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, HashLookupModuleFactory.getModuleName(), hashSetName));
274  try {
275  org.sleuthkit.datamodel.Blackboard tskBlackboard = skCase.getBlackboard();
276  if (tskBlackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, attributesList) == false) {
277  postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
278  }
279  } catch (TskCoreException ex) {
280  logger.log(Level.SEVERE, String.format(
281  "A problem occurred while checking for existing artifacts for file '%s' (id=%d).", name, fileId), ex); //NON-NLS
283  HashLookupModuleFactory.getModuleName(),
284  Bundle.HashDbIngestModule_dialogTitle_errorFindingArtifacts(name),
285  Bundle.HashDbIngestModule_errorMessage_lookingForFileArtifacts(name)));
286  ret = ProcessResult.ERROR;
287  }
288  }
289  long delta = (System.currentTimeMillis() - lookupstart);
290  totals.totalLookuptime.addAndGet(delta);
291 
292  } catch (TskException ex) {
293  logger.log(Level.WARNING, String.format(
294  "Couldn't lookup notable hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex); //NON-NLS
296  HashLookupModuleFactory.getModuleName(),
297  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name),
298  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.lookingUpKnownBadHashValueErr", name)));
299  ret = ProcessResult.ERROR;
300  }
301  }
302 
303  // If the file is not in the notable sets, search for it in the known sets.
304  // Any hit is sufficient to classify it as known, and there is no need to create
305  // a hit artifact or send a message to the application inbox.
306  if (!foundBad) {
307  for (HashDb db : knownHashSets) {
308  try {
309  long lookupstart = System.currentTimeMillis();
310  if (db.lookupMD5Quick(file)) {
311  file.setKnown(TskData.FileKnown.KNOWN);
312  break;
313  }
314  long delta = (System.currentTimeMillis() - lookupstart);
315  totals.totalLookuptime.addAndGet(delta);
316 
317  } catch (TskException ex) {
318  logger.log(Level.WARNING, String.format(
319  "Couldn't lookup known hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex); //NON-NLS
321  HashLookupModuleFactory.getModuleName(),
322  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name),
323  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.lookingUpKnownHashValueErr", name)));
324  ret = ProcessResult.ERROR;
325  }
326  }
327  }
328 
329  return ret;
330  }
331 
342  @Messages({
343  "HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."
344  })
345  private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
346  try {
347  String moduleName = HashLookupModuleFactory.getModuleName();
348  BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
349  Collection<BlackboardAttribute> attributes = new ArrayList<>();
350  //TODO Revisit usage of deprecated constructor as per TSK-583
351  //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName);
352  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, moduleName, hashSetName));
353  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash));
354  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment));
355 
356  badFile.addAttributes(attributes);
357 
358  try {
359  // index the artifact for keyword search
360  blackboard.indexArtifact(badFile);
361  } catch (Blackboard.BlackboardException ex) {
362  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS
364  Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
365  }
366 
367  if (showInboxMessage) {
368  StringBuilder detailsSb = new StringBuilder();
369  //details
370  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
371  //hit
372  detailsSb.append("<tr>"); //NON-NLS
373  detailsSb.append("<th>") //NON-NLS
374  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.fileName"))
375  .append("</th>"); //NON-NLS
376  detailsSb.append("<td>") //NON-NLS
377  .append(abstractFile.getName())
378  .append("</td>"); //NON-NLS
379  detailsSb.append("</tr>"); //NON-NLS
380 
381  detailsSb.append("<tr>"); //NON-NLS
382  detailsSb.append("<th>") //NON-NLS
383  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.md5Hash"))
384  .append("</th>"); //NON-NLS
385  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
386  detailsSb.append("</tr>"); //NON-NLS
387 
388  detailsSb.append("<tr>"); //NON-NLS
389  detailsSb.append("<th>") //NON-NLS
390  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.hashsetName"))
391  .append("</th>"); //NON-NLS
392  detailsSb.append("<td>").append(hashSetName).append("</td>"); //NON-NLS
393  detailsSb.append("</tr>"); //NON-NLS
394 
395  detailsSb.append("</table>"); //NON-NLS
396 
398  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()),
399  detailsSb.toString(),
400  abstractFile.getName() + md5Hash,
401  badFile));
402  }
403  services.fireModuleDataEvent(new ModuleDataEvent(moduleName, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
404  } catch (TskException ex) {
405  logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
406  }
407  }
408 
416  private static synchronized void postSummary(long jobId,
417  List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
418  IngestJobTotals jobTotals = getTotalsForIngestJobs(jobId);
419  totalsForIngestJobs.remove(jobId);
420 
421  if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
422  StringBuilder detailsSb = new StringBuilder();
423  //details
424  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
425 
426  detailsSb.append("<tr><td>") //NON-NLS
427  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.knownBadsFound"))
428  .append("</td>"); //NON-NLS
429  detailsSb.append("<td>").append(jobTotals.totalKnownBadCount.get()).append("</td></tr>"); //NON-NLS
430 
431  detailsSb.append("<tr><td>") //NON-NLS
432  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalCalcTime"))
433  .append("</td><td>").append(jobTotals.totalCalctime.get()).append("</td></tr>\n"); //NON-NLS
434  detailsSb.append("<tr><td>") //NON-NLS
435  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalLookupTime"))
436  .append("</td><td>").append(jobTotals.totalLookuptime.get()).append("</td></tr>\n"); //NON-NLS
437  detailsSb.append("</table>"); //NON-NLS
438 
439  detailsSb.append("<p>") //NON-NLS
440  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.databasesUsed"))
441  .append("</p>\n<ul>"); //NON-NLS
442  for (HashDb db : knownBadHashSets) {
443  detailsSb.append("<li>").append(db.getHashSetName()).append("</li>\n"); //NON-NLS
444  }
445 
446  detailsSb.append("</ul>"); //NON-NLS
447 
450  HashLookupModuleFactory.getModuleName(),
451  NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.hashLookupResults"),
452  detailsSb.toString()));
453  }
454  }
455 
456  @Override
457  public void shutDown() {
458  if (refCounter.decrementAndGet(jobId) == 0) {
459  postSummary(jobId, knownBadHashSets, knownHashSets);
460  }
461  }
462 }
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
static TimingMetric getTimingMetric(String name)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
static void submitTimingMetric(TimingMetric metric)
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:58
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
static void submitNormalizedTimingMetric(TimingMetric metric, long normalization)
static synchronized IngestServices getInstance()

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.