Autopsy  4.20.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
ThunderbirdMboxFileIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2012-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.thunderbirdparser;
20 
21 import java.io.File;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.Collection;
26 import java.util.Collections;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.Iterator;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.concurrent.ConcurrentHashMap;
34 import java.util.concurrent.ConcurrentMap;
35 import java.util.logging.Level;
36 import java.util.regex.Matcher;
37 import java.util.regex.Pattern;
38 import org.apache.james.mime4j.MimeException;
39 import org.openide.util.NbBundle;
40 import org.openide.util.NbBundle.Messages;
55 import org.sleuthkit.datamodel.AbstractFile;
56 import org.sleuthkit.datamodel.Account;
57 import org.sleuthkit.datamodel.AccountFileInstance;
58 import org.sleuthkit.datamodel.Blackboard;
59 import org.sleuthkit.datamodel.BlackboardArtifact;
60 import org.sleuthkit.datamodel.BlackboardAttribute;
61 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
62 import org.sleuthkit.datamodel.DerivedFile;
63 import org.sleuthkit.datamodel.ReadContentInputStream;
64 import org.sleuthkit.datamodel.Relationship;
65 import org.sleuthkit.datamodel.Score;
66 import org.sleuthkit.datamodel.TskCoreException;
67 import org.sleuthkit.datamodel.TskData;
68 import org.sleuthkit.datamodel.TskDataException;
69 import org.sleuthkit.datamodel.TskException;
70 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
71 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments;
72 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment;
73 
79 public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
80 
81  private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
85  private Blackboard blackboard;
86  private CommunicationArtifactsHelper communicationArtifactsHelper;
87 
88  // A cache of custom attributes for the VcardParser unique to each ingest run, but consistent across threads.
89  private static ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache = new ConcurrentHashMap<>();
90  private static Object customAttributeCacheLock = new Object();
91 
92  private static final int MBOX_SIZE_TO_SPLIT = 1048576000;
93  private Case currentCase;
94 
99  }
100 
101  @Override
102  @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
103  public void startUp(IngestJobContext context) throws IngestModuleException {
104  this.context = context;
105 
106  synchronized(customAttributeCacheLock) {
107  if (!customAttributeCache.isEmpty()) {
108  customAttributeCache.clear();
109  }
110  }
111 
112  try {
113  currentCase = Case.getCurrentCaseThrows();
115  } catch (NoCurrentCaseException ex) {
116  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
117  throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noOpenCase_errMsg(), ex);
118  }
119  }
120 
121  @Override
122  public ProcessResult process(AbstractFile abstractFile) {
123 
124  blackboard = currentCase.getSleuthkitCase().getBlackboard();
125 
126  // skip known
127  if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) {
128  return ProcessResult.OK;
129  }
130 
131  //skip unalloc
132  if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS))
133  || (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
134  return ProcessResult.OK;
135  }
136 
137  if ((abstractFile.isFile() == false)) {
138  return ProcessResult.OK;
139  }
140 
141  // check its signature
142  boolean isMbox = false;
143  boolean isEMLFile = false;
144 
145  try {
146  byte[] t = new byte[64];
147  if (abstractFile.getSize() > 64) {
148  int byteRead = abstractFile.read(t, 0, 64);
149  if (byteRead > 0) {
150  isMbox = MboxParser.isValidMimeTypeMbox(t, abstractFile);
151  isEMLFile = EMLParser.isEMLFile(abstractFile, t);
152  }
153  }
154  } catch (TskException ex) {
155  logger.log(Level.WARNING, null, ex);
156  }
157 
158  boolean isPstFile = PstParser.isPstFile(abstractFile);
159  boolean isVcardFile = VcardParser.isVcardFile(abstractFile);
160 
161  if (context.fileIngestIsCancelled()) {
162  return ProcessResult.OK;
163  }
164 
165  if (isMbox || isEMLFile || isPstFile || isVcardFile) {
166  try {
167  communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(),
168  EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL, context.getJobId());
169  } catch (TskCoreException ex) {
170  logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file with object id = %d", abstractFile.getId()), ex);
171  return ProcessResult.ERROR;
172  }
173  }
174 
175  if (isMbox) {
176  return processMBox(abstractFile);
177  }
178 
179  if (isEMLFile) {
180  return processEMLFile(abstractFile);
181  }
182 
183  if (isPstFile) {
184  return processPst(abstractFile);
185  }
186 
187  if (isVcardFile) {
188  return processVcard(abstractFile);
189  }
190 
191  return ProcessResult.OK;
192  }
193 
201  @Messages({"ThunderbirdMboxFileIngestModule.processPst.indexError.message=Failed to index encryption detected artifact for keyword search."})
202  private ProcessResult processPst(AbstractFile abstractFile) {
203  String fileName;
204  try {
205  fileName = getTempPath() + File.separator + abstractFile.getName()
206  + "-" + String.valueOf(abstractFile.getId());
207  } catch (NoCurrentCaseException ex) {
208  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
209  return ProcessResult.ERROR;
210  }
211  File file = new File(fileName);
212 
213  long freeSpace = services.getFreeDiskSpace();
214  if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) {
215  logger.log(Level.WARNING, "Not enough disk space to write file to disk."); //NON-NLS
217  NbBundle.getMessage(this.getClass(),
218  "ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace",
219  abstractFile.getName()));
220  services.postMessage(msg);
221  return ProcessResult.OK;
222  }
223 
224  try (PstParser parser = new PstParser(services)) {
225  try {
226  ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled);
227  } catch (IOException ex) {
228  logger.log(Level.WARNING, "Failed writing pst file to disk.", ex); //NON-NLS
229  return ProcessResult.OK;
230  }
231 
232  PstParser.ParseResult result = parser.open(file, abstractFile.getId());
233 
234  switch (result) {
235  case OK:
236  Iterator<EmailMessage> pstMsgIterator = parser.getEmailMessageIterator();
237  if (pstMsgIterator != null) {
238  processEmails(parser.getPartialEmailMessages(), pstMsgIterator, abstractFile);
239  if (context.fileIngestIsCancelled()) {
240  return ProcessResult.OK;
241  }
242  } else {
243  // sometimes parser returns ParseResult=OK but there are no messages
244  postErrorMessage(
245  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
246  abstractFile.getName()),
247  NbBundle.getMessage(this.getClass(),
248  "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
249  logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
250  return ProcessResult.ERROR;
251  }
252  break;
253 
254  case ENCRYPT:
255  // encrypted pst: Add encrypted file artifact
256  try {
257 
258  String encryptionFileLevel = NbBundle.getMessage(this.getClass(),
259  "ThunderbirdMboxFileIngestModule.encryptionFileLevel");
260  BlackboardArtifact artifact = abstractFile.newAnalysisResult(
261  BlackboardArtifact.Type.TSK_ENCRYPTION_DETECTED,
262  Score.SCORE_NOTABLE, null, null, encryptionFileLevel, Arrays.asList(
263  new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
264  EmailParserModuleFactory.getModuleName(),
265  encryptionFileLevel)
266  ))
267  .getAnalysisResult();
268 
269  try {
270  // index the artifact for keyword search
271  blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName(), context.getJobId());
272  } catch (Blackboard.BlackboardException ex) {
273  MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName());
274  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
275  }
276  } catch (TskCoreException ex) {
277  logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
278  }
279  break;
280  default:
281  // parsing error: log message
282  postErrorMessage(
283  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
284  abstractFile.getName()),
285  NbBundle.getMessage(this.getClass(),
286  "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
287  logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
288  return ProcessResult.ERROR;
289  }
290  } catch (Exception ex) {
291  logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath()));
292  } finally {
293  file.delete();
294  }
295  return ProcessResult.OK;
296  }
297 
305  private ProcessResult processMBox(AbstractFile abstractFile) {
306  String mboxFileName = abstractFile.getName();
307  String mboxParentDir = abstractFile.getParentPath();
308  // use the local path to determine the e-mail folder structure
309  String emailFolder = "";
310  // email folder is everything after "Mail" or ImapMail
311  if (mboxParentDir.contains("/Mail/")) { //NON-NLS
312  emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/Mail/") + 5); //NON-NLS
313  } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS
314  emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS
315  }
316  emailFolder += mboxFileName;
317  emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS
318 
319  String fileName;
320  try {
321  fileName = getTempPath() + File.separator + abstractFile.getName()
322  + "-" + String.valueOf(abstractFile.getId());
323  } catch (NoCurrentCaseException ex) {
324  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
325  return ProcessResult.ERROR;
326  }
327  File file = new File(fileName);
328 
329  long freeSpace = services.getFreeDiskSpace();
330  if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) {
331  logger.log(Level.WARNING, "Not enough disk space to write file to disk."); //NON-NLS
332  postErrorMessage(
333  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg",
334  abstractFile.getName()),
335  NbBundle.getMessage(this.getClass(),
336  "ThunderbirdMboxFileIngestModule.processMBox.errProfFile.details"));
337  return ProcessResult.OK;
338  }
339 
340  if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) {
341 
342  try {
343  ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled);
344  } catch (IOException ex) {
345  logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS
346  return ProcessResult.OK;
347  }
348 
349  try {
350  processMboxFile(file, abstractFile, emailFolder);
351  if (context.fileIngestIsCancelled()) {
352  return ProcessResult.OK;
353  }
354  } finally {
355  file.delete();
356  }
357  } else {
358 
359  List<Long> mboxSplitOffsets = new ArrayList<>();
360  try {
361  mboxSplitOffsets = findMboxSplitOffset(abstractFile, file);
362  } catch (IOException ex) {
363  logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS
364  return ProcessResult.OK;
365  }
366 
367  long startingOffset = 0;
368  for (Long mboxSplitOffset : mboxSplitOffsets) {
369  File splitFile = new File(fileName + "-" + mboxSplitOffset);
370  try {
371  ContentUtils.writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset);
372  } catch (IOException ex) {
373  logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS
374  return ProcessResult.OK;
375  }
376  try {
377  processMboxFile(splitFile, abstractFile, emailFolder);
378  startingOffset = mboxSplitOffset;
379  } finally {
380  splitFile.delete();
381  }
382 
383  if (context.fileIngestIsCancelled()) {
384  return ProcessResult.OK;
385  }
386  }
387  }
388 
389  return ProcessResult.OK;
390  }
391 
392  private List<Long> findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException {
393 
394  List<Long> mboxSplitOffset = new ArrayList<>();
395 
396  byte[] buffer = new byte[7];
397  ReadContentInputStream in = new ReadContentInputStream(abstractFile);
398  in.skip(MBOX_SIZE_TO_SPLIT);
399  int len = in.read(buffer);
400  while (len != -1) {
401  len = in.read(buffer);
402  if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114
403  && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) {
404  mboxSplitOffset.add(in.getCurPosition() - 5);
405  in.skip(MBOX_SIZE_TO_SPLIT);
406  }
407  }
408 
409  return mboxSplitOffset;
410 
411  }
412 
413  private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) {
414 
415  try (MboxParser emailIterator = MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId())) {
416  List<EmailMessage> emails = new ArrayList<>();
417  if (emailIterator != null) {
418  while (emailIterator.hasNext()) {
419  if (context.fileIngestIsCancelled()) {
420  return;
421  }
422  EmailMessage emailMessage = emailIterator.next();
423  if (emailMessage != null) {
424  emails.add(emailMessage);
425  }
426  }
427 
428  String errors = emailIterator.getErrors();
429  if (!errors.isEmpty()) {
430  postErrorMessage(
431  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg2",
432  abstractFile.getName()), errors);
433  }
434  }
435  processEmails(emails, MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId()), abstractFile);
436  } catch (Exception ex) {
437  logger.log(Level.WARNING, String.format("Failed to close mbox temp file %s", file.getAbsolutePath()));
438  }
439 
440  }
441 
450  @Messages({
451  "# {0} - file name",
452  "# {1} - file ID",
453  "ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse."
454  })
455  private ProcessResult processVcard(AbstractFile abstractFile) {
456  try {
457  VcardParser parser = new VcardParser(currentCase, context, customAttributeCache);
458  parser.parse(abstractFile);
459  } catch (IOException | NoCurrentCaseException ex) {
460  logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS
461  return ProcessResult.OK;
462  }
463  return ProcessResult.OK;
464  }
465 
466  private ProcessResult processEMLFile(AbstractFile abstractFile) {
467  try {
468  EmailMessage message = EMLParser.parse(abstractFile);
469 
470  if (message == null) {
471  return ProcessResult.OK;
472  }
473 
474  List<AbstractFile> derivedFiles = new ArrayList<>();
475 
476  AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase);
477  createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles);
478  accountFileInstanceCache.clear();
479 
480  if (derivedFiles.isEmpty() == false) {
481  for (AbstractFile derived : derivedFiles) {
482  services.fireModuleContentEvent(new ModuleContentEvent(derived));
483  }
484  }
485  context.addFilesToJob(derivedFiles);
486 
487  } catch (IOException ex) {
488  logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex);
489  return ProcessResult.ERROR;
490  } catch (MimeException ex) {
491  logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex);
492  return ProcessResult.ERROR;
493  }
494 
495  return ProcessResult.OK;
496  }
497 
504  static String getTempPath() throws NoCurrentCaseException {
505  String tmpDir = Case.getCurrentCaseThrows().getTempDirectory() + File.separator
506  + "EmailParser"; //NON-NLS
507  File dir = new File(tmpDir);
508  if (dir.exists() == false) {
509  dir.mkdirs();
510  }
511  return tmpDir;
512  }
513 
521  static String getModuleOutputPath() throws NoCurrentCaseException {
522  String outDir = Case.getCurrentCaseThrows().getModuleDirectory() + File.separator
523  + EmailParserModuleFactory.getModuleName();
524  File dir = new File(outDir);
525  if (dir.exists() == false) {
526  dir.mkdirs();
527  }
528  return outDir;
529  }
530 
537  static String getRelModuleOutputPath() throws NoCurrentCaseException {
538  return Case.getCurrentCaseThrows().getModuleOutputDirectoryRelativePath() + File.separator
539  + EmailParserModuleFactory.getModuleName();
540  }
541 
550  private void processEmails(List<EmailMessage> partialEmailsForThreading, Iterator<EmailMessage> fullMessageIterator,
551  AbstractFile abstractFile) {
552 
553  // Create cache for accounts
554  AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase);
555 
556  // Putting try/catch around this to catch any exception and still allow
557  // the creation of the artifacts to continue.
558  try {
559  EmailMessageThreader.threadMessages(partialEmailsForThreading);
560  } catch (Exception ex) {
561  logger.log(Level.WARNING, String.format("Exception thrown parsing emails from %s", abstractFile.getName()), ex);
562  }
563 
564  List<AbstractFile> derivedFiles = new ArrayList<>();
565 
566  int msgCnt = 0;
567  while (fullMessageIterator.hasNext()) {
568  if (context.fileIngestIsCancelled()) {
569  return;
570  }
571 
572  EmailMessage current = fullMessageIterator.next();
573 
574  if (current == null) {
575  continue;
576  }
577 
578  if (partialEmailsForThreading.size() > msgCnt) {
579  EmailMessage threaded = partialEmailsForThreading.get(msgCnt++);
580 
581  if (threaded.getMessageID().equals(current.getMessageID())
582  && threaded.getSubject().equals(current.getSubject())) {
583  current.setMessageThreadID(threaded.getMessageThreadID());
584  }
585  }
586  createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles);
587  }
588 
589  if (derivedFiles.isEmpty() == false) {
590  for (AbstractFile derived : derivedFiles) {
591  if (context.fileIngestIsCancelled()) {
592  return;
593  }
594  services.fireModuleContentEvent(new ModuleContentEvent(derived));
595  }
596  }
597  context.addFilesToJob(derivedFiles);
598  }
599 
600  void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List<AbstractFile> derivedFiles) {
601  BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile, accountFileInstanceCache);
602 
603  if ((msgArtifact != null) && (email.hasAttachment())) {
604  derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact));
605 
606  for (EmailMessage.Attachment attach : email.getAttachments()) {
607  if (attach instanceof AttachedEmailMessage) {
608  createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles);
609  }
610  }
611  }
612  }
613 
624  @NbBundle.Messages({
625  "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message."
626  })
627  private List<AbstractFile> handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) {
628  List<AbstractFile> files = new ArrayList<>();
629  List<FileAttachment> fileAttachments = new ArrayList<>();
630  for (EmailMessage.Attachment attach : attachments) {
631  String filename = attach.getName();
632  long crTime = attach.getCrTime();
633  long mTime = attach.getmTime();
634  long aTime = attach.getaTime();
635  long cTime = attach.getcTime();
636  String relPath = attach.getLocalPath();
637  long size = attach.getSize();
638  TskData.EncodingType encodingType = attach.getEncodingType();
639 
640  try {
641  DerivedFile df = fileManager.addDerivedFile(filename, relPath,
642  size, cTime, crTime, aTime, mTime, true, abstractFile, "",
643  EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType);
644 
645  files.add(df);
646 
647  fileAttachments.add(new FileAttachment(df));
648  } catch (TskCoreException ex) {
649  postErrorMessage(
650  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.errMsg",
651  abstractFile.getName()),
652  NbBundle.getMessage(this.getClass(),
653  "ThunderbirdMboxFileIngestModule.handleAttch.errMsg.details", filename));
654  logger.log(Level.INFO, "", ex);
655  }
656  }
657 
658  try {
659  communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList()));
660  } catch (TskCoreException ex) {
661  postErrorMessage(
662  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"),
663  "");
664  logger.log(Level.INFO, "Failed to add attachments to email message.", ex);
665  }
666 
667  return files;
668  }
669 
678  private Set<String> findEmailAddresess(String input) {
679  Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b",
680  Pattern.CASE_INSENSITIVE);
681  Matcher m = p.matcher(input);
682  Set<String> emailAddresses = new HashSet<>();
683  while (m.find()) {
684  emailAddresses.add(m.group());
685  }
686  return emailAddresses;
687  }
688 
698  @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."})
699  private BlackboardArtifact addEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache) {
700  BlackboardArtifact bbart = null;
701  List<BlackboardAttribute> bbattributes = new ArrayList<>();
702  String to = email.getRecipients();
703  String cc = email.getCc();
704  String bcc = email.getBcc();
705  String from = email.getSender();
706  long dateL = email.getSentDate();
707  String headers = email.getHeaders();
708  String body = email.getTextBody();
709  String bodyHTML = email.getHtmlBody();
710  String rtf = email.getRtfBody();
711  String subject = email.getSubject();
712  long id = email.getId();
713  String localPath = email.getLocalPath();
714  String threadID = email.getMessageThreadID();
715 
716  List<String> senderAddressList = new ArrayList<>();
717  String senderAddress;
718  senderAddressList.addAll(findEmailAddresess(from));
719 
720  if (context.fileIngestIsCancelled()) {
721  return null;
722  }
723 
724  AccountFileInstance senderAccountInstance = null;
725 
726  if (senderAddressList.size() == 1) {
727  senderAddress = senderAddressList.get(0);
728  try {
729  senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress, context);
730  } catch (TskCoreException ex) {
731  logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS
732  }
733  } else {
734  logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS
735  }
736 
737  if (context.fileIngestIsCancelled()) {
738  return null;
739  }
740 
741  List<String> recipientAddresses = new ArrayList<>();
742  recipientAddresses.addAll(findEmailAddresess(to));
743  recipientAddresses.addAll(findEmailAddresess(cc));
744  recipientAddresses.addAll(findEmailAddresess(bcc));
745 
746  List<AccountFileInstance> recipientAccountInstances = new ArrayList<>();
747  for (String addr : recipientAddresses) {
748  if (context.fileIngestIsCancelled()) {
749  return null;
750  }
751  try {
752  AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr, context);
753  recipientAccountInstances.add(recipientAccountInstance);
754  } catch (TskCoreException ex) {
755  logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS
756  }
757  }
758 
759  addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes);
760  addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes);
761  addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes);
762  addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes);
763 
764  addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes);
765  addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes);
766 
767  addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes);
768 
769  addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)),
770  ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes);
771 
772  try {
773  addArtifactAttribute((email.hasAttachment() ? "Yes" : ""),
774  blackboard.getOrAddAttributeType("EMAIL_HAS_ATTACHMENT", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Has Attachments"),
775  bbattributes);
776  } catch (Blackboard.BlackboardException ex) {
777  logger.log(Level.SEVERE, "Unable to create EMAIL_HAS_ATTACHMENT attribute" , ex); //NON-NLS
778  }
779 
780  addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""),
781  ATTRIBUTE_TYPE.TSK_PATH, bbattributes);
782 
783  addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes);
784  addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes);
785  addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes);
786  addArtifactAttribute(threadID, ATTRIBUTE_TYPE.TSK_THREAD_ID, bbattributes);
787 
788  try {
789  if (context.fileIngestIsCancelled()) {
790  return null;
791  }
792 
793  bbart = abstractFile.newDataArtifact(
794  new BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG),
795  bbattributes);
796 
797  if (context.fileIngestIsCancelled()) {
798  return null;
799  }
800 
801  // Add account relationships
802  currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL);
803 
804  if (context.fileIngestIsCancelled()) {
805  return null;
806  }
807 
808  try {
809  // index the artifact for keyword search
810  blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName(), context.getJobId());
811  } catch (Blackboard.BlackboardException ex) {
812  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS
813  MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_addArtifact_indexError_message(), bbart.getDisplayName());
814  }
815  } catch (TskCoreException | TskDataException ex) {
816  logger.log(Level.WARNING, null, ex);
817  }
818 
819  return bbart;
820  }
821 
829  static void addArtifactAttribute(String stringVal, BlackboardAttribute.Type attrType, Collection<BlackboardAttribute> bbattributes) {
830  if (stringVal.isEmpty() == false) {
831  bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal));
832  }
833  }
834 
842  static void addArtifactAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
843  if (stringVal.isEmpty() == false) {
844  bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal));
845  }
846  }
847 
855  static void addArtifactAttribute(long longVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
856  if (longVal > 0) {
857  bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal));
858  }
859  }
860 
866  static private class AccountFileInstanceCache {
867 
868  private final Map<String, AccountFileInstance> cacheMap;
869  private final AbstractFile file;
870  private final Case currentCase;
871 
878  AccountFileInstanceCache(AbstractFile file, Case currentCase) {
879  cacheMap = new HashMap<>();
880  this.file = file;
881  this.currentCase = currentCase;
882  }
883 
894  AccountFileInstance getAccountInstance(String email, IngestJobContext context) throws TskCoreException {
895  if (cacheMap.containsKey(email)) {
896  return cacheMap.get(email);
897  }
898 
899  AccountFileInstance accountInstance
900  = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email,
901  EmailParserModuleFactory.getModuleName(), file, null, context.getJobId());
902  cacheMap.put(email, accountInstance);
903  return accountInstance;
904  }
905 
909  void clear() {
910  cacheMap.clear();
911  }
912  }
913 
920  void postErrorMessage(String subj, String details) {
921  IngestMessage ingestMessage = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleVersion(), subj, details);
922  services.postMessage(ingestMessage);
923  }
924 
930  IngestServices getServices() {
931  return services;
932  }
933 
934  @Override
935  public void shutDown() {
936  synchronized(customAttributeCacheLock) {
937  if (!customAttributeCache.isEmpty()) {
938  customAttributeCache.clear();
939  }
940  }
941  }
942 
943 }
static ConcurrentMap< String, BlackboardAttribute.Type > customAttributeCache
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void processEmails(List< EmailMessage > partialEmailsForThreading, Iterator< EmailMessage > fullMessageIterator, AbstractFile abstractFile)
static< T > long writeToFile(Content content, java.io.File outputFile, ProgressHandle progress, Future< T > worker, boolean source)
void addFilesToJob(List< AbstractFile > files)
void postMessage(final IngestMessage message)
BlackboardArtifact addEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache)
void fireModuleContentEvent(ModuleContentEvent moduleContentEvent)
void processMboxFile(File file, AbstractFile abstractFile, String emailFolder)
static void error(String title, String message)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
List< AbstractFile > handleAttachments(List< EmailMessage.Attachment > attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact)
DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
static synchronized IngestServices getInstance()

Copyright © 2012-2022 Basis Technology. Generated on: Tue Aug 1 2023
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.