19 package org.sleuthkit.autopsy.thunderbirdparser;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.Collection;
26 import java.util.Collections;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.Iterator;
30 import java.util.List;
33 import java.util.concurrent.ConcurrentHashMap;
34 import java.util.concurrent.ConcurrentMap;
35 import java.util.logging.Level;
36 import java.util.regex.Matcher;
37 import java.util.regex.Pattern;
38 import org.apache.james.mime4j.MimeException;
39 import org.openide.util.NbBundle;
40 import org.openide.util.NbBundle.Messages;
61 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
70 import org.
sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
71 import org.
sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments;
72 import org.
sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment;
89 private static ConcurrentMap<String, BlackboardAttribute.Type>
customAttributeCache =
new ConcurrentHashMap<>();
102 @Messages({
"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
116 logger.log(Level.SEVERE,
"Exception while getting open case.", ex);
127 if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) {
132 if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS))
133 || (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
137 if ((abstractFile.isFile() ==
false)) {
142 boolean isMbox =
false;
143 boolean isEMLFile =
false;
146 byte[] t =
new byte[64];
147 if (abstractFile.getSize() > 64) {
148 int byteRead = abstractFile.read(t, 0, 64);
150 isMbox = MboxParser.isValidMimeTypeMbox(t, abstractFile);
151 isEMLFile = EMLParser.isEMLFile(abstractFile, t);
154 }
catch (TskException ex) {
155 logger.log(Level.WARNING, null, ex);
158 boolean isPstFile = PstParser.isPstFile(abstractFile);
159 boolean isVcardFile = VcardParser.isVcardFile(abstractFile);
165 if (isMbox || isEMLFile || isPstFile || isVcardFile) {
167 communicationArtifactsHelper =
new CommunicationArtifactsHelper(currentCase.
getSleuthkitCase(),
169 }
catch (TskCoreException ex) {
170 logger.log(Level.SEVERE, String.format(
"Failed to create CommunicationArtifactsHelper for file with object id = %d", abstractFile.getId()), ex);
201 @Messages({
"ThunderbirdMboxFileIngestModule.processPst.indexError.message=Failed to index encryption detected artifact for keyword search."})
205 fileName = getTempPath() + File.separator + abstractFile.getName()
206 +
"-" + String.valueOf(abstractFile.getId());
208 logger.log(Level.SEVERE,
"Exception while getting open case.", ex);
211 File file =
new File(fileName);
215 logger.log(Level.WARNING,
"Not enough disk space to write file to disk.");
217 NbBundle.getMessage(this.getClass(),
218 "ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace",
219 abstractFile.getName()));
224 try (PstParser parser =
new PstParser(services)) {
227 }
catch (IOException ex) {
228 logger.log(Level.WARNING,
"Failed writing pst file to disk.", ex);
232 PstParser.ParseResult result = parser.open(file, abstractFile.getId());
236 Iterator<EmailMessage> pstMsgIterator = parser.getEmailMessageIterator();
237 if (pstMsgIterator != null) {
238 processEmails(parser.getPartialEmailMessages(), pstMsgIterator, abstractFile);
245 NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
246 abstractFile.getName()),
247 NbBundle.getMessage(
this.getClass(),
248 "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
249 logger.log(Level.INFO,
"PSTParser failed to parse {0}", abstractFile.getName());
258 String encryptionFileLevel = NbBundle.getMessage(this.getClass(),
259 "ThunderbirdMboxFileIngestModule.encryptionFileLevel");
260 BlackboardArtifact artifact = abstractFile.newAnalysisResult(
261 BlackboardArtifact.Type.TSK_ENCRYPTION_DETECTED,
262 Score.SCORE_NOTABLE, null, null, encryptionFileLevel, Arrays.asList(
263 new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
267 .getAnalysisResult();
272 }
catch (Blackboard.BlackboardException ex) {
274 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + artifact.getArtifactID(), ex);
276 }
catch (TskCoreException ex) {
277 logger.log(Level.INFO,
"Failed to add encryption attribute to file: {0}", abstractFile.getName());
283 NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
284 abstractFile.getName()),
285 NbBundle.getMessage(
this.getClass(),
286 "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
287 logger.log(Level.INFO,
"PSTParser failed to parse {0}", abstractFile.getName());
290 }
catch (Exception ex) {
291 logger.log(Level.WARNING, String.format(
"Failed to close temp pst file %s", file.getAbsolutePath()));
306 String mboxFileName = abstractFile.getName();
307 String mboxParentDir = abstractFile.getParentPath();
309 String emailFolder =
"";
311 if (mboxParentDir.contains(
"/Mail/")) {
312 emailFolder = mboxParentDir.substring(mboxParentDir.indexOf(
"/Mail/") + 5);
313 }
else if (mboxParentDir.contains(
"/ImapMail/")) {
314 emailFolder = mboxParentDir.substring(mboxParentDir.indexOf(
"/ImapMail/") + 9);
316 emailFolder += mboxFileName;
317 emailFolder = emailFolder.replaceAll(
".sbd",
"");
321 fileName = getTempPath() + File.separator + abstractFile.getName()
322 +
"-" + String.valueOf(abstractFile.getId());
324 logger.log(Level.SEVERE,
"Exception while getting open case.", ex);
327 File file =
new File(fileName);
331 logger.log(Level.WARNING,
"Not enough disk space to write file to disk.");
333 NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg",
334 abstractFile.getName()),
335 NbBundle.getMessage(
this.getClass(),
336 "ThunderbirdMboxFileIngestModule.processMBox.errProfFile.details"));
344 }
catch (IOException ex) {
345 logger.log(Level.WARNING,
"Failed writing mbox file to disk.", ex);
359 List<Long> mboxSplitOffsets =
new ArrayList<>();
362 }
catch (IOException ex) {
363 logger.log(Level.WARNING, String.format(
"Failed finding split offsets for mbox file {0}.", fileName), ex);
367 long startingOffset = 0;
368 for (Long mboxSplitOffset : mboxSplitOffsets) {
369 File splitFile =
new File(fileName +
"-" + mboxSplitOffset);
371 ContentUtils.
writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset);
372 }
catch (IOException ex) {
373 logger.log(Level.WARNING,
"Failed writing split mbox file to disk.", ex);
378 startingOffset = mboxSplitOffset;
394 List<Long> mboxSplitOffset =
new ArrayList<>();
396 byte[] buffer =
new byte[7];
397 ReadContentInputStream in =
new ReadContentInputStream(abstractFile);
398 in.skip(MBOX_SIZE_TO_SPLIT);
399 int len = in.read(buffer);
401 len = in.read(buffer);
402 if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114
403 && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) {
404 mboxSplitOffset.add(in.getCurPosition() - 5);
405 in.skip(MBOX_SIZE_TO_SPLIT);
409 return mboxSplitOffset;
413 private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) {
415 try (MboxParser emailIterator = MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId())) {
416 List<EmailMessage> emails =
new ArrayList<>();
417 if (emailIterator != null) {
418 while (emailIterator.hasNext()) {
422 EmailMessage emailMessage = emailIterator.next();
423 if (emailMessage != null) {
424 emails.add(emailMessage);
428 String errors = emailIterator.getErrors();
429 if (!errors.isEmpty()) {
431 NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg2",
432 abstractFile.getName()), errors);
435 processEmails(emails, MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId()), abstractFile);
436 }
catch (Exception ex) {
437 logger.log(Level.WARNING, String.format(
"Failed to close mbox temp file %s", file.getAbsolutePath()));
453 "ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse."
458 parser.parse(abstractFile);
460 logger.log(Level.WARNING, String.format(
"Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex);
468 EmailMessage message = EMLParser.parse(abstractFile);
470 if (message == null) {
474 List<AbstractFile> derivedFiles =
new ArrayList<>();
477 createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles);
478 accountFileInstanceCache.clear();
480 if (derivedFiles.isEmpty() ==
false) {
481 for (AbstractFile derived : derivedFiles) {
487 }
catch (IOException ex) {
488 logger.log(Level.WARNING, String.format(
"Error reading eml file %s", abstractFile.getName()), ex);
490 }
catch (MimeException ex) {
491 logger.log(Level.WARNING, String.format(
"Error reading eml file %s", abstractFile.getName()), ex);
507 File dir =
new File(tmpDir);
508 if (dir.exists() ==
false) {
521 static String getModuleOutputPath() throws NoCurrentCaseException {
522 String outDir = Case.getCurrentCaseThrows().getModuleDirectory() + File.separator
523 + EmailParserModuleFactory.getModuleName();
524 File dir =
new File(outDir);
525 if (dir.exists() ==
false) {
537 static String getRelModuleOutputPath() throws NoCurrentCaseException {
538 return Case.getCurrentCaseThrows().getModuleOutputDirectoryRelativePath() + File.separator
539 + EmailParserModuleFactory.getModuleName();
550 private void processEmails(List<EmailMessage> partialEmailsForThreading, Iterator<EmailMessage> fullMessageIterator,
551 AbstractFile abstractFile) {
559 EmailMessageThreader.threadMessages(partialEmailsForThreading);
560 }
catch (Exception ex) {
561 logger.log(Level.WARNING, String.format(
"Exception thrown parsing emails from %s", abstractFile.getName()), ex);
564 List<AbstractFile> derivedFiles =
new ArrayList<>();
567 while (fullMessageIterator.hasNext()) {
572 EmailMessage current = fullMessageIterator.next();
574 if (current == null) {
578 if (partialEmailsForThreading.size() > msgCnt) {
579 EmailMessage threaded = partialEmailsForThreading.get(msgCnt++);
581 if (threaded.getMessageID().equals(current.getMessageID())
582 && threaded.getSubject().equals(current.getSubject())) {
583 current.setMessageThreadID(threaded.getMessageThreadID());
586 createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles);
589 if (derivedFiles.isEmpty() ==
false) {
590 for (AbstractFile derived : derivedFiles) {
600 void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List<AbstractFile> derivedFiles) {
601 BlackboardArtifact msgArtifact =
addEmailArtifact(email, abstractFile, accountFileInstanceCache);
603 if ((msgArtifact != null) && (email.hasAttachment())) {
604 derivedFiles.addAll(
handleAttachments(email.getAttachments(), abstractFile, msgArtifact));
606 for (EmailMessage.Attachment attach : email.getAttachments()) {
607 if (attach instanceof AttachedEmailMessage) {
608 createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles);
625 "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message."
627 private List<AbstractFile>
handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) {
628 List<AbstractFile> files =
new ArrayList<>();
629 List<FileAttachment> fileAttachments =
new ArrayList<>();
630 for (EmailMessage.Attachment attach : attachments) {
631 String filename = attach.getName();
632 long crTime = attach.getCrTime();
633 long mTime = attach.getmTime();
634 long aTime = attach.getaTime();
635 long cTime = attach.getcTime();
636 String relPath = attach.getLocalPath();
637 long size = attach.getSize();
638 TskData.EncodingType encodingType = attach.getEncodingType();
642 size, cTime, crTime, aTime, mTime,
true, abstractFile,
"",
647 fileAttachments.add(
new FileAttachment(df));
648 }
catch (TskCoreException ex) {
650 NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.handleAttch.errMsg",
651 abstractFile.getName()),
652 NbBundle.getMessage(
this.getClass(),
653 "ThunderbirdMboxFileIngestModule.handleAttch.errMsg.details", filename));
654 logger.log(Level.INFO,
"", ex);
659 communicationArtifactsHelper.addAttachments(messageArtifact,
new MessageAttachments(fileAttachments, Collections.emptyList()));
660 }
catch (TskCoreException ex) {
662 NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"),
664 logger.log(Level.INFO,
"Failed to add attachments to email message.", ex);
679 Pattern p = Pattern.compile(
"\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b",
680 Pattern.CASE_INSENSITIVE);
681 Matcher m = p.matcher(input);
682 Set<String> emailAddresses =
new HashSet<>();
684 emailAddresses.add(m.group());
686 return emailAddresses;
698 @Messages({
"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."})
700 BlackboardArtifact bbart = null;
701 List<BlackboardAttribute> bbattributes =
new ArrayList<>();
702 String to = email.getRecipients();
703 String cc = email.getCc();
704 String bcc = email.getBcc();
705 String from = email.getSender();
706 long dateL = email.getSentDate();
707 String headers = email.getHeaders();
708 String body = email.getTextBody();
709 String bodyHTML = email.getHtmlBody();
710 String rtf = email.getRtfBody();
711 String subject = email.getSubject();
712 long id = email.getId();
713 String localPath = email.getLocalPath();
714 String threadID = email.getMessageThreadID();
716 List<String> senderAddressList =
new ArrayList<>();
717 String senderAddress;
724 AccountFileInstance senderAccountInstance = null;
726 if (senderAddressList.size() == 1) {
727 senderAddress = senderAddressList.get(0);
729 senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress, context);
730 }
catch (TskCoreException ex) {
731 logger.log(Level.WARNING,
"Failed to create account for email address " + senderAddress, ex);
734 logger.log(Level.WARNING,
"Failed to find sender address, from = {0}", from);
741 List<String> recipientAddresses =
new ArrayList<>();
746 List<AccountFileInstance> recipientAccountInstances =
new ArrayList<>();
747 for (String addr : recipientAddresses) {
752 AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr, context);
753 recipientAccountInstances.add(recipientAccountInstance);
754 }
catch (TskCoreException ex) {
755 logger.log(Level.WARNING,
"Failed to create account for email address " + addr, ex);
759 addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes);
760 addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes);
761 addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes);
762 addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes);
764 addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes);
765 addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes);
767 addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes);
769 addArtifactAttribute(((
id < 0L) ? NbBundle.getMessage(
this.getClass(),
"ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(
id)),
770 ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes);
773 addArtifactAttribute((email.hasAttachment() ?
"Yes" :
""),
774 blackboard.getOrAddAttributeType(
"EMAIL_HAS_ATTACHMENT", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING,
"Has Attachments"),
776 }
catch (Blackboard.BlackboardException ex) {
777 logger.log(Level.SEVERE,
"Unable to create EMAIL_HAS_ATTACHMENT attribute" , ex);
780 addArtifactAttribute(((localPath.isEmpty() ==
false) ? localPath :
""),
781 ATTRIBUTE_TYPE.TSK_PATH, bbattributes);
783 addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes);
784 addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes);
785 addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes);
786 addArtifactAttribute(threadID, ATTRIBUTE_TYPE.TSK_THREAD_ID, bbattributes);
793 bbart = abstractFile.newDataArtifact(
794 new BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG),
802 currentCase.
getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL);
811 }
catch (Blackboard.BlackboardException ex) {
812 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + bbart.getArtifactID(), ex);
815 }
catch (TskCoreException | TskDataException ex) {
816 logger.log(Level.WARNING, null, ex);
829 static void addArtifactAttribute(String stringVal, BlackboardAttribute.Type attrType, Collection<BlackboardAttribute> bbattributes) {
830 if (stringVal.isEmpty() ==
false) {
842 static void addArtifactAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
843 if (stringVal.isEmpty() ==
false) {
844 bbattributes.add(
new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal));
855 static void addArtifactAttribute(
long longVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
857 bbattributes.add(
new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal));
868 private final Map<String, AccountFileInstance>
cacheMap;
869 private final AbstractFile
file;
879 cacheMap =
new HashMap<>();
894 AccountFileInstance getAccountInstance(String email,
IngestJobContext context)
throws TskCoreException {
895 if (cacheMap.containsKey(email)) {
896 return cacheMap.get(email);
899 AccountFileInstance accountInstance
900 = currentCase.
getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email,
902 cacheMap.put(email, accountInstance);
903 return accountInstance;
920 void postErrorMessage(String subj, String details) {
921 IngestMessage ingestMessage = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleVersion(), subj, details);
930 IngestServices getServices() {
List< Long > findMboxSplitOffset(AbstractFile abstractFile, File file)
static ConcurrentMap< String, BlackboardAttribute.Type > customAttributeCache
Set< String > findEmailAddresess(String input)
static final Logger logger
FileManager getFileManager()
String getTempDirectory()
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void processEmails(List< EmailMessage > partialEmailsForThreading, Iterator< EmailMessage > fullMessageIterator, AbstractFile abstractFile)
final IngestServices services
static< T > long writeToFile(Content content, java.io.File outputFile, ProgressHandle progress, Future< T > worker, boolean source)
final Map< String, AccountFileInstance > cacheMap
ProcessResult processMBox(AbstractFile abstractFile)
static Object customAttributeCacheLock
static final int DISK_FREE_SPACE_UNKNOWN
ProcessResult processVcard(AbstractFile abstractFile)
ProcessResult processEMLFile(AbstractFile abstractFile)
void addFilesToJob(List< AbstractFile > files)
void postMessage(final IngestMessage message)
ProcessResult process(AbstractFile abstractFile)
SleuthkitCase getSleuthkitCase()
boolean fileIngestIsCancelled()
BlackboardArtifact addEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache)
ProcessResult processPst(AbstractFile abstractFile)
void startUp(IngestJobContext context)
void fireModuleContentEvent(ModuleContentEvent moduleContentEvent)
void processMboxFile(File file, AbstractFile abstractFile, String emailFolder)
static final int MBOX_SIZE_TO_SPLIT
static void error(String title, String message)
synchronized static Logger getLogger(String name)
static Case getCurrentCaseThrows()
List< AbstractFile > handleAttachments(List< EmailMessage.Attachment > attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact)
CommunicationArtifactsHelper communicationArtifactsHelper
DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
static synchronized IngestServices getInstance()