Autopsy  4.19.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
PstParser.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2019 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.thunderbirdparser;
20 
21 import com.google.common.collect.Iterables;
22 import com.pff.PSTAttachment;
23 import com.pff.PSTException;
24 import com.pff.PSTFile;
25 import com.pff.PSTFolder;
26 import com.pff.PSTMessage;
27 import java.io.File;
28 import java.io.FileOutputStream;
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.io.RandomAccessFile;
32 import java.nio.ByteBuffer;
33 import java.util.ArrayList;
34 import java.util.Iterator;
35 import java.util.List;
36 import java.util.Scanner;
37 import java.util.logging.Level;
39 import org.openide.util.NbBundle;
44 import static org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule.getRelModuleOutputPath;
45 import org.sleuthkit.datamodel.AbstractFile;
46 import org.sleuthkit.datamodel.EncodedFileOutputStream;
47 import org.sleuthkit.datamodel.TskCoreException;
48 import org.sleuthkit.datamodel.TskData;
49 
55 class PstParser implements AutoCloseable{
56 
57  private static final Logger logger = Logger.getLogger(PstParser.class.getName());
61  private static int PST_HEADER = 0x2142444E;
62 
63  private final IngestServices services;
64 
65  private PSTFile pstFile;
66  private long fileID;
67 
68  private int failureCount = 0;
69 
70  private final List<String> errorList = new ArrayList<>();
71 
72  PstParser(IngestServices services) {
73  this.services = services;
74  }
75 
76  enum ParseResult {
77 
78  OK, ERROR, ENCRYPT;
79  }
80 
96  ParseResult open(File file, long fileID) {
97  if (file == null) {
98  return ParseResult.ERROR;
99  }
100 
101  try {
102  pstFile = new PSTFile(file);
103  } catch (PSTException ex) {
104  // This is the message thrown from the PSTFile constructor if it
105  // detects that the file is encrypted.
106  if (ex.getMessage().equals("Only unencrypted and compressable PST files are supported at this time")) { //NON-NLS
107  logger.log(Level.INFO, "Found encrypted PST file."); //NON-NLS
108  return ParseResult.ENCRYPT;
109  }
110  if (ex.getMessage().toLowerCase().startsWith("unable to")) {
111  logger.log(Level.WARNING, ex.getMessage());
112  logger.log(Level.WARNING, String.format("Error in parsing PST file %s, file may be empty or corrupt", file.getName()));
113  return ParseResult.ERROR;
114  }
115  String msg = file.getName() + ": Failed to create internal java-libpst PST file to parse:\n" + ex.getMessage(); //NON-NLS
116  logger.log(Level.WARNING, msg, ex);
117  return ParseResult.ERROR;
118  } catch (IOException ex) {
119  String msg = file.getName() + ": Failed to create internal java-libpst PST file to parse:\n" + ex.getMessage(); //NON-NLS
120  logger.log(Level.WARNING, msg, ex);
121  return ParseResult.ERROR;
122  } catch (IllegalArgumentException ex) { // Not sure if this is true, was in previous version of code.
123  logger.log(Level.INFO, "Found encrypted PST file."); //NON-NLS
124  return ParseResult.ENCRYPT;
125  }
126 
127  return ParseResult.OK;
128  }
129 
130  @Override
131  public void close() throws IOException{
132  if(pstFile != null) {
133  RandomAccessFile file = pstFile.getFileHandle();
134  if(file != null) {
135  file.close();
136  }
137  }
138  }
139 
146  Iterator<EmailMessage> getEmailMessageIterator() {
147  if (pstFile == null) {
148  return null;
149  }
150 
151  Iterable<EmailMessage> iterable = null;
152 
153  try {
154  iterable = getEmailMessageIterator(pstFile.getRootFolder(), "\\", fileID, true);
155  } catch (PSTException | IOException ex) {
156  logger.log(Level.WARNING, String.format("Exception thrown while parsing fileID: %d", fileID), ex);
157  }
158 
159  if (iterable == null) {
160  return null;
161  }
162 
163  return iterable.iterator();
164  }
165 
172  List<EmailMessage> getPartialEmailMessages() {
173  List<EmailMessage> messages = new ArrayList<>();
174  Iterator<EmailMessage> iterator = getPartialEmailMessageIterator();
175  if (iterator != null) {
176  while (iterator.hasNext()) {
177  messages.add(iterator.next());
178  }
179  }
180 
181  return messages;
182  }
183 
189  String getErrors() {
190  String result = "";
191  for (String msg: errorList) {
192  result += "<li>" + msg + "</li>";
193  }
194  return result;
195  }
196 
202  int getFailureCount() {
203  return failureCount;
204  }
205 
213  private Iterator<EmailMessage> getPartialEmailMessageIterator() {
214  if (pstFile == null) {
215  return null;
216  }
217 
218  Iterable<EmailMessage> iterable = null;
219 
220  try {
221  iterable = getEmailMessageIterator(pstFile.getRootFolder(), "\\", fileID, false);
222  } catch (PSTException | IOException ex) {
223  logger.log(Level.WARNING, String.format("Exception thrown while parsing fileID: %d", fileID), ex);
224  }
225 
226  if (iterable == null) {
227  return null;
228  }
229 
230  return iterable.iterator();
231  }
232 
247  private Iterable<EmailMessage> getEmailMessageIterator(PSTFolder folder, String path, long fileID, boolean wholeMsg) throws PSTException, IOException {
248  Iterable<EmailMessage> iterable = null;
249 
250  if (folder.getContentCount() > 0) {
251  iterable = new PstEmailIterator(folder, path, fileID, wholeMsg).getIterable();
252  }
253 
254  if (folder.hasSubfolders()) {
255  List<PSTFolder> subFolders = folder.getSubFolders();
256  for (PSTFolder subFolder : subFolders) {
257  String newpath = path + "\\" + subFolder.getDisplayName();
258  Iterable<EmailMessage> subIterable = getEmailMessageIterator(subFolder, newpath, fileID, wholeMsg);
259  if (subIterable == null) {
260  continue;
261  }
262 
263  if (iterable != null) {
264  iterable = Iterables.concat(iterable, subIterable);
265  } else {
266  iterable = subIterable;
267  }
268 
269  }
270  }
271 
272  return iterable;
273  }
274 
283  private EmailMessage extractEmailMessage(PSTMessage msg, String localPath, long fileID) {
284  EmailMessage email = new EmailMessage();
285  String toAddress = msg.getDisplayTo();
286  String ccAddress = msg.getDisplayCC();
287  String bccAddress = msg.getDisplayBCC();
288  String receivedByName = msg.getReceivedByName();
289  String receivedBySMTPAddress = msg.getReceivedBySMTPAddress();
290 
291  if (toAddress.contains(receivedByName)) {
292  toAddress = toAddress.replace(receivedByName, receivedBySMTPAddress);
293  }
294  if (ccAddress.contains(receivedByName)) {
295  ccAddress = ccAddress.replace(receivedByName, receivedBySMTPAddress);
296  }
297  if (bccAddress.contains(receivedByName)) {
298  bccAddress = bccAddress.replace(receivedByName, receivedBySMTPAddress);
299  }
300  email.setRecipients(toAddress);
301  email.setCc(ccAddress);
302  email.setBcc(bccAddress);
303  email.setSender(getSender(msg.getSenderName(), msg.getSentRepresentingSMTPAddress()));
304  email.setSentDate(msg.getMessageDeliveryTime());
305  email.setTextBody(msg.getBody());
306  if (false == msg.getTransportMessageHeaders().isEmpty()) {
307  email.setHeaders("\n-----HEADERS-----\n\n" + msg.getTransportMessageHeaders() + "\n\n---END HEADERS--\n\n");
308  }
309  email.setHtmlBody(msg.getBodyHTML());
310  String rtf = "";
311  try {
312  rtf = msg.getRTFBody();
313  } catch (PSTException | IOException ex) {
314  logger.log(Level.INFO, "Failed to get RTF content from pst email."); //NON-NLS
315  }
316  email.setRtfBody(rtf);
317  email.setLocalPath(localPath);
318  email.setSubject(msg.getSubject());
319  email.setId(msg.getDescriptorNodeId());
320  email.setMessageID(msg.getInternetMessageId());
321 
322  String inReplyToID = msg.getInReplyToId();
323  email.setInReplyToID(inReplyToID);
324 
325  if (msg.hasAttachments()) {
326  extractAttachments(email, msg, fileID);
327  }
328 
329  List<String> references = extractReferences(msg.getTransportMessageHeaders());
330  if (inReplyToID != null && !inReplyToID.isEmpty()) {
331  if (references == null) {
332  references = new ArrayList<>();
333  references.add(inReplyToID);
334  } else if (!references.contains(inReplyToID)) {
335  references.add(inReplyToID);
336  }
337  }
338  email.setReferences(references);
339 
340  return email;
341  }
342 
350  private EmailMessage extractPartialEmailMessage(PSTMessage msg) {
351  EmailMessage email = new EmailMessage();
352  email.setSubject(msg.getSubject());
353  email.setId(msg.getDescriptorNodeId());
354  email.setMessageID(msg.getInternetMessageId());
355  String inReplyToID = msg.getInReplyToId();
356  email.setInReplyToID(inReplyToID);
357  List<String> references = extractReferences(msg.getTransportMessageHeaders());
358  if (inReplyToID != null && !inReplyToID.isEmpty()) {
359  if (references == null) {
360  references = new ArrayList<>();
361  references.add(inReplyToID);
362  } else if (!references.contains(inReplyToID)) {
363  references.add(inReplyToID);
364  }
365  }
366  email.setReferences(references);
367 
368  return email;
369  }
370 
377  @NbBundle.Messages({"PstParser.noOpenCase.errMsg=Exception while getting open case."})
378  private void extractAttachments(EmailMessage email, PSTMessage msg, long fileID) {
379  int numberOfAttachments = msg.getNumberOfAttachments();
380  String outputDirPath;
381  try {
382  outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator;
383  } catch (NoCurrentCaseException ex) {
384  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
385  return;
386  }
387  for (int x = 0; x < numberOfAttachments; x++) {
388  String filename = "";
389  try {
390  PSTAttachment attach = msg.getAttachment(x);
391  long size = attach.getAttachSize();
392  long freeSpace = services.getFreeDiskSpace();
393  if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (size >= freeSpace)) {
394  continue;
395  }
396  // both long and short filenames can be used for attachments
397  filename = attach.getLongFilename();
398  if (filename.isEmpty()) {
399  filename = attach.getFilename();
400  }
401  String uniqueFilename = fileID + "-" + msg.getDescriptorNodeId() + "-" + attach.getContentId() + "-" + FileUtil.escapeFileName(filename);
402  String outPath = outputDirPath + uniqueFilename;
403  saveAttachmentToDisk(attach, outPath);
404 
405  EmailMessage.Attachment attachment = new EmailMessage.Attachment();
406 
407  long crTime = attach.getCreationTime() != null ? attach.getCreationTime().getTime() / 1000 : 0;
408  long mTime = attach.getModificationTime() != null ? attach.getModificationTime().getTime() / 1000 : 0;
409  String relPath = getRelModuleOutputPath() + File.separator + uniqueFilename;
410  attachment.setName(filename);
411  attachment.setCrTime(crTime);
412  attachment.setmTime(mTime);
413  attachment.setLocalPath(relPath);
414  attachment.setSize(attach.getFilesize());
415  attachment.setEncodingType(TskData.EncodingType.XOR1);
416  email.addAttachment(attachment);
417  } catch (PSTException | IOException | NullPointerException ex) {
422  addErrorMessage(
423  NbBundle.getMessage(this.getClass(), "PstParser.extractAttch.errMsg.failedToExtractToDisk",
424  filename));
425  logger.log(Level.WARNING, "Failed to extract attachment from pst file.", ex); //NON-NLS
426  } catch (NoCurrentCaseException ex) {
427  addErrorMessage(Bundle.PstParser_noOpenCase_errMsg());
428  logger.log(Level.SEVERE, Bundle.PstParser_noOpenCase_errMsg(), ex); //NON-NLS
429  }
430  }
431  }
432 
442  private void saveAttachmentToDisk(PSTAttachment attach, String outPath) throws IOException, PSTException {
443  try (InputStream attachmentStream = attach.getFileInputStream();
444  EncodedFileOutputStream out = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1)) {
445  // 8176 is the block size used internally and should give the best performance
446  int bufferSize = 8176;
447  byte[] buffer = new byte[bufferSize];
448  int count = attachmentStream.read(buffer);
449 
450  if (count == -1) {
451  throw new IOException("attachmentStream invalid (read() fails). File " + attach.getLongFilename() + " skipped");
452  }
453 
454  while (count == bufferSize) {
455  out.write(buffer);
456  count = attachmentStream.read(buffer);
457  }
458  if (count != -1) {
459  byte[] endBuffer = new byte[count];
460  System.arraycopy(buffer, 0, endBuffer, 0, count);
461  out.write(endBuffer);
462  }
463  }
464  }
465 
474  private String getSender(String name, String addr) {
475  if (name.isEmpty() && addr.isEmpty()) {
476  return "";
477  } else if (name.isEmpty()) {
478  return addr;
479  } else if (addr.isEmpty()) {
480  return name;
481  } else {
482  return name + ": " + addr;
483  }
484  }
485 
493  public static boolean isPstFile(AbstractFile file) {
494  byte[] buffer = new byte[4];
495  try {
496  int read = file.read(buffer, 0, 4);
497  if (read != 4) {
498  return false;
499  }
500  ByteBuffer bb = ByteBuffer.wrap(buffer);
501  return bb.getInt() == PST_HEADER;
502  } catch (TskCoreException ex) {
503  logger.log(Level.WARNING, "Exception while detecting if a file is a pst file."); //NON-NLS
504  return false;
505  }
506  }
507 
513  private void addErrorMessage(String msg) {
514  errorList.add(msg);
515  }
516 
524  private List<String> extractReferences(String emailHeader) {
525  Scanner scanner = new Scanner(emailHeader);
526  StringBuilder buffer = null;
527  while (scanner.hasNextLine()) {
528  String token = scanner.nextLine();
529 
530  if (token.matches("^References:.*")) {
531  buffer = new StringBuilder();
532  buffer.append((token.substring(token.indexOf(':') + 1)).trim());
533  } else if (buffer != null) {
534  if (token.matches("^\\w+:.*$")) {
535  List<String> references = new ArrayList<>();
536  for (String id : buffer.toString().split(">")) {
537  references.add(id.trim() + ">");
538  }
539  return references;
540  } else {
541  buffer.append(token.trim());
542  }
543  }
544  }
545 
546  return null;
547  }
548 
553  private final class PstEmailIterator implements Iterator<EmailMessage> {
554 
555  private final PSTFolder folder;
556  private EmailMessage currentMsg;
557  private EmailMessage nextMsg;
558 
559  private final String currentPath;
560  private final long fileID;
561  private final boolean wholeMsg;
562 
571  PstEmailIterator(PSTFolder folder, String path, long fileID, boolean wholeMsg) {
572  this.folder = folder;
573  this.fileID = fileID;
574  this.currentPath = path;
575  this.wholeMsg = wholeMsg;
576 
577  if (folder.getContentCount() > 0) {
578  try {
579  PSTMessage message = (PSTMessage) folder.getNextChild();
580  if (message != null) {
581  if (wholeMsg) {
582  nextMsg = extractEmailMessage(message, currentPath, fileID);
583  } else {
584  nextMsg = extractPartialEmailMessage(message);
585  }
586  }
587  } catch (PSTException | IOException ex) {
588  failureCount++;
589  logger.log(Level.WARNING, String.format("Unable to extract emails for path: %s file ID: %d ", path, fileID), ex);
590  }
591  }
592  }
593 
594  @Override
595  public boolean hasNext() {
596  return nextMsg != null;
597  }
598 
599  @Override
600  public EmailMessage next() {
601 
602  currentMsg = nextMsg;
603 
604  try {
605  PSTMessage message = (PSTMessage) folder.getNextChild();
606  if (message != null) {
607  if (wholeMsg) {
608  nextMsg = extractEmailMessage(message, currentPath, fileID);
609  } else {
610  nextMsg = extractPartialEmailMessage(message);
611  }
612  } else {
613  nextMsg = null;
614  }
615  } catch (PSTException | IOException ex) {
616  logger.log(Level.WARNING, String.format("Unable to extract emails for path: %s file ID: %d ", currentPath, fileID), ex);
617  failureCount++;
618  nextMsg = null;
619  }
620 
621  return currentMsg;
622  }
623 
629  Iterable<EmailMessage> getIterable() {
630  return new Iterable<EmailMessage>() {
631  @Override
632  public Iterator<EmailMessage> iterator() {
633  return PstEmailIterator.this;
634  }
635  };
636  }
637 
638  }
639 }

Copyright © 2012-2021 Basis Technology. Generated on: Fri Aug 6 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.