Autopsy  4.17.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
SevenZipExtractor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2015-2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.embeddedfileextractor;
20 
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.nio.charset.Charset;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.Date;
32 import java.util.HashMap;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.concurrent.ConcurrentHashMap;
36 import java.util.logging.Level;
37 import net.sf.sevenzipjbinding.ArchiveFormat;
38 import static net.sf.sevenzipjbinding.ArchiveFormat.RAR;
39 import net.sf.sevenzipjbinding.ExtractAskMode;
40 import net.sf.sevenzipjbinding.ExtractOperationResult;
41 import net.sf.sevenzipjbinding.IArchiveExtractCallback;
42 import net.sf.sevenzipjbinding.ICryptoGetTextPassword;
43 import net.sf.sevenzipjbinding.IInArchive;
44 import net.sf.sevenzipjbinding.ISequentialOutStream;
45 import net.sf.sevenzipjbinding.PropID;
46 import net.sf.sevenzipjbinding.SevenZip;
47 import net.sf.sevenzipjbinding.SevenZipException;
48 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
49 import org.apache.tika.parser.txt.CharsetDetector;
50 import org.apache.tika.parser.txt.CharsetMatch;
51 import org.netbeans.api.progress.ProgressHandle;
52 import org.openide.util.NbBundle;
53 import org.openide.util.NbBundle.Messages;
54 import org.sleuthkit.autopsy.casemodule.Case;
55 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
56 import org.sleuthkit.autopsy.casemodule.services.FileManager;
57 import org.sleuthkit.autopsy.coreutils.FileUtil;
58 import org.sleuthkit.autopsy.coreutils.Logger;
59 import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
60 import org.sleuthkit.autopsy.ingest.IngestJobContext;
61 import org.sleuthkit.autopsy.ingest.IngestMessage;
62 import org.sleuthkit.autopsy.ingest.IngestMonitor;
63 import org.sleuthkit.autopsy.ingest.IngestServices;
64 import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
65 import org.sleuthkit.autopsy.modules.embeddedfileextractor.FileTaskExecutor.FileTaskFailedException;
66 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
67 import org.sleuthkit.datamodel.AbstractFile;
68 import org.sleuthkit.datamodel.Blackboard;
69 import org.sleuthkit.datamodel.BlackboardArtifact;
70 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT;
71 import org.sleuthkit.datamodel.BlackboardAttribute;
72 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
73 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION;
74 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
75 import org.sleuthkit.datamodel.Content;
76 import org.sleuthkit.datamodel.DerivedFile;
77 import org.sleuthkit.datamodel.EncodedFileOutputStream;
78 import org.sleuthkit.datamodel.ReadContentInputStream;
79 import org.sleuthkit.datamodel.TskCoreException;
80 import org.sleuthkit.datamodel.TskData;
81 
86 class SevenZipExtractor {
87 
88  private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
89 
90  private static final String MODULE_NAME = EmbeddedFileExtractorModuleFactory.getModuleName();
91 
92  //encryption type strings
93  private static final String ENCRYPTION_FILE_LEVEL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
94  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
95  private static final String ENCRYPTION_FULL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
96  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull");
97 
98  //zip bomb detection
99  private static final int MAX_DEPTH = 4;
100  private static final int MAX_COMPRESSION_RATIO = 600;
101  private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
102  private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L; //1GB
103 
104  private IngestServices services = IngestServices.getInstance();
105  private final IngestJobContext context;
106  private final FileTypeDetector fileTypeDetector;
107  private final FileTaskExecutor fileTaskExecutor;
108 
109  private String moduleDirRelative;
110  private String moduleDirAbsolute;
111 
112  private Blackboard blackboard;
113 
114  private ProgressHandle progress;
115  private int numItems;
116  private String currentArchiveName;
117 
121  private enum SupportedArchiveExtractionFormats {
122 
123  ZIP("application/zip"), //NON-NLS
124  SEVENZ("application/x-7z-compressed"), //NON-NLS
125  GZIP("application/gzip"), //NON-NLS
126  XGZIP("application/x-gzip"), //NON-NLS
127  XBZIP2("application/x-bzip2"), //NON-NLS
128  XTAR("application/x-tar"), //NON-NLS
129  XGTAR("application/x-gtar"),
130  XRAR("application/x-rar-compressed"); //NON-NLS
131 
132  private final String mimeType;
133 
134  SupportedArchiveExtractionFormats(final String mimeType) {
135  this.mimeType = mimeType;
136  }
137 
138  @Override
139  public String toString() {
140  return this.mimeType;
141  }
142  // TODO Expand to support more formats after upgrading Tika
143  }
144 
165  SevenZipExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute, FileTaskExecutor fileTaskExecutor) throws SevenZipNativeInitializationException {
166  if (!SevenZip.isInitializedSuccessfully()) {
167  throw new SevenZipNativeInitializationException("SevenZip has not been previously initialized.");
168  }
169  this.context = context;
170  this.fileTypeDetector = fileTypeDetector;
171  this.moduleDirRelative = moduleDirRelative;
172  this.moduleDirAbsolute = moduleDirAbsolute;
173  this.fileTaskExecutor = fileTaskExecutor;
174  }
175 
184  boolean isSevenZipExtractionSupported(AbstractFile file) {
185  String fileMimeType = fileTypeDetector.getMIMEType(file);
186  for (SupportedArchiveExtractionFormats mimeType : SupportedArchiveExtractionFormats.values()) {
187  if (mimeType.toString().equals(fileMimeType)) {
188  return true;
189  }
190  }
191  return false;
192  }
193 
217  private boolean isZipBombArchiveItemCheck(AbstractFile archiveFile, IInArchive inArchive, int inArchiveItemIndex, ConcurrentHashMap<Long, Archive> depthMap, String escapedFilePath) {
218  //If a file is corrupted as a result of reconstructing it from unallocated space, then
219  //7zip does a poor job estimating the original uncompressed file size.
220  //As a result, many corrupted files have wonky compression ratios and could flood the UI
221  //with false zip bomb notifications. The decision was made to skip compression ratio checks
222  //for unallocated zip files. Instead, we let the depth be an indicator of a zip bomb.
223  //Gzip archives compress a single file. They may have a sparse file,
224  //and that file could be much larger, however it won't be the exponential growth seen with more dangerous zip bombs.
225  //In addition a fair number of browser cache files will be gzip archives,
226  //and their file sizes are frequently retrieved incorrectly so ignoring gzip files is a reasonable decision.
227  if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.UNALLOC) || archiveFile.getMIMEType().equalsIgnoreCase(SupportedArchiveExtractionFormats.XGZIP.toString())) {
228  return false;
229  }
230 
231  try {
232  final Long archiveItemSize = (Long) inArchive.getProperty(
233  inArchiveItemIndex, PropID.SIZE);
234 
235  //skip the check for small files
236  if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
237  return false;
238  }
239 
240  final Long archiveItemPackedSize = (Long) inArchive.getProperty(
241  inArchiveItemIndex, PropID.PACKED_SIZE);
242 
243  if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
244  logger.log(Level.WARNING, "Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}", //NON-NLS
245  new Object[]{archiveFile.getName(), (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH)}); //NON-NLS
246  return false;
247  }
248 
249  int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
250 
251  if (cRatio >= MAX_COMPRESSION_RATIO) {
252  Archive rootArchive = depthMap.get(depthMap.get(archiveFile.getId()).getRootArchiveId());
253  String details = NbBundle.getMessage(SevenZipExtractor.class,
254  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails",
255  cRatio, FileUtil.escapeFileName(getArchiveFilePath(rootArchive.getArchiveFile())));
256 
257  flagRootArchiveAsZipBomb(rootArchive, archiveFile, details, escapedFilePath);
258  return true;
259  } else {
260  return false;
261  }
262 
263  } catch (SevenZipException ex) {
264  logger.log(Level.WARNING, "Error getting archive item size and cannot detect if zipbomb. ", ex); //NON-NLS
265  return false;
266  }
267  }
268 
280  private void flagRootArchiveAsZipBomb(Archive rootArchive, AbstractFile archiveFile, String details, String escapedFilePath) {
281  rootArchive.flagAsZipBomb();
282  logger.log(Level.INFO, details);
283  try {
284  Collection<BlackboardAttribute> attributes = Arrays.asList(
285  new BlackboardAttribute(
286  TSK_SET_NAME, MODULE_NAME,
287  "Possible Zip Bomb"),
288  new BlackboardAttribute(
289  TSK_DESCRIPTION, MODULE_NAME,
290  Bundle.SevenZipExtractor_zipBombArtifactCreation_text(archiveFile.getName())),
291  new BlackboardAttribute(
292  TSK_COMMENT, MODULE_NAME,
293  details));
294 
295  if (!blackboard.artifactExists(archiveFile, TSK_INTERESTING_FILE_HIT, attributes)) {
296  BlackboardArtifact artifact = rootArchive.getArchiveFile().newArtifact(TSK_INTERESTING_FILE_HIT);
297  artifact.addAttributes(attributes);
298  try {
299  /*
300  * post the artifact which will index the artifact for
301  * keyword search, and fire an event to notify UI of this
302  * new artifact
303  */
304  blackboard.postArtifact(artifact, MODULE_NAME);
305 
306  String msg = NbBundle.getMessage(SevenZipExtractor.class,
307  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), escapedFilePath);//NON-NLS
308 
309  services.postMessage(IngestMessage.createWarningMessage(MODULE_NAME, msg, details));
310 
311  } catch (Blackboard.BlackboardException ex) {
312  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
313  MessageNotifyUtil.Notify.error(
314  Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
315  }
316  }
317  } catch (TskCoreException ex) {
318  logger.log(Level.SEVERE, "Error creating blackboard artifact for Zip Bomb Detection for file: " + escapedFilePath, ex); //NON-NLS
319  }
320  }
321 
330  private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
331  // try to get the file type from the BB
332  String detectedFormat;
333  detectedFormat = archiveFile.getMIMEType();
334 
335  if (detectedFormat == null) {
336  logger.log(Level.WARNING, "Could not detect format for file: {0}", archiveFile); //NON-NLS
337 
338  // if we don't have attribute info then use file extension
339  String extension = archiveFile.getNameExtension();
340  if ("rar".equals(extension)) //NON-NLS
341  {
342  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
343  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
344  return RAR;
345  }
346 
347  // Otherwise open the archive using 7zip's built-in auto-detect functionality
348  return null;
349  } else if (detectedFormat.contains("application/x-rar-compressed")) //NON-NLS
350  {
351  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
352  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
353  return RAR;
354  }
355 
356  // Otherwise open the archive using 7zip's built-in auto-detect functionality
357  return null;
358  }
359 
370  private long getRootArchiveId(AbstractFile file) throws TskCoreException {
371  long id = file.getId();
372  Content parentContent = file.getParent();
373  while (parentContent != null) {
374  id = parentContent.getId();
375  parentContent = parentContent.getParent();
376  }
377  return id;
378  }
379 
399  private List<AbstractFile> getAlreadyExtractedFiles(AbstractFile archiveFile, String archiveFilePath) throws TskCoreException, InterruptedException, FileTaskExecutor.FileTaskFailedException {
400  /*
401  * TODO (Jira-7145): Is this logic correct?
402  */
403  List<AbstractFile> extractedFiles = new ArrayList<>();
404  File outputDirectory = new File(moduleDirAbsolute, EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
405  if (archiveFile.hasChildren() && fileTaskExecutor.exists(outputDirectory)) {
406  Case currentCase = Case.getCurrentCase();
407  FileManager fileManager = currentCase.getServices().getFileManager();
408  extractedFiles.addAll(fileManager.findFilesByParentPath(getRootArchiveId(archiveFile), archiveFilePath));
409  }
410  return extractedFiles;
411  }
412 
420  private String getArchiveFilePath(AbstractFile archiveFile) {
421  return archiveFile.getParentPath() + archiveFile.getName();
422  }
423 
433  private boolean makeExtractedFilesDirectory(String uniqueArchiveFileName) {
434  boolean success = true;
435  Path rootDirectoryPath = Paths.get(moduleDirAbsolute, uniqueArchiveFileName);
436  File rootDirectory = rootDirectoryPath.toFile();
437  try {
438  if (!fileTaskExecutor.exists(rootDirectory)) {
439  success = fileTaskExecutor.mkdirs(rootDirectory);
440  }
441  } catch (SecurityException | FileTaskFailedException | InterruptedException ex) {
442  logger.log(Level.SEVERE, String.format("Error creating root extracted files directory %s", rootDirectory), ex); //NON-NLS
443  success = false;
444  }
445  return success;
446  }
447 
460  private String getPathInArchive(IInArchive archive, int inArchiveItemIndex, AbstractFile archiveFile) throws SevenZipException {
461  String pathInArchive = (String) archive.getProperty(inArchiveItemIndex, PropID.PATH);
462 
463  if (pathInArchive == null || pathInArchive.isEmpty()) {
464  //some formats (.tar.gz) may not be handled correctly -- file in archive has no name/path
465  //handle this for .tar.gz and tgz but assuming the child is tar,
466  //otherwise, unpack using itemNumber as name
467 
468  //TODO this should really be signature based, not extension based
469  String archName = archiveFile.getName();
470  int dotI = archName.lastIndexOf(".");
471  String useName = null;
472  if (dotI != -1) {
473  String base = archName.substring(0, dotI);
474  String ext = archName.substring(dotI);
475  int colonIndex = ext.lastIndexOf(":");
476  if (colonIndex != -1) {
477  // If alternate data stream is found, fix the name
478  // so Windows doesn't choke on the colon character.
479  ext = ext.substring(0, colonIndex);
480  }
481  switch (ext) {
482  case ".gz": //NON-NLS
483  useName = base;
484  break;
485  case ".tgz": //NON-NLS
486  useName = base + ".tar"; //NON-NLS
487  break;
488  case ".bz2": //NON-NLS
489  useName = base;
490  break;
491  }
492  }
493  if (useName == null) {
494  pathInArchive = "/" + archName + "/" + Integer.toString(inArchiveItemIndex);
495  } else {
496  pathInArchive = "/" + useName;
497  }
498  }
499  return pathInArchive;
500  }
501 
502  private byte[] getPathBytesInArchive(IInArchive archive, int inArchiveItemIndex, AbstractFile archiveFile) throws SevenZipException {
503  return (byte[]) archive.getProperty(inArchiveItemIndex, PropID.PATH_BYTES);
504  }
505 
506  /*
507  * Get the String that will represent the key for the hashmap which keeps
508  * track of existing files from an AbstractFile
509  */
510  private String getKeyAbstractFile(AbstractFile fileInDatabase) {
511  return fileInDatabase == null ? null : fileInDatabase.getParentPath() + fileInDatabase.getName();
512  }
513 
514  /*
515  * Get the String that will represent the key for the hashmap which keeps
516  * track of existing files from an unpacked node and the archiveFilePath
517  */
518  private String getKeyFromUnpackedNode(UnpackedTree.UnpackedNode node, String archiveFilePath) {
519  return node == null ? null : archiveFilePath + "/" + node.getFileName();
520  }
521 
529  void unpack(AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap) {
530  unpack(archiveFile, depthMap, null);
531  }
532 
544  @Messages({"SevenZipExtractor.indexError.message=Failed to index encryption detected artifact for keyword search.",
545  "# {0} - rootArchive",
546  "SevenZipExtractor.zipBombArtifactCreation.text=Zip Bomb Detected {0}"})
547  boolean unpack(AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap, String password) {
548  boolean unpackSuccessful = true; //initialized to true change to false if any files fail to extract and
549  boolean hasEncrypted = false;
550  boolean fullEncryption = true;
551  boolean progressStarted = false;
552  final String archiveFilePath = getArchiveFilePath(archiveFile);
553  final String escapedArchiveFilePath = FileUtil.escapeFileName(archiveFilePath);
554  HashMap<String, ZipFileStatusWrapper> statusMap = new HashMap<>();
555  List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
556 
557  currentArchiveName = archiveFile.getName();
558 
559  SevenZipContentReadStream stream = null;
560  progress = ProgressHandle.createHandle(Bundle.EmbeddedFileExtractorIngestModule_ArchiveExtractor_moduleName());
561  //recursion depth check for zip bomb
562  Archive parentAr;
563  try {
564  blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
565  } catch (NoCurrentCaseException ex) {
566  logger.log(Level.INFO, "Exception while getting open case.", ex); //NON-NLS
567  unpackSuccessful = false;
568  return unpackSuccessful;
569  }
570 
571  try {
572  List<AbstractFile> existingFiles = getAlreadyExtractedFiles(archiveFile, archiveFilePath);
573  for (AbstractFile file : existingFiles) {
574  statusMap.put(getKeyAbstractFile(file), new ZipFileStatusWrapper(file, ZipFileStatus.EXISTS));
575  }
576  } catch (TskCoreException | FileTaskFailedException | InterruptedException ex) {
577  logger.log(Level.SEVERE, String.format("Error checking if %s has already been processed, skipping", escapedArchiveFilePath), ex); //NON-NLS
578  unpackSuccessful = false;
579  return unpackSuccessful;
580  }
581 
582  parentAr = depthMap.get(archiveFile.getId());
583  if (parentAr == null) {
584  parentAr = new Archive(0, archiveFile.getId(), archiveFile);
585  depthMap.put(archiveFile.getId(), parentAr);
586  } else {
587  Archive rootArchive = depthMap.get(parentAr.getRootArchiveId());
588  if (rootArchive.isFlaggedAsZipBomb()) {
589  //skip this archive as the root archive has already been determined to contain a zip bomb
590  unpackSuccessful = false;
591  return unpackSuccessful;
592  } else if (parentAr.getDepth() == MAX_DEPTH) {
593  String details = NbBundle.getMessage(SevenZipExtractor.class,
594  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
595  parentAr.getDepth(), FileUtil.escapeFileName(getArchiveFilePath(rootArchive.getArchiveFile())));
596  flagRootArchiveAsZipBomb(rootArchive, archiveFile, details, escapedArchiveFilePath);
597  unpackSuccessful = false;
598  return unpackSuccessful;
599  }
600  }
601  IInArchive inArchive = null;
602  try {
603  stream = new SevenZipContentReadStream(new ReadContentInputStream(archiveFile));
604  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
605  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality.
606  // All other archive formats are still opened using 7zip built-in auto-detect functionality.
607  ArchiveFormat options = get7ZipOptions(archiveFile);
608  if (password == null) {
609  inArchive = SevenZip.openInArchive(options, stream);
610  } else {
611  inArchive = SevenZip.openInArchive(options, stream, password);
612  }
613  numItems = inArchive.getNumberOfItems();
614  progress.start(numItems);
615  progressStarted = true;
616 
617  //setup the archive local root folder
618  final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
619  if (!makeExtractedFilesDirectory(uniqueArchiveFileName)) {
620  return false;
621  }
622 
623  //initialize tree hierarchy to keep track of unpacked file structure
624  SevenZipExtractor.UnpackedTree unpackedTree = new SevenZipExtractor.UnpackedTree(moduleDirRelative + "/" + uniqueArchiveFileName, archiveFile);
625 
626  long freeDiskSpace;
627  try {
628  freeDiskSpace = services.getFreeDiskSpace();
629  } catch (NullPointerException ex) {
630  //If ingest has not been run at least once getFreeDiskSpace() will throw a null pointer exception
631  //currently getFreeDiskSpace always returns DISK_FREE_SPACE_UNKNOWN
632  freeDiskSpace = IngestMonitor.DISK_FREE_SPACE_UNKNOWN;
633  }
634 
635  Map<Integer, InArchiveItemDetails> archiveDetailsMap = new HashMap<>();
636  for (int inArchiveItemIndex = 0; inArchiveItemIndex < numItems; inArchiveItemIndex++) {
637  progress.progress(String.format("%s: Analyzing archive metadata and creating local files (%d of %d)", currentArchiveName, inArchiveItemIndex + 1, numItems), 0);
638  if (isZipBombArchiveItemCheck(archiveFile, inArchive, inArchiveItemIndex, depthMap, escapedArchiveFilePath)) {
639  unpackSuccessful = false;
640  return unpackSuccessful;
641  }
642 
643  String pathInArchive = getPathInArchive(inArchive, inArchiveItemIndex, archiveFile);
644  byte[] pathBytesInArchive = getPathBytesInArchive(inArchive, inArchiveItemIndex, archiveFile);
645  UnpackedTree.UnpackedNode unpackedNode = unpackedTree.addNode(pathInArchive, pathBytesInArchive);
646 
647  final boolean isEncrypted = (Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.ENCRYPTED);
648 
649  if (isEncrypted && password == null) {
650  logger.log(Level.WARNING, "Skipping encrypted file in archive: {0}", pathInArchive); //NON-NLS
651  hasEncrypted = true;
652  unpackSuccessful = false;
653  continue;
654  } else {
655  fullEncryption = false;
656  }
657 
658  // NOTE: item size may return null in case of certain
659  // archiving formats. Eg: BZ2
660  //check if unpacking this file will result in out of disk space
661  //this is additional to zip bomb prevention mechanism
662  Long archiveItemSize = (Long) inArchive.getProperty(
663  inArchiveItemIndex, PropID.SIZE);
664  if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && archiveItemSize != null && archiveItemSize > 0) { //if free space is known and file is not empty.
665  String archiveItemPath = (String) inArchive.getProperty(
666  inArchiveItemIndex, PropID.PATH);
667  long newDiskSpace = freeDiskSpace - archiveItemSize;
668  if (newDiskSpace < MIN_FREE_DISK_SPACE) {
669  String msg = NbBundle.getMessage(SevenZipExtractor.class,
670  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
671  escapedArchiveFilePath, archiveItemPath);
672  String details = NbBundle.getMessage(SevenZipExtractor.class,
673  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
674  services.postMessage(IngestMessage.createErrorMessage(MODULE_NAME, msg, details));
675  logger.log(Level.INFO, "Skipping archive item due to insufficient disk space: {0}, {1}", new String[]{escapedArchiveFilePath, archiveItemPath}); //NON-NLS
676  logger.log(Level.INFO, "Available disk space: {0}", new Object[]{freeDiskSpace}); //NON-NLS
677  unpackSuccessful = false;
678  continue; //skip this file
679  } else {
680  //update est. disk space during this archive, so we don't need to poll for every file extracted
681  freeDiskSpace = newDiskSpace;
682  }
683  }
684  final String uniqueExtractedName = FileUtil.escapeFileName(uniqueArchiveFileName + File.separator + (inArchiveItemIndex / 1000) + File.separator + inArchiveItemIndex + "_" + new File(pathInArchive).getName());
685  final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
686  final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
687 
688  //create local dirs and empty files before extracted
689  //cannot rely on files in top-bottom order
690  File localFile = new File(localAbsPath);
691  boolean localFileExists;
692  try {
693  if ((Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER)) {
694  localFileExists = findOrCreateDirectory(localFile);
695  } else {
696  localFileExists = findOrCreateEmptyFile(localFile);
697  }
698  } catch (FileTaskFailedException | InterruptedException ex) {
699  localFileExists = false;
700  logger.log(Level.SEVERE, String.format("Error fiding or creating %s", localFile.getAbsolutePath()), ex); //NON-NLS
701  }
702 
703  // skip the rest of this loop if we couldn't create the file
704  //continue will skip details from being added to the map
705  if (!localFileExists) {
706  logger.log(Level.SEVERE, String.format("Skipping %s because it could not be created", localFile.getAbsolutePath())); //NON-NLS
707  continue;
708  }
709 
710  //Store archiveItemIndex with local paths and unpackedNode reference.
711  //Necessary for the extract call back to write the current archive
712  //file to the correct disk location and to correctly update it's
713  //corresponding unpackedNode
714  archiveDetailsMap.put(inArchiveItemIndex, new InArchiveItemDetails(
715  unpackedNode, localAbsPath, localRelPath));
716  }
717 
718  int[] extractionIndices = getExtractableFilesFromDetailsMap(archiveDetailsMap);
719 
720  StandardIArchiveExtractCallback archiveCallBack
721  = new StandardIArchiveExtractCallback(
722  inArchive, archiveFile, progress,
723  archiveDetailsMap, password, freeDiskSpace);
724 
725  //According to the documentation, indices in sorted order are optimal
726  //for efficiency. Hence, the HashMap and linear processing of
727  //inArchiveItemIndex. False indicates non-test mode
728  inArchive.extract(extractionIndices, false, archiveCallBack);
729 
730  unpackSuccessful &= archiveCallBack.wasSuccessful();
731 
732  archiveDetailsMap = null;
733 
734  // add them to the DB. We wait until the end so that we have the metadata on all of the
735  // intermediate nodes since the order is not guaranteed
736  try {
737  unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath);
738  unpackedFiles = unpackedTree.getAllFileObjects();
739  //check if children are archives, update archive depth tracking
740  for (int i = 0; i < unpackedFiles.size(); i++) {
741  progress.progress(String.format("%s: Searching for nested archives (%d of %d)", currentArchiveName, i + 1, unpackedFiles.size()));
742  AbstractFile unpackedFile = unpackedFiles.get(i);
743  if (unpackedFile == null) {
744  continue;
745  }
746  if (isSevenZipExtractionSupported(unpackedFile)) {
747  Archive child = new Archive(parentAr.getDepth() + 1, parentAr.getRootArchiveId(), archiveFile);
748  parentAr.addChild(child);
749  depthMap.put(unpackedFile.getId(), child);
750  }
751  unpackedFile.close();
752  }
753 
754  } catch (TskCoreException | NoCurrentCaseException e) {
755  logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
756  //TODO decide if anything to cleanup, for now bailing
757  }
758 
759  } catch (SevenZipException | IllegalArgumentException ex) {
760  logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
761  //inbox message
762 
763  // print a message if the file is allocated
764  if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
765  String msg = NbBundle.getMessage(SevenZipExtractor.class,
766  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
767  currentArchiveName);
768  String details = NbBundle.getMessage(SevenZipExtractor.class,
769  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
770  escapedArchiveFilePath, ex.getMessage());
771  services.postMessage(IngestMessage.createErrorMessage(MODULE_NAME, msg, details));
772  }
773  } finally {
774  if (inArchive != null) {
775  try {
776  inArchive.close();
777  } catch (SevenZipException e) {
778  logger.log(Level.SEVERE, "Error closing archive: " + archiveFile, e); //NON-NLS
779  }
780  }
781 
782  if (stream != null) {
783  try {
784  stream.close();
785  } catch (IOException ex) {
786  logger.log(Level.SEVERE, "Error closing stream after unpacking archive: " + archiveFile, ex); //NON-NLS
787  }
788  }
789 
790  //close progress bar
791  if (progressStarted) {
792  progress.finish();
793  }
794  }
795 
796  //create artifact and send user message
797  if (hasEncrypted) {
798  String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
799  try {
800  BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
801  artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, encryptionType));
802 
803  try {
804  /*
805  * post the artifact which will index the artifact for
806  * keyword search, and fire an event to notify UI of this
807  * new artifact
808  */
809  blackboard.postArtifact(artifact, MODULE_NAME);
810  } catch (Blackboard.BlackboardException ex) {
811  logger.log(Level.SEVERE, "Unable to post blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
812  MessageNotifyUtil.Notify.error(
813  Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
814  }
815 
816  } catch (TskCoreException ex) {
817  logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + escapedArchiveFilePath, ex); //NON-NLS
818  }
819 
820  String msg = NbBundle.getMessage(SevenZipExtractor.class,
821  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
822  String details = NbBundle.getMessage(SevenZipExtractor.class,
823  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
824  currentArchiveName, MODULE_NAME);
825  services.postMessage(IngestMessage.createWarningMessage(MODULE_NAME, msg, details));
826  }
827 
828  // adding unpacked extracted derived files to the job after closing relevant resources.
829  if (!unpackedFiles.isEmpty()) {
830  //currently sending a single event for all new files
831  services.fireModuleContentEvent(new ModuleContentEvent(archiveFile));
832  if (context != null) {
833  context.addFilesToJob(unpackedFiles);
834  }
835  }
836 
837  return unpackSuccessful;
838  }
839 
847  private boolean findOrCreateDirectory(File directory) throws FileTaskFailedException, InterruptedException {
848  if (!fileTaskExecutor.exists(directory)) {
849  return fileTaskExecutor.mkdirs(directory);
850  } else {
851  return true;
852  }
853  }
854 
862  private boolean findOrCreateEmptyFile(File file) throws FileTaskFailedException, InterruptedException {
863  if (!fileTaskExecutor.exists(file)) {
864  fileTaskExecutor.mkdirs(file.getParentFile());
865  return fileTaskExecutor.createNewFile(file);
866  } else {
867  return true;
868  }
869  }
870 
871  private Charset detectFilenamesCharset(List<byte[]> byteDatas) {
872  Charset detectedCharset = null;
873  CharsetDetector charsetDetector = new CharsetDetector();
874  int byteSum = 0;
875  int fileNum = 0;
876  for (byte[] byteData : byteDatas) {
877  fileNum++;
878  byteSum += byteData.length;
879  // Only read ~1000 bytes of filenames in this directory
880  if (byteSum >= 1000) {
881  break;
882  }
883  }
884  byte[] allBytes = new byte[byteSum];
885  int start = 0;
886  for (int i = 0; i < fileNum; i++) {
887  byte[] byteData = byteDatas.get(i);
888  System.arraycopy(byteData, 0, allBytes, start, byteData.length);
889  start += byteData.length;
890  }
891  charsetDetector.setText(allBytes);
892  CharsetMatch cm = charsetDetector.detect();
893  if (cm.getConfidence() >= 90 && Charset.isSupported(cm.getName())) {
894  detectedCharset = Charset.forName(cm.getName());
895  }
896  return detectedCharset;
897  }
898 
903  private int[] getExtractableFilesFromDetailsMap(
904  Map<Integer, InArchiveItemDetails> archiveDetailsMap) {
905 
906  Integer[] wrappedExtractionIndices = archiveDetailsMap.keySet()
907  .toArray(new Integer[archiveDetailsMap.size()]);
908 
909  return Arrays.stream(wrappedExtractionIndices)
910  .mapToInt(Integer::intValue)
911  .toArray();
912 
913  }
914 
922  private final static class UnpackStream implements ISequentialOutStream {
923 
924  private EncodedFileOutputStream output;
925  private String localAbsPath;
926  private int bytesWritten;
927 
928  UnpackStream(String localAbsPath) throws IOException {
929  this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1);
930  this.localAbsPath = localAbsPath;
931  this.bytesWritten = 0;
932  }
933 
934  public void setNewOutputStream(String localAbsPath) throws IOException {
935  this.output.close();
936  this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1);
937  this.localAbsPath = localAbsPath;
938  this.bytesWritten = 0;
939  }
940 
941  public int getSize() {
942  return bytesWritten;
943  }
944 
945  @Override
946  public int write(byte[] bytes) throws SevenZipException {
947  try {
948  output.write(bytes);
949  this.bytesWritten += bytes.length;
950  } catch (IOException ex) {
951  throw new SevenZipException(
952  NbBundle.getMessage(SevenZipExtractor.class,
953  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
954  localAbsPath), ex);
955  }
956  return bytes.length;
957  }
958 
959  public void close() throws IOException {
960  try (EncodedFileOutputStream out = output) {
961  out.flush();
962  }
963  }
964 
965  }
966 
970  private static class InArchiveItemDetails {
971 
972  private final SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode;
973  private final String localAbsPath;
974  private final String localRelPath;
975 
976  InArchiveItemDetails(
977  SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode,
978  String localAbsPath, String localRelPath) {
979  this.unpackedNode = unpackedNode;
980  this.localAbsPath = localAbsPath;
981  this.localRelPath = localRelPath;
982  }
983 
984  public SevenZipExtractor.UnpackedTree.UnpackedNode getUnpackedNode() {
985  return unpackedNode;
986  }
987 
988  public String getLocalAbsPath() {
989  return localAbsPath;
990  }
991 
992  public String getLocalRelPath() {
993  return localRelPath;
994  }
995  }
996 
1001  private static class StandardIArchiveExtractCallback
1002  implements IArchiveExtractCallback, ICryptoGetTextPassword {
1003 
1004  private final AbstractFile archiveFile;
1005  private final IInArchive inArchive;
1006  private UnpackStream unpackStream = null;
1007  private final Map<Integer, InArchiveItemDetails> archiveDetailsMap;
1008  private final ProgressHandle progressHandle;
1009 
1010  private int inArchiveItemIndex;
1011 
1012  private long createTimeInSeconds;
1013  private long modTimeInSeconds;
1014  private long accessTimeInSeconds;
1015 
1016  private boolean isFolder;
1017  private final String password;
1018 
1019  private boolean unpackSuccessful = true;
1020 
1021  StandardIArchiveExtractCallback(IInArchive inArchive,
1022  AbstractFile archiveFile, ProgressHandle progressHandle,
1023  Map<Integer, InArchiveItemDetails> archiveDetailsMap,
1024  String password, long freeDiskSpace) {
1025  this.inArchive = inArchive;
1026  this.progressHandle = progressHandle;
1027  this.archiveFile = archiveFile;
1028  this.archiveDetailsMap = archiveDetailsMap;
1029  this.password = password;
1030  }
1031 
1046  @Override
1047  public ISequentialOutStream getStream(int inArchiveItemIndex,
1048  ExtractAskMode mode) throws SevenZipException {
1049 
1050  this.inArchiveItemIndex = inArchiveItemIndex;
1051 
1052  isFolder = (Boolean) inArchive
1053  .getProperty(inArchiveItemIndex, PropID.IS_FOLDER);
1054  if (isFolder || mode != ExtractAskMode.EXTRACT) {
1055  return null;
1056  }
1057 
1058  final String localAbsPath = archiveDetailsMap.get(
1059  inArchiveItemIndex).getLocalAbsPath();
1060 
1061  //If the Unpackstream has been allocated, then set the Outputstream
1062  //to another file rather than creating a new unpack stream. The 7Zip
1063  //binding has a memory leak, so creating new unpack streams will not be
1064  //dereferenced. As a fix, we create one UnpackStream, and mutate its state,
1065  //so that there only exists one 8192 byte buffer in memory per archive.
1066  try {
1067  if (unpackStream != null) {
1068  unpackStream.setNewOutputStream(localAbsPath);
1069  } else {
1070  unpackStream = new UnpackStream(localAbsPath);
1071  }
1072  } catch (IOException ex) {
1073  logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS
1074  + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS
1075  return null;
1076  }
1077 
1078  return unpackStream;
1079  }
1080 
1089  @Override
1090  public void prepareOperation(ExtractAskMode mode) throws SevenZipException {
1091  final Date createTime = (Date) inArchive.getProperty(
1092  inArchiveItemIndex, PropID.CREATION_TIME);
1093  final Date accessTime = (Date) inArchive.getProperty(
1094  inArchiveItemIndex, PropID.LAST_ACCESS_TIME);
1095  final Date writeTime = (Date) inArchive.getProperty(
1096  inArchiveItemIndex, PropID.LAST_MODIFICATION_TIME);
1097 
1098  createTimeInSeconds = createTime == null ? 0L
1099  : createTime.getTime() / 1000;
1100  modTimeInSeconds = writeTime == null ? 0L
1101  : writeTime.getTime() / 1000;
1102  accessTimeInSeconds = accessTime == null ? 0L
1103  : accessTime.getTime() / 1000;
1104 
1105  progressHandle.progress(archiveFile.getName() + ": "
1106  + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH),
1107  inArchiveItemIndex);
1108 
1109  }
1110 
1119  @Override
1120  public void setOperationResult(ExtractOperationResult result) throws SevenZipException {
1121 
1122  final SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode
1123  = archiveDetailsMap.get(inArchiveItemIndex).getUnpackedNode();
1124  final String localRelPath = archiveDetailsMap.get(
1125  inArchiveItemIndex).getLocalRelPath();
1126  if (isFolder) {
1127  unpackedNode.addDerivedInfo(0,
1128  !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER),
1129  0L, createTimeInSeconds, accessTimeInSeconds, modTimeInSeconds,
1130  localRelPath);
1131  return;
1132  }
1133 
1134  final String localAbsPath = archiveDetailsMap.get(
1135  inArchiveItemIndex).getLocalAbsPath();
1136  if (result != ExtractOperationResult.OK) {
1137  if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.UNALLOC)) {
1138  logger.log(Level.WARNING, "Extraction of : {0} encountered error {1} (file is unallocated and may be corrupt)", //NON-NLS
1139  new Object[]{localAbsPath, result});
1140  } else {
1141  logger.log(Level.WARNING, "Extraction of : {0} encountered error {1}", //NON-NLS
1142  new Object[]{localAbsPath, result});
1143  }
1144  unpackSuccessful = false;
1145  }
1146 
1147  //record derived data in unode, to be traversed later after unpacking the archive
1148  unpackedNode.addDerivedInfo(unpackStream.getSize(),
1149  !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER),
1150  0L, createTimeInSeconds, accessTimeInSeconds, modTimeInSeconds, localRelPath);
1151 
1152  try {
1153  unpackStream.close();
1154  } catch (IOException e) {
1155  logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS
1156  }
1157  }
1158 
1159  @Override
1160  public void setTotal(long value) throws SevenZipException {
1161  //Not necessary for extract, left intenionally blank
1162  }
1163 
1164  @Override
1165  public void setCompleted(long value) throws SevenZipException {
1166  //Not necessary for extract, left intenionally blank
1167  }
1168 
1176  @Override
1177  public String cryptoGetTextPassword() throws SevenZipException {
1178  return password;
1179  }
1180 
1181  public boolean wasSuccessful() {
1182  return unpackSuccessful;
1183  }
1184  }
1185 
1193  private class UnpackedTree {
1194 
1195  final UnpackedNode rootNode;
1196  private int nodesProcessed = 0;
1197 
1204  UnpackedTree(String localPathRoot, AbstractFile archiveFile) {
1205  this.rootNode = new UnpackedNode();
1206  this.rootNode.setFile(archiveFile);
1207  this.rootNode.setFileName(archiveFile.getName());
1208  this.rootNode.setLocalRelPath(localPathRoot);
1209  }
1210 
1220  UnpackedNode addNode(String filePath, byte[] filePathBytes) {
1221  String[] toks = filePath.split("[\\/\\\\]");
1222  List<String> tokens = new ArrayList<>();
1223  for (int i = 0; i < toks.length; ++i) {
1224  if (!toks[i].isEmpty()) {
1225  tokens.add(toks[i]);
1226  }
1227  }
1228 
1229  List<byte[]> byteTokens = null;
1230  if (filePathBytes == null) {
1231  return addNode(rootNode, tokens, null);
1232  } else {
1233  byteTokens = new ArrayList<>(tokens.size());
1234  int last = 0;
1235  for (int i = 0; i < filePathBytes.length; i++) {
1236  if (filePathBytes[i] == '/') {
1237  int len = i - last;
1238  byte[] arr = new byte[len];
1239  System.arraycopy(filePathBytes, last, arr, 0, len);
1240  byteTokens.add(arr);
1241  last = i + 1;
1242  }
1243  }
1244  int len = filePathBytes.length - last;
1245  if (len > 0) {
1246  byte[] arr = new byte[len];
1247  System.arraycopy(filePathBytes, last, arr, 0, len);
1248  byteTokens.add(arr);
1249  }
1250 
1251  if (tokens.size() != byteTokens.size()) {
1252  logger.log(Level.WARNING, "Could not map path bytes to path string");
1253  return addNode(rootNode, tokens, null);
1254  }
1255  }
1256 
1257  return addNode(rootNode, tokens, byteTokens);
1258  }
1259 
1269  private UnpackedNode addNode(UnpackedNode parent,
1270  List<String> tokenPath, List<byte[]> tokenPathBytes) {
1271  // we found all of the tokens
1272  if (tokenPath.isEmpty()) {
1273  return parent;
1274  }
1275 
1276  // get the next name in the path and look it up
1277  String childName = tokenPath.remove(0);
1278  byte[] childNameBytes = null;
1279  if (tokenPathBytes != null) {
1280  childNameBytes = tokenPathBytes.remove(0);
1281  }
1282  UnpackedNode child = parent.getChild(childName);
1283  // create new node
1284  if (child == null) {
1285  child = new UnpackedNode(childName, parent);
1286  child.setFileNameBytes(childNameBytes);
1287  parent.addChild(child);
1288  }
1289 
1290  // go down one more level
1291  return addNode(child, tokenPath, tokenPathBytes);
1292  }
1293 
1300  List<AbstractFile> getRootFileObjects() {
1301  List<AbstractFile> ret = new ArrayList<>();
1302  rootNode.getChildren().forEach((child) -> {
1303  ret.add(child.getFile());
1304  });
1305  return ret;
1306  }
1307 
1314  List<AbstractFile> getAllFileObjects() {
1315  List<AbstractFile> ret = new ArrayList<>();
1316  rootNode.getChildren().forEach((child) -> {
1317  getAllFileObjectsRec(ret, child);
1318  });
1319  return ret;
1320  }
1321 
1322  private void getAllFileObjectsRec(List<AbstractFile> list, UnpackedNode parent) {
1323  list.add(parent.getFile());
1324  parent.getChildren().forEach((child) -> {
1325  getAllFileObjectsRec(list, child);
1326  });
1327  }
1328 
1333  void updateOrAddFileToCaseRec(HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath) throws TskCoreException, NoCurrentCaseException {
1334  final FileManager fileManager = Case.getCurrentCaseThrows().getServices().getFileManager();
1335  for (UnpackedNode child : rootNode.getChildren()) {
1336  updateOrAddFileToCaseRec(child, fileManager, statusMap, archiveFilePath);
1337  }
1338  }
1339 
1354  private void updateOrAddFileToCaseRec(UnpackedNode node, FileManager fileManager, HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath) throws TskCoreException {
1355  DerivedFile df;
1356  progress.progress(String.format("%s: Adding/updating files in case database (%d of %d)", currentArchiveName, ++nodesProcessed, numItems));
1357  try {
1358  String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
1359  ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
1360  if (existingFile == null) {
1361  df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
1362  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1363  node.isIsFile(), node.getParent().getFile(), "", MODULE_NAME,
1364  "", "", TskData.EncodingType.XOR1);
1365  statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
1366  } else {
1367  String key = getKeyAbstractFile(existingFile.getFile());
1368  if (existingFile.getStatus() == ZipFileStatus.EXISTS && existingFile.getFile().getSize() < node.getSize()) {
1369  existingFile.setStatus(ZipFileStatus.UPDATE);
1370  statusMap.put(key, existingFile);
1371  }
1372  if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
1373  //if the we are updating a file and its mime type was octet-stream we want to re-type it
1374  String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
1375  df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
1376  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1377  node.isIsFile(), mimeType, "", MODULE_NAME,
1378  "", "", TskData.EncodingType.XOR1);
1379  } else {
1380  //ALREADY CURRENT - SKIP
1381  statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
1382  df = (DerivedFile) existingFile.getFile();
1383  }
1384  }
1385  node.setFile(df);
1386  } catch (TskCoreException ex) {
1387  logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
1388  throw new TskCoreException(
1389  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
1390  node.getFileName()), ex);
1391  }
1392 
1393  // Determine encoding of children
1394  if (node.getChildren().size() > 0) {
1395  String names = "";
1396  ArrayList<byte[]> byteDatas = new ArrayList<>();
1397  for (UnpackedNode child : node.getChildren()) {
1398  byte[] childBytes = child.getFileNameBytes();
1399  if (childBytes != null) {
1400  byteDatas.add(childBytes);
1401  }
1402  names += child.getFileName();
1403  }
1404  Charset detectedCharset = detectFilenamesCharset(byteDatas);
1405 
1406  // If a charset was detected, transcode filenames accordingly
1407  if (detectedCharset != null && detectedCharset.canEncode()) {
1408  for (UnpackedNode child : node.getChildren()) {
1409  byte[] childBytes = child.getFileNameBytes();
1410  if (childBytes != null) {
1411  String decodedName = new String(childBytes, detectedCharset);
1412  child.setFileName(decodedName);
1413  }
1414  }
1415  }
1416  }
1417 
1418  //recurse adding the children if this file was incomplete the children presumably need to be added
1419  for (UnpackedNode child : node.getChildren()) {
1420  updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath));
1421  }
1422  }
1423 
1427  private class UnpackedNode {
1428 
1429  private String fileName;
1430  private byte[] fileNameBytes;
1431  private AbstractFile file;
1432  private final List<UnpackedNode> children = new ArrayList<>();
1433  private String localRelPath = "";
1434  private long size;
1435  private long ctime, crtime, atime, mtime;
1436  private boolean isFile;
1437  private UnpackedNode parent;
1438 
1439  //root constructor
1440  UnpackedNode() {
1441  }
1442 
1443  //child node constructor
1444  UnpackedNode(String fileName, UnpackedNode parent) {
1445  this.fileName = fileName;
1446  this.parent = parent;
1447  this.localRelPath = parent.getLocalRelPath() + File.separator + fileName;
1448  }
1449 
1450  long getCtime() {
1451  return ctime;
1452  }
1453 
1454  long getCrtime() {
1455  return crtime;
1456  }
1457 
1458  long getAtime() {
1459  return atime;
1460  }
1461 
1462  long getMtime() {
1463  return mtime;
1464  }
1465 
1466  void setFileName(String fileName) {
1467  this.fileName = fileName;
1468  }
1469 
1475  void addChild(UnpackedNode child) {
1476  children.add(child);
1477  }
1478 
1485  List<UnpackedNode> getChildren() {
1486  return children;
1487  }
1488 
1494  UnpackedNode getParent() {
1495  return parent;
1496  }
1497 
1498  void addDerivedInfo(long size,
1499  boolean isFile,
1500  long ctime, long crtime, long atime, long mtime, String relLocalPath) {
1501  this.size = size;
1502  this.isFile = isFile;
1503  this.ctime = ctime;
1504  this.crtime = crtime;
1505  this.atime = atime;
1506  this.mtime = mtime;
1507  this.localRelPath = relLocalPath;
1508  }
1509 
1510  void setFile(AbstractFile file) {
1511  this.file = file;
1512  }
1513 
1521  UnpackedNode getChild(String childFileName) {
1522  UnpackedNode ret = null;
1523  for (UnpackedNode child : children) {
1524  if (child.getFileName().equals(childFileName)) {
1525  ret = child;
1526  break;
1527  }
1528  }
1529  return ret;
1530  }
1531 
1532  String getFileName() {
1533  return fileName;
1534  }
1535 
1536  AbstractFile getFile() {
1537  return file;
1538  }
1539 
1540  String getLocalRelPath() {
1541  return localRelPath;
1542  }
1543 
1550  void setLocalRelPath(String localRelativePath) {
1551  localRelPath = localRelativePath;
1552  }
1553 
1554  long getSize() {
1555  return size;
1556  }
1557 
1558  boolean isIsFile() {
1559  return isFile;
1560  }
1561 
1562  void setFileNameBytes(byte[] fileNameBytes) {
1563  if (fileNameBytes != null) {
1564  this.fileNameBytes = Arrays.copyOf(fileNameBytes, fileNameBytes.length);
1565  }
1566  }
1567 
1568  byte[] getFileNameBytes() {
1569  if (fileNameBytes == null) {
1570  return null;
1571  }
1572  return Arrays.copyOf(fileNameBytes, fileNameBytes.length);
1573  }
1574  }
1575  }
1576 
1581  static class Archive {
1582 
1583  //depth will be 0 for the root archive unpack was called on, and increase as unpack recurses down through archives contained within
1584  private final int depth;
1585  private final List<Archive> children;
1586  private final long rootArchiveId;
1587  private boolean flaggedAsZipBomb = false;
1588  private final AbstractFile archiveFile;
1589 
1602  Archive(int depth, long rootArchiveId, AbstractFile archiveFile) {
1603  this.children = new ArrayList<>();
1604  this.depth = depth;
1605  this.rootArchiveId = rootArchiveId;
1606  this.archiveFile = archiveFile;
1607  }
1608 
1615  void addChild(Archive child) {
1616  children.add(child);
1617  }
1618 
1623  synchronized void flagAsZipBomb() {
1624  flaggedAsZipBomb = true;
1625  }
1626 
1632  synchronized boolean isFlaggedAsZipBomb() {
1633  return flaggedAsZipBomb;
1634  }
1635 
1641  AbstractFile getArchiveFile() {
1642  return archiveFile;
1643  }
1644 
1650  long getRootArchiveId() {
1651  return rootArchiveId;
1652  }
1653 
1659  long getObjectId() {
1660  return archiveFile.getId();
1661  }
1662 
1670  int getDepth() {
1671  return depth;
1672  }
1673  }
1674 
1679  private final class ZipFileStatusWrapper {
1680 
1681  private final AbstractFile abstractFile;
1682  private ZipFileStatus zipStatus;
1683 
1691  private ZipFileStatusWrapper(AbstractFile file, ZipFileStatus status) {
1692  abstractFile = file;
1693  zipStatus = status;
1694  }
1695 
1701  private AbstractFile getFile() {
1702  return abstractFile;
1703  }
1704 
1710  private ZipFileStatus getStatus() {
1711  return zipStatus;
1712  }
1713 
1719  private void setStatus(ZipFileStatus status) {
1720  zipStatus = status;
1721  }
1722 
1723  }
1724 
1729  private enum ZipFileStatus {
1730  UPDATE, //Should be updated //NON-NLS
1731  SKIP, //File is current can be skipped //NON-NLS
1732  EXISTS //File exists but it is unknown if it is current //NON-NLS
1733  }
1734 }
org.sleuthkit.autopsy.modules.embeddedfileextractor.SevenZipExtractor.UnpackedTree.updateOrAddFileToCaseRec
void updateOrAddFileToCaseRec(UnpackedNode node, FileManager fileManager, HashMap< String, ZipFileStatusWrapper > statusMap, String archiveFilePath)
Definition: SevenZipExtractor.java:1354
org.sleuthkit.autopsy.casemodule.services.FileManager.addDerivedFile
synchronized DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
Definition: FileManager.java:372
org.sleuthkit.autopsy.modules.embeddedfileextractor.SevenZipExtractor.UnpackedTree.addNode
UnpackedNode addNode(UnpackedNode parent, List< String > tokenPath, List< byte[]> tokenPathBytes)
Definition: SevenZipExtractor.java:1269
org.sleuthkit.autopsy.casemodule.services.FileManager.updateDerivedFile
synchronized DerivedFile updateDerivedFile(DerivedFile derivedFile, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, String mimeType, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
Definition: FileManager.java:418
org.sleuthkit.autopsy.ingest.ModuleContentEvent
Definition: ModuleContentEvent.java:27

Copyright © 2012-2021 Basis Technology. Generated on: Tue Jan 19 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.