19 package org.sleuthkit.autopsy.modules.embeddedfileextractor;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.nio.charset.Charset;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.Date;
32 import java.util.HashMap;
33 import java.util.List;
35 import java.util.concurrent.ConcurrentHashMap;
36 import java.util.logging.Level;
37 import net.sf.sevenzipjbinding.ArchiveFormat;
38 import static net.sf.sevenzipjbinding.ArchiveFormat.RAR;
39 import net.sf.sevenzipjbinding.ExtractAskMode;
40 import net.sf.sevenzipjbinding.ExtractOperationResult;
41 import net.sf.sevenzipjbinding.IArchiveExtractCallback;
42 import net.sf.sevenzipjbinding.ICryptoGetTextPassword;
43 import net.sf.sevenzipjbinding.IInArchive;
44 import net.sf.sevenzipjbinding.ISequentialOutStream;
45 import net.sf.sevenzipjbinding.PropID;
46 import net.sf.sevenzipjbinding.SevenZip;
47 import net.sf.sevenzipjbinding.SevenZipException;
48 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
49 import org.apache.tika.Tika;
50 import org.apache.tika.parser.txt.CharsetDetector;
51 import org.apache.tika.parser.txt.CharsetMatch;
52 import org.netbeans.api.progress.ProgressHandle;
53 import org.openide.util.NbBundle;
54 import org.openide.util.NbBundle.Messages;
89 class SevenZipExtractor {
91 private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
93 private static final String MODULE_NAME = EmbeddedFileExtractorModuleFactory.getModuleName();
96 private static final String ENCRYPTION_FILE_LEVEL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
97 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
98 private static final String ENCRYPTION_FULL = EncryptionDetectionModuleFactory.PASSWORD_PROTECT_MESSAGE;
101 private static final int MAX_DEPTH = 4;
102 private static final int MAX_COMPRESSION_RATIO = 600;
103 private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
104 private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L;
106 private IngestServices services = IngestServices.getInstance();
107 private final IngestJobContext context;
108 private final FileTypeDetector fileTypeDetector;
109 private final FileTaskExecutor fileTaskExecutor;
111 private String moduleDirRelative;
112 private String moduleDirAbsolute;
114 private Blackboard blackboard;
116 private ProgressHandle progress;
117 private int numItems;
118 private String currentArchiveName;
132 XRAR(
"application/x-rar-compressed");
137 this.mimeType = mimeType;
142 return this.mimeType;
167 SevenZipExtractor(
IngestJobContext context,
FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute, FileTaskExecutor fileTaskExecutor)
throws SevenZipNativeInitializationException {
168 if (!SevenZip.isInitializedSuccessfully()) {
169 throw new SevenZipNativeInitializationException(
"SevenZip has not been previously initialized.");
171 this.context = context;
172 this.fileTypeDetector = fileTypeDetector;
173 this.moduleDirRelative = moduleDirRelative;
174 this.moduleDirAbsolute = moduleDirAbsolute;
175 this.fileTaskExecutor = fileTaskExecutor;
186 boolean isSevenZipExtractionSupported(AbstractFile file) {
187 String fileMimeType = fileTypeDetector.getMIMEType(file);
188 for (SupportedArchiveExtractionFormats mimeType : SupportedArchiveExtractionFormats.values()) {
189 if (checkForIngestCancellation(file)) {
192 if (mimeType.toString().equals(fileMimeType)) {
199 boolean isSevenZipExtractionSupported(String mimeType) {
200 for (SupportedArchiveExtractionFormats supportedMimeType : SupportedArchiveExtractionFormats.values()) {
201 if (mimeType.contains(supportedMimeType.toString())) {
218 private boolean checkForIngestCancellation(AbstractFile file) {
219 if (fileTaskExecutor != null && context != null && context.fileIngestIsCancelled()) {
220 logger.log(Level.INFO,
"Ingest was cancelled. Results extracted from the following archive file may be incomplete. Name: {0}Object ID: {1}",
new Object[]{file.getName(), file.getId()});
249 private boolean isZipBombArchiveItemCheck(AbstractFile archiveFile, IInArchive inArchive,
int inArchiveItemIndex, ConcurrentHashMap<Long, Archive> depthMap, String escapedFilePath) {
259 if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.UNALLOC) || archiveFile.getMIMEType().equalsIgnoreCase(SupportedArchiveExtractionFormats.XGZIP.toString())) {
264 final Long archiveItemSize = (Long) inArchive.getProperty(
265 inArchiveItemIndex, PropID.SIZE);
268 if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
272 final Long archiveItemPackedSize = (Long) inArchive.getProperty(
273 inArchiveItemIndex, PropID.PACKED_SIZE);
275 if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
276 logger.log(Level.WARNING,
"Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}",
277 new Object[]{archiveFile.getName(), (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH)});
281 int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
283 if (cRatio >= MAX_COMPRESSION_RATIO) {
284 Archive rootArchive = depthMap.get(depthMap.get(archiveFile.getId()).getRootArchiveId());
285 String details = NbBundle.getMessage(SevenZipExtractor.class,
286 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails",
287 cRatio, FileUtil.escapeFileName(getArchiveFilePath(rootArchive.getArchiveFile())));
289 flagRootArchiveAsZipBomb(rootArchive, archiveFile, details, escapedFilePath);
295 }
catch (SevenZipException ex) {
296 logger.log(Level.WARNING,
"Error getting archive item size and cannot detect if zipbomb. ", ex);
312 private void flagRootArchiveAsZipBomb(Archive rootArchive, AbstractFile archiveFile, String details, String escapedFilePath) {
313 rootArchive.flagAsZipBomb();
314 logger.log(Level.INFO, details);
316 String setName =
"Possible Zip Bomb";
318 Collection<BlackboardAttribute> attributes = Arrays.asList(
319 new BlackboardAttribute(
320 TSK_SET_NAME, MODULE_NAME,
322 new BlackboardAttribute(
323 TSK_DESCRIPTION, MODULE_NAME,
324 Bundle.SevenZipExtractor_zipBombArtifactCreation_text(archiveFile.getName())),
325 new BlackboardAttribute(
326 TSK_COMMENT, MODULE_NAME,
329 if (!blackboard.artifactExists(archiveFile, BlackboardArtifact.Type.TSK_INTERESTING_ITEM, attributes)) {
330 BlackboardArtifact artifact = rootArchive.getArchiveFile().newAnalysisResult(
331 BlackboardArtifact.Type.TSK_INTERESTING_ITEM, Score.SCORE_LIKELY_NOTABLE,
334 .getAnalysisResult();
342 blackboard.postArtifact(artifact, MODULE_NAME, context.getJobId());
344 String msg = NbBundle.getMessage(SevenZipExtractor.class,
345 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), escapedFilePath);
347 services.postMessage(IngestMessage.createWarningMessage(MODULE_NAME, msg, details));
349 }
catch (Blackboard.BlackboardException ex) {
350 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + artifact.getArtifactID(), ex);
351 MessageNotifyUtil.Notify.error(
352 Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
355 }
catch (TskCoreException ex) {
356 logger.log(Level.SEVERE,
"Error creating blackboard artifact for Zip Bomb Detection for file: " + escapedFilePath, ex);
368 private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
370 String detectedFormat;
371 detectedFormat = archiveFile.getMIMEType();
373 if (detectedFormat == null) {
374 logger.log(Level.WARNING,
"Could not detect format for file: {0}", archiveFile);
377 String extension = archiveFile.getNameExtension();
378 if (
"rar".equals(extension))
387 }
else if (detectedFormat.contains(
"application/x-rar-compressed"))
408 private long getRootArchiveId(AbstractFile file)
throws TskCoreException {
409 long id = file.getId();
410 Content parentContent = file.getParent();
411 while (parentContent != null) {
412 id = parentContent.getId();
413 parentContent = parentContent.getParent();
437 private List<AbstractFile> getAlreadyExtractedFiles(AbstractFile archiveFile, String archiveFilePath)
throws TskCoreException, InterruptedException, FileTaskExecutor.FileTaskFailedException {
441 List<AbstractFile> extractedFiles =
new ArrayList<>();
442 File outputDirectory =
new File(moduleDirAbsolute, EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
443 if (archiveFile.hasChildren() && fileTaskExecutor.exists(outputDirectory)) {
444 Case currentCase = Case.getCurrentCase();
445 FileManager fileManager = currentCase.getServices().getFileManager();
446 extractedFiles.addAll(fileManager.findFilesByParentPath(getRootArchiveId(archiveFile), archiveFilePath));
448 return extractedFiles;
458 private String getArchiveFilePath(AbstractFile archiveFile) {
459 return archiveFile.getParentPath() + archiveFile.getName();
471 private boolean makeExtractedFilesDirectory(String uniqueArchiveFileName) {
472 boolean success =
true;
473 Path rootDirectoryPath = Paths.get(moduleDirAbsolute, uniqueArchiveFileName);
474 File rootDirectory = rootDirectoryPath.toFile();
476 if (!fileTaskExecutor.exists(rootDirectory)) {
477 success = fileTaskExecutor.mkdirs(rootDirectory);
479 }
catch (SecurityException | FileTaskFailedException | InterruptedException ex) {
480 logger.log(Level.SEVERE, String.format(
"Error creating root extracted files directory %s", rootDirectory), ex);
498 private String getPathInArchive(IInArchive archive,
int inArchiveItemIndex, AbstractFile archiveFile)
throws SevenZipException {
499 String pathInArchive = (String) archive.getProperty(inArchiveItemIndex, PropID.PATH);
501 if (pathInArchive == null || pathInArchive.isEmpty()) {
507 String archName = archiveFile.getName();
508 int dotI = archName.lastIndexOf(
".");
509 String useName = null;
511 String base = archName.substring(0, dotI);
512 String ext = archName.substring(dotI);
513 int colonIndex = ext.lastIndexOf(
":");
514 if (colonIndex != -1) {
517 ext = ext.substring(0, colonIndex);
524 useName = base +
".tar";
531 if (useName == null) {
532 pathInArchive =
"/" + archName +
"/" + Integer.toString(inArchiveItemIndex);
534 pathInArchive =
"/" + useName;
537 return pathInArchive;
540 private byte[] getPathBytesInArchive(IInArchive archive,
int inArchiveItemIndex, AbstractFile archiveFile)
throws SevenZipException {
541 return (byte[]) archive.getProperty(inArchiveItemIndex, PropID.PATH_BYTES);
548 private String getKeyAbstractFile(AbstractFile fileInDatabase) {
549 return fileInDatabase == null ? null : fileInDatabase.getParentPath() + fileInDatabase.getName();
556 private String getKeyFromUnpackedNode(UnpackedTree.UnpackedNode node, String archiveFilePath) {
557 return node == null ? null : archiveFilePath +
"/" + node.getFileName();
567 void unpack(AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap) {
568 unpack(archiveFile, depthMap, null);
582 @Messages({
"SevenZipExtractor.indexError.message=Failed to index encryption detected artifact for keyword search.",
583 "# {0} - rootArchive",
584 "SevenZipExtractor.zipBombArtifactCreation.text=Zip Bomb Detected {0}"})
585 boolean unpack(AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap, String password) {
586 boolean unpackSuccessful =
true;
587 boolean hasEncrypted =
false;
588 boolean fullEncryption =
true;
589 boolean progressStarted =
false;
590 final String archiveFilePath = getArchiveFilePath(archiveFile);
591 final String escapedArchiveFilePath = FileUtil.escapeFileName(archiveFilePath);
592 HashMap<String, ZipFileStatusWrapper> statusMap =
new HashMap<>();
593 List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
595 currentArchiveName = archiveFile.getName();
597 SevenZipContentReadStream stream = null;
598 progress = ProgressHandle.createHandle(Bundle.EmbeddedFileExtractorIngestModule_ArchiveExtractor_moduleName());
602 blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
603 }
catch (NoCurrentCaseException ex) {
604 logger.log(Level.INFO,
"Exception while getting open case.", ex);
605 unpackSuccessful =
false;
606 return unpackSuccessful;
608 if (checkForIngestCancellation(archiveFile)) {
612 List<AbstractFile> existingFiles = getAlreadyExtractedFiles(archiveFile, archiveFilePath);
613 for (AbstractFile file : existingFiles) {
614 statusMap.put(getKeyAbstractFile(file),
new ZipFileStatusWrapper(file, ZipFileStatus.EXISTS));
616 }
catch (TskCoreException | FileTaskFailedException | InterruptedException ex) {
617 logger.log(Level.SEVERE, String.format(
"Error checking if %s has already been processed, skipping", escapedArchiveFilePath), ex);
618 unpackSuccessful =
false;
619 return unpackSuccessful;
621 if (checkForIngestCancellation(archiveFile)) {
624 parentAr = depthMap.get(archiveFile.getId());
625 if (parentAr == null) {
626 parentAr =
new Archive(0, archiveFile.getId(), archiveFile);
627 depthMap.put(archiveFile.getId(), parentAr);
629 Archive rootArchive = depthMap.get(parentAr.getRootArchiveId());
630 if (rootArchive.isFlaggedAsZipBomb()) {
632 unpackSuccessful =
false;
633 return unpackSuccessful;
634 }
else if (parentAr.getDepth() == MAX_DEPTH) {
635 String details = NbBundle.getMessage(SevenZipExtractor.class,
636 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
637 parentAr.getDepth(), FileUtil.escapeFileName(getArchiveFilePath(rootArchive.getArchiveFile())));
638 flagRootArchiveAsZipBomb(rootArchive, archiveFile, details, escapedArchiveFilePath);
639 unpackSuccessful =
false;
640 return unpackSuccessful;
643 if (checkForIngestCancellation(archiveFile)) {
646 IInArchive inArchive = null;
648 stream =
new SevenZipContentReadStream(
new ReadContentInputStream(archiveFile));
652 ArchiveFormat options = get7ZipOptions(archiveFile);
653 if (checkForIngestCancellation(archiveFile)) {
656 if (password == null) {
657 inArchive = SevenZip.openInArchive(options, stream);
659 inArchive = SevenZip.openInArchive(options, stream, password);
661 numItems = inArchive.getNumberOfItems();
662 progress.start(numItems);
663 progressStarted =
true;
664 if (checkForIngestCancellation(archiveFile)) {
668 final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
669 if (!makeExtractedFilesDirectory(uniqueArchiveFileName)) {
674 SevenZipExtractor.UnpackedTree unpackedTree =
new SevenZipExtractor.UnpackedTree(moduleDirRelative +
"/" + uniqueArchiveFileName, archiveFile);
678 freeDiskSpace = services.getFreeDiskSpace();
679 }
catch (NullPointerException ex) {
682 freeDiskSpace = IngestMonitor.DISK_FREE_SPACE_UNKNOWN;
685 Map<Integer, InArchiveItemDetails> archiveDetailsMap =
new HashMap<>();
686 for (
int inArchiveItemIndex = 0; inArchiveItemIndex < numItems; inArchiveItemIndex++) {
687 if (checkForIngestCancellation(archiveFile)) {
690 progress.progress(String.format(
"%s: Analyzing archive metadata and creating local files (%d of %d)", currentArchiveName, inArchiveItemIndex + 1, numItems), 0);
691 if (isZipBombArchiveItemCheck(archiveFile, inArchive, inArchiveItemIndex, depthMap, escapedArchiveFilePath)) {
692 unpackSuccessful =
false;
693 return unpackSuccessful;
696 String pathInArchive = getPathInArchive(inArchive, inArchiveItemIndex, archiveFile);
697 byte[] pathBytesInArchive = getPathBytesInArchive(inArchive, inArchiveItemIndex, archiveFile);
698 UnpackedTree.UnpackedNode unpackedNode = unpackedTree.addNode(pathInArchive, pathBytesInArchive);
699 if (checkForIngestCancellation(archiveFile)) {
702 final boolean isEncrypted = (Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.ENCRYPTED);
704 if (isEncrypted && password == null) {
705 logger.log(Level.WARNING,
"Skipping encrypted file in archive: {0}", pathInArchive);
707 unpackSuccessful =
false;
710 fullEncryption =
false;
717 Long archiveItemSize = (Long) inArchive.getProperty(
718 inArchiveItemIndex, PropID.SIZE);
719 if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && archiveItemSize != null && archiveItemSize > 0) {
720 String archiveItemPath = (String) inArchive.getProperty(
721 inArchiveItemIndex, PropID.PATH);
722 long newDiskSpace = freeDiskSpace - archiveItemSize;
723 if (newDiskSpace < MIN_FREE_DISK_SPACE) {
724 String msg = NbBundle.getMessage(SevenZipExtractor.class,
725 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
726 escapedArchiveFilePath, archiveItemPath);
727 String details = NbBundle.getMessage(SevenZipExtractor.class,
728 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
729 services.postMessage(IngestMessage.createErrorMessage(MODULE_NAME, msg, details));
730 logger.log(Level.INFO,
"Skipping archive item due to insufficient disk space: {0}, {1}",
new String[]{escapedArchiveFilePath, archiveItemPath});
731 logger.log(Level.INFO,
"Available disk space: {0}",
new Object[]{freeDiskSpace});
732 unpackSuccessful =
false;
736 freeDiskSpace = newDiskSpace;
739 if (checkForIngestCancellation(archiveFile)) {
742 final String uniqueExtractedName = FileUtil.escapeFileName(uniqueArchiveFileName + File.separator + (inArchiveItemIndex / 1000) + File.separator + inArchiveItemIndex);
743 final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
744 final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
748 File localFile =
new File(localAbsPath);
749 boolean localFileExists;
751 if ((Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER)) {
752 localFileExists = findOrCreateDirectory(localFile);
754 localFileExists = findOrCreateEmptyFile(localFile);
756 }
catch (FileTaskFailedException | InterruptedException ex) {
757 localFileExists =
false;
758 logger.log(Level.SEVERE, String.format(
"Error fiding or creating %s", localFile.getAbsolutePath()), ex);
760 if (checkForIngestCancellation(archiveFile)) {
765 if (!localFileExists) {
766 logger.log(Level.SEVERE, String.format(
"Skipping %s because it could not be created", localFile.getAbsolutePath()));
774 archiveDetailsMap.put(inArchiveItemIndex,
new InArchiveItemDetails(
775 unpackedNode, localAbsPath, localRelPath));
778 int[] extractionIndices = getExtractableFilesFromDetailsMap(archiveDetailsMap);
779 if (checkForIngestCancellation(archiveFile)) {
782 StandardIArchiveExtractCallback archiveCallBack
783 =
new StandardIArchiveExtractCallback(
784 inArchive, archiveFile, progress,
785 archiveDetailsMap, password, freeDiskSpace);
790 inArchive.extract(extractionIndices,
false, archiveCallBack);
791 if (checkForIngestCancellation(archiveFile)) {
794 unpackSuccessful &= archiveCallBack.wasSuccessful();
796 archiveDetailsMap = null;
801 unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath, parentAr, archiveFile, depthMap);
802 unpackedTree.commitCurrentTransaction();
803 }
catch (TskCoreException | NoCurrentCaseException ex) {
804 logger.log(Level.SEVERE,
"Error populating complete derived file hierarchy from the unpacked dir structure", ex);
806 unpackedTree.rollbackCurrentTransaction();
809 if (checkForIngestCancellation(archiveFile)) {
814 unpackedFiles = unpackedTree.getAllFileObjects();
815 }
catch (SevenZipException | IllegalArgumentException ex) {
816 logger.log(Level.WARNING,
"Error unpacking file: " + archiveFile, ex);
820 if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
821 String msg = NbBundle.getMessage(SevenZipExtractor.class,
822 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
824 String details = NbBundle.getMessage(SevenZipExtractor.class,
825 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
826 escapedArchiveFilePath, ex.getMessage());
827 services.postMessage(IngestMessage.createErrorMessage(MODULE_NAME, msg, details));
830 if (inArchive != null) {
833 }
catch (SevenZipException e) {
834 logger.log(Level.SEVERE,
"Error closing archive: " + archiveFile, e);
838 if (stream != null) {
841 }
catch (IOException ex) {
842 logger.log(Level.SEVERE,
"Error closing stream after unpacking archive: " + archiveFile, ex);
847 if (progressStarted) {
851 if (checkForIngestCancellation(archiveFile)) {
856 String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
858 BlackboardArtifact artifact = archiveFile.newAnalysisResult(
859 new BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED),
861 null, null, encryptionType,
862 Arrays.asList(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, encryptionType)))
863 .getAnalysisResult();
871 blackboard.postArtifact(artifact, MODULE_NAME, context.getJobId());
872 }
catch (Blackboard.BlackboardException ex) {
873 logger.log(Level.SEVERE,
"Unable to post blackboard artifact " + artifact.getArtifactID(), ex);
874 MessageNotifyUtil.Notify.error(
875 Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
878 }
catch (TskCoreException ex) {
879 logger.log(Level.SEVERE,
"Error creating blackboard artifact for encryption detected for file: " + escapedArchiveFilePath, ex);
882 String msg = NbBundle.getMessage(SevenZipExtractor.class,
883 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
884 String details = NbBundle.getMessage(SevenZipExtractor.class,
885 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
886 currentArchiveName, MODULE_NAME);
887 services.postMessage(IngestMessage.createWarningMessage(MODULE_NAME, msg, details));
891 if (!unpackedFiles.isEmpty()) {
893 services.fireModuleContentEvent(
new ModuleContentEvent(archiveFile));
894 if (context != null) {
895 context.addFilesToJob(unpackedFiles);
899 return unpackSuccessful;
909 private boolean findOrCreateDirectory(File directory)
throws FileTaskFailedException, InterruptedException {
910 if (!fileTaskExecutor.exists(directory)) {
911 return fileTaskExecutor.mkdirs(directory);
924 private boolean findOrCreateEmptyFile(File file)
throws FileTaskFailedException, InterruptedException {
925 if (!fileTaskExecutor.exists(file)) {
926 fileTaskExecutor.mkdirs(file.getParentFile());
927 return fileTaskExecutor.createNewFile(file);
933 private Charset detectFilenamesCharset(List<byte[]> byteDatas) {
934 Charset detectedCharset = null;
935 CharsetDetector charsetDetector =
new CharsetDetector();
938 for (byte[] byteData : byteDatas) {
940 byteSum += byteData.length;
942 if (byteSum >= 1000) {
946 byte[] allBytes =
new byte[byteSum];
948 for (
int i = 0; i < fileNum; i++) {
949 byte[] byteData = byteDatas.get(i);
950 System.arraycopy(byteData, 0, allBytes, start, byteData.length);
951 start += byteData.length;
953 charsetDetector.setText(allBytes);
954 CharsetMatch cm = charsetDetector.detect();
955 if (cm != null && cm.getConfidence() >= 90 && Charset.isSupported(cm.getName())) {
956 detectedCharset = Charset.forName(cm.getName());
958 return detectedCharset;
965 private int[] getExtractableFilesFromDetailsMap(
966 Map<Integer, InArchiveItemDetails> archiveDetailsMap) {
968 Integer[] wrappedExtractionIndices = archiveDetailsMap.keySet()
969 .toArray(
new Integer[archiveDetailsMap.size()]);
971 return Arrays.stream(wrappedExtractionIndices)
972 .mapToInt(Integer::intValue)
984 private final static class UnpackStream implements ISequentialOutStream {
989 private static final Tika
tika =
new Tika();
990 private String mimeType =
"";
995 this.bytesWritten = 0;
1002 this.bytesWritten = 0;
1011 public int write(byte[] bytes)
throws SevenZipException {
1014 if (bytesWritten == 0) {
1015 mimeType = tika.detect(bytes);
1017 output.
write(bytes);
1018 this.bytesWritten += bytes.length;
1019 }
catch (IOException ex) {
1020 throw new SevenZipException(
1021 NbBundle.getMessage(SevenZipExtractor.class,
1022 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
1025 return bytes.length;
1032 public void close() throws IOException {
1050 SevenZipExtractor.UnpackedTree.UnpackedNode
unpackedNode,
1051 String localAbsPath, String localRelPath) {
1075 implements IArchiveExtractCallback, ICryptoGetTextPassword {
1092 private boolean unpackSuccessful =
true;
1095 AbstractFile archiveFile, ProgressHandle progressHandle,
1096 Map<Integer, InArchiveItemDetails> archiveDetailsMap,
1097 String password,
long freeDiskSpace) {
1120 public ISequentialOutStream
getStream(
int inArchiveItemIndex,
1121 ExtractAskMode mode)
throws SevenZipException {
1125 isFolder = (Boolean) inArchive
1126 .getProperty(inArchiveItemIndex, PropID.IS_FOLDER);
1127 if (isFolder || mode != ExtractAskMode.EXTRACT) {
1131 final String localAbsPath = archiveDetailsMap.get(
1132 inArchiveItemIndex).getLocalAbsPath();
1140 if (unpackStream != null) {
1145 }
catch (IOException ex) {
1146 logger.log(Level.WARNING, String.format(
"Error opening or setting new stream "
1147 +
"for archive file at %s", localAbsPath), ex.getMessage());
1164 final Date createTime = (Date) inArchive.getProperty(
1165 inArchiveItemIndex, PropID.CREATION_TIME);
1166 final Date accessTime = (Date) inArchive.getProperty(
1167 inArchiveItemIndex, PropID.LAST_ACCESS_TIME);
1168 final Date writeTime = (Date) inArchive.getProperty(
1169 inArchiveItemIndex, PropID.LAST_MODIFICATION_TIME);
1171 createTimeInSeconds = createTime == null ? 0L
1172 : createTime.getTime() / 1000;
1173 modTimeInSeconds = writeTime == null ? 0L
1174 : writeTime.getTime() / 1000;
1175 accessTimeInSeconds = accessTime == null ? 0L
1176 : accessTime.getTime() / 1000;
1178 progressHandle.progress(archiveFile.
getName() +
": "
1179 + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH),
1195 final SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode
1196 = archiveDetailsMap.get(inArchiveItemIndex).getUnpackedNode();
1197 final String localRelPath = archiveDetailsMap.get(
1198 inArchiveItemIndex).getLocalRelPath();
1200 unpackedNode.addDerivedInfo(0,
1201 !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER),
1206 unpackedNode.setMimeType(unpackStream.
getMIMEType());
1209 final String localAbsPath = archiveDetailsMap.get(
1210 inArchiveItemIndex).getLocalAbsPath();
1211 if (result != ExtractOperationResult.OK) {
1213 logger.log(Level.WARNING,
"Extraction of : {0} encountered error {1} (file is unallocated and may be corrupt)",
1214 new Object[]{localAbsPath, result});
1216 logger.log(Level.WARNING,
"Extraction of : {0} encountered error {1}",
1217 new Object[]{localAbsPath, result});
1219 unpackSuccessful =
false;
1223 unpackedNode.addDerivedInfo(unpackStream.
getSize(),
1224 !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER),
1228 unpackStream.
close();
1229 }
catch (IOException e) {
1230 logger.log(Level.WARNING,
"Error closing unpack stream for file: {0}", localAbsPath);
1235 public void setTotal(
long value)
throws SevenZipException {
1290 this.rootNode.setFile(archiveFile);
1291 this.rootNode.setFileName(archiveFile.
getName());
1292 this.rootNode.setLocalRelPath(localPathRoot);
1304 UnpackedNode addNode(String filePath, byte[] filePathBytes) {
1305 String[] toks = filePath.split(
"[\\/\\\\]");
1306 List<String> tokens =
new ArrayList<>();
1307 for (
int i = 0; i < toks.length; ++i) {
1308 if (!toks[i].isEmpty()) {
1309 tokens.add(toks[i]);
1313 List<byte[]> byteTokens;
1314 if (filePathBytes == null) {
1315 return addNode(rootNode, tokens, null);
1317 byteTokens =
new ArrayList<>(tokens.size());
1319 for (
int i = 0; i < filePathBytes.length; i++) {
1320 if (filePathBytes[i] ==
'/') {
1323 byte[] arr =
new byte[len];
1324 System.arraycopy(filePathBytes, last, arr, 0, len);
1325 byteTokens.add(arr);
1330 int len = filePathBytes.length - last;
1332 byte[] arr =
new byte[len];
1333 System.arraycopy(filePathBytes, last, arr, 0, len);
1334 byteTokens.add(arr);
1337 if (tokens.size() != byteTokens.size()) {
1338 String rootFileInfo =
"(unknown)";
1339 if (rootNode.getFile() != null) {
1341 +
"(ID: " + rootNode.getFile().
getId() +
")";
1343 logger.log(Level.WARNING,
"Could not map path bytes to path string while extracting archive {0} (path string: \"{1}\", bytes: {2})",
1344 new Object[]{rootFileInfo, this.rootNode.getFile().getId(), filePath, bytesToString(filePathBytes)});
1345 return addNode(rootNode, tokens, null);
1349 return addNode(rootNode, tokens, byteTokens);
1360 StringBuilder result =
new StringBuilder();
1361 for (byte b : bytes) {
1362 result.append(String.format(
"%02x", b));
1364 return result.toString();
1377 List<String> tokenPath, List<byte[]> tokenPathBytes) {
1379 if (tokenPath.isEmpty()) {
1384 String childName = tokenPath.remove(0);
1385 byte[] childNameBytes = null;
1386 if (tokenPathBytes != null) {
1387 childNameBytes = tokenPathBytes.remove(0);
1391 if (child == null) {
1393 child.setFileNameBytes(childNameBytes);
1394 parent.addChild(child);
1398 return addNode(child, tokenPath, tokenPathBytes);
1407 List<AbstractFile> getRootFileObjects() {
1408 List<AbstractFile> ret =
new ArrayList<>();
1409 rootNode.getChildren().forEach((child) -> {
1410 ret.add(child.getFile());
1421 List<AbstractFile> getAllFileObjects() {
1422 List<AbstractFile> ret =
new ArrayList<>();
1423 rootNode.getChildren().forEach((child) -> {
1430 list.add(parent.getFile());
1431 parent.getChildren().forEach((child) -> {
1440 void updateOrAddFileToCaseRec(HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath, Archive parentAr,
AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap)
throws TskCoreException,
NoCurrentCaseException {
1442 for (UnpackedNode child : rootNode.getChildren()) {
1443 updateOrAddFileToCaseRec(child, fileManager, statusMap, archiveFilePath, parentAr, archiveFile, depthMap);
1466 progress.progress(String.format(
"%s: Adding/updating files in case database (%d of %d)", currentArchiveName, ++nodesProcessed, numItems));
1468 String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
1469 ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
1470 if (existingFile == null) {
1472 node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1473 node.isIsFile(), node.getParent().getFile(),
"", MODULE_NAME,
1477 String key = getKeyAbstractFile(existingFile.
getFile());
1480 statusMap.put(key, existingFile);
1486 node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1487 node.isIsFile(), mimeType,
"", MODULE_NAME,
1497 logger.log(Level.SEVERE,
"Error adding a derived file to db:" + node.getFileName(), ex);
1498 throw new TskCoreException(
1499 NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
1500 node.getFileName()), ex);
1504 if (node.getChildren().size() > 0) {
1506 ArrayList<byte[]> byteDatas =
new ArrayList<>();
1508 byte[] childBytes = child.getFileNameBytes();
1509 if (childBytes != null) {
1510 byteDatas.add(childBytes);
1512 names += child.getFileName();
1514 Charset detectedCharset = detectFilenamesCharset(byteDatas);
1517 if (detectedCharset != null && detectedCharset.canEncode()) {
1519 byte[] childBytes = child.getFileNameBytes();
1520 if (childBytes != null) {
1521 String decodedName =
new String(childBytes, detectedCharset);
1522 child.setFileName(decodedName);
1529 if (isSevenZipExtractionSupported(node.getMimeType())) {
1530 Archive child =
new Archive(parentAr.getDepth() + 1, parentAr.getRootArchiveId(), archiveFile);
1531 parentAr.addChild(child);
1532 depthMap.put(node.getFile().getId(), child);
1537 updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath), parentAr, archiveFile, depthMap);
1553 if (currentTransaction == null) {
1557 if (transactionCounter > MAX_TRANSACTION_SIZE) {
1562 transactionCounter++;
1574 transactionCounter = 0;
1576 throw new TskCoreException(
"Case is closed");
1586 if (currentTransaction != null) {
1587 currentTransaction.
commit();
1588 currentTransaction = null;
1596 if (currentTransaction != null) {
1599 currentTransaction = null;
1600 }
catch (TskCoreException ex) {
1614 private final List<UnpackedNode>
children =
new ArrayList<>();
1615 private String localRelPath =
"";
1617 private long ctime, crtime, atime, mtime;
1619 private String mimeType =
"";
1630 this.localRelPath = parent.getLocalRelPath() + File.separator +
fileName;
1649 void setFileName(String fileName) {
1658 void addChild(UnpackedNode child) {
1659 children.add(child);
1668 List<UnpackedNode> getChildren() {
1677 UnpackedNode getParent() {
1681 void addDerivedInfo(
long size,
1683 long ctime,
long crtime,
long atime,
long mtime, String relLocalPath) {
1687 this.crtime = crtime;
1690 this.localRelPath = relLocalPath;
1693 void setFile(AbstractFile file) {
1697 void setMimeType(String mimeType) {
1701 String getMimeType() {
1712 UnpackedNode getChild(String childFileName) {
1713 UnpackedNode ret = null;
1714 for (UnpackedNode child : children) {
1715 if (child.getFileName().equals(childFileName)) {
1723 String getFileName() {
1727 AbstractFile getFile() {
1731 String getLocalRelPath() {
1741 void setLocalRelPath(String localRelativePath) {
1742 localRelPath = localRelativePath;
1749 boolean isIsFile() {
1753 void setFileNameBytes(byte[] fileNameBytes) {
1754 if (fileNameBytes != null) {
1755 this.fileNameBytes = Arrays.copyOf(fileNameBytes, fileNameBytes.length);
1759 byte[] getFileNameBytes() {
1760 if (fileNameBytes == null) {
1763 return Arrays.copyOf(fileNameBytes, fileNameBytes.length);
1772 static class Archive {
1775 private final int depth;
1776 private final List<Archive> children;
1777 private final long rootArchiveId;
1778 private boolean flaggedAsZipBomb =
false;
1779 private final AbstractFile archiveFile;
1793 Archive(
int depth,
long rootArchiveId, AbstractFile archiveFile) {
1794 this.children =
new ArrayList<>();
1796 this.rootArchiveId = rootArchiveId;
1797 this.archiveFile = archiveFile;
1806 void addChild(Archive child) {
1807 children.add(child);
1814 synchronized void flagAsZipBomb() {
1815 flaggedAsZipBomb =
true;
1823 synchronized boolean isFlaggedAsZipBomb() {
1824 return flaggedAsZipBomb;
1832 AbstractFile getArchiveFile() {
1841 long getRootArchiveId() {
1842 return rootArchiveId;
1850 long getObjectId() {
1851 return archiveFile.getId();
1883 abstractFile = file;
FileManager getFileManager()
CaseDbTransaction beginTransaction()
DerivedFile updateDerivedFile(DerivedFile derivedFile, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, String mimeType, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
boolean isMetaFlagSet(TSK_FS_META_FLAG_ENUM metaFlag)
SleuthkitCase getSleuthkitCase()
static Case getCurrentCaseThrows()