19 package org.sleuthkit.autopsy.modules.photoreccarver;
22 import java.io.IOException;
23 import java.lang.ProcessBuilder.Redirect;
24 import java.nio.file.DirectoryStream;
25 import java.nio.file.FileAlreadyExistsException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.text.DateFormat;
30 import java.text.SimpleDateFormat;
31 import java.util.ArrayList;
32 import java.util.Arrays;
33 import java.util.Date;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
39 import java.util.concurrent.ConcurrentHashMap;
40 import java.util.concurrent.atomic.AtomicLong;
41 import java.util.logging.Level;
42 import java.util.stream.Collectors;
43 import org.openide.modules.InstalledFileLocator;
44 import org.openide.util.NbBundle;
68 import org.
sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException;
78 "PhotoRecIngestModule.PermissionsNotSufficient=Insufficient permissions accessing",
79 "PhotoRecIngestModule.PermissionsNotSufficientSeeReference=See 'Shared Drive Authentication' in Autopsy help.",
80 "# {0} - output directory name",
"cannotCreateOutputDir.message=Unable to create output directory: {0}.",
81 "unallocatedSpaceProcessingSettingsError.message=The selected file ingest filter ignores unallocated space. This module carves unallocated space. Please choose a filter which does not ignore unallocated space or disable this module.",
82 "unsupportedOS.message=PhotoRec module is supported on Windows platforms only.",
83 "missingExecutable.message=Unable to locate PhotoRec executable.",
84 "cannotRunExecutable.message=Unable to execute PhotoRec.",
85 "PhotoRecIngestModule.nonHostnameUNCPathUsed=PhotoRec cannot operate with a UNC path containing IP addresses."
89 static final boolean DEFAULT_CONFIG_KEEP_CORRUPTED_FILES =
false;
90 static final PhotoRecCarverIngestJobSettings.ExtensionFilterOption DEFAULT_CONFIG_EXTENSION_FILTER
91 = PhotoRecCarverIngestJobSettings.ExtensionFilterOption.NO_FILTER;
93 static final boolean DEFAULT_CONFIG_INCLUDE_ELSE_EXCLUDE =
false;
95 private static final String PHOTOREC_TEMP_SUBDIR =
"PhotoRec Carver";
96 private static final String PHOTOREC_DIRECTORY =
"photorec_exec";
97 private static final String PHOTOREC_SUBDIRECTORY =
"bin";
98 private static final String PHOTOREC_EXECUTABLE =
"photorec_win.exe";
99 private static final String PHOTOREC_LINUX_EXECUTABLE =
"photorec";
100 private static final String PHOTOREC_RESULTS_BASE =
"results";
101 private static final String PHOTOREC_RESULTS_EXTENDED =
"results.1";
102 private static final String PHOTOREC_REPORT =
"report.xml";
103 private static final String LOG_FILE =
"run_log.txt";
104 private static final String SEP = System.getProperty(
"line.separator");
105 private static final Logger logger =
Logger.
getLogger(PhotoRecCarverFileIngestModule.class.getName());
106 private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs =
new HashMap<>();
108 private static final Map<Long, WorkingPaths> pathsByJob =
new ConcurrentHashMap<>();
110 private Path rootOutputDirPath;
111 private Path rootTempDirPath;
112 private File executableFile;
115 private final PhotoRecCarverIngestJobSettings settings;
116 private String optionsString;
121 private final AtomicLong totalItemsRecovered =
new AtomicLong(0);
122 private final AtomicLong totalItemsWithErrors =
new AtomicLong(0);
123 private final AtomicLong totalWritetime =
new AtomicLong(0);
124 private final AtomicLong totalParsetime =
new AtomicLong(0);
132 PhotoRecCarverFileIngestModule(PhotoRecCarverIngestJobSettings settings) {
133 this.settings = settings;
144 private String getPhotorecOptions(PhotoRecCarverIngestJobSettings settings) {
145 List<String> toRet =
new ArrayList<String>();
147 if (settings.isKeepCorruptedFiles()) {
148 toRet.addAll(Arrays.asList(
"options",
"keep_corrupted_file"));
151 if (settings.getExtensionFilterOption()
152 != PhotoRecCarverIngestJobSettings.ExtensionFilterOption.NO_FILTER) {
155 toRet.add(
"fileopt");
157 String enable =
"enable";
158 String disable =
"disable";
162 String everythingEnable = settings.getExtensionFilterOption()
163 == PhotoRecCarverIngestJobSettings.ExtensionFilterOption.INCLUDE
166 toRet.addAll(Arrays.asList(
"everything", everythingEnable));
168 final String itemEnable = settings.getExtensionFilterOption()
169 == PhotoRecCarverIngestJobSettings.ExtensionFilterOption.INCLUDE
172 settings.getExtensions().forEach((extension) -> {
173 toRet.addAll(Arrays.asList(extension, itemEnable));
178 return String.join(
",", toRet);
181 private static synchronized IngestJobTotals getTotalsForIngestJobs(
long ingestJobId) {
182 IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
183 if (totals == null) {
184 totals =
new PhotoRecCarverFileIngestModule.IngestJobTotals();
185 totalsForIngestJobs.put(ingestJobId, totals);
190 private static synchronized void initTotalsForIngestJob(
long ingestJobId) {
191 IngestJobTotals totals =
new PhotoRecCarverFileIngestModule.IngestJobTotals();
192 totalsForIngestJobs.put(ingestJobId, totals);
200 "# {0} - extensions",
201 "PhotoRecCarverFileIngestModule_startUp_invalidExtensions_description=The following extensions are invalid: {0}",
202 "PhotoRecCarverFileIngestModule_startUp_noExtensionsProvided_description=No extensions provided for PhotoRec to carve."
204 public void startUp(IngestJobContext context)
throws IngestModule.IngestModuleException {
206 if (this.settings.getExtensionFilterOption() != PhotoRecCarverIngestJobSettings.ExtensionFilterOption.NO_FILTER) {
207 if (this.settings.getExtensions().isEmpty()
208 && this.settings.getExtensionFilterOption() == PhotoRecCarverIngestJobSettings.ExtensionFilterOption.INCLUDE) {
210 throw new IngestModule.IngestModuleException(
211 Bundle.PhotoRecCarverFileIngestModule_startUp_noExtensionsProvided_description());
214 List<String> invalidExtensions = this.settings.getExtensions().stream()
215 .filter((ext) -> !PhotoRecCarverFileOptExtensions.isValidExtension(ext))
216 .collect(Collectors.toList());
218 if (!invalidExtensions.isEmpty()) {
219 throw new IngestModule.IngestModuleException(
220 Bundle.PhotoRecCarverFileIngestModule_startUp_invalidExtensions_description(
221 String.join(
",", invalidExtensions)));
225 this.optionsString = getPhotorecOptions(this.settings);
227 this.context = context;
229 this.jobId = this.context.getJobId();
235 if (!this.context.processingUnallocatedSpace()) {
236 throw new IngestModule.IngestModuleException(Bundle.unallocatedSpaceProcessingSettingsError_message());
239 this.rootOutputDirPath = createModuleOutputDirectoryForCase();
240 this.rootTempDirPath = createTempOutputDirectoryForCase();
243 executableFile = locateExecutable();
245 if (PhotoRecCarverFileIngestModule.refCounter.incrementAndGet(
this.jobId) == 1) {
248 DateFormat dateFormat =
new SimpleDateFormat(
"MM-dd-yyyy-HH-mm-ss-SSSS");
249 Date date =
new Date();
250 String folder = this.context.getDataSource().getId() +
"_" + dateFormat.format(date);
251 Path outputDirPath = Paths.get(this.rootOutputDirPath.toAbsolutePath().toString(), folder);
252 Files.createDirectories(outputDirPath);
255 Path tempDirPath = Paths.get(this.rootTempDirPath.toString(), folder);
256 Files.createDirectory(tempDirPath);
259 PhotoRecCarverFileIngestModule.pathsByJob.put(this.jobId,
new WorkingPaths(outputDirPath, tempDirPath));
262 initTotalsForIngestJob(jobId);
263 }
catch (SecurityException | IOException | UnsupportedOperationException ex) {
264 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
273 public IngestModule.ProcessResult process(AbstractFile file) {
275 if (file.getType() != TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) {
276 return IngestModule.ProcessResult.OK;
280 IngestJobTotals totals = getTotalsForIngestJobs(jobId);
282 Path tempFilePath = null;
285 if (null == this.executableFile) {
286 logger.log(Level.SEVERE,
"PhotoRec carver called after failed start up");
287 return IngestModule.ProcessResult.ERROR;
293 long freeDiskSpace = IngestServices.getInstance().getFreeDiskSpace();
294 if ((freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && ((file.getSize() * 1.2) > freeDiskSpace)) {
295 logger.log(Level.SEVERE,
"PhotoRec error processing {0} with {1} Not enough space on primary disk to save unallocated space.",
296 new Object[]{file.getName(), PhotoRecCarverIngestModuleFactory.getModuleName()});
297 MessageNotifyUtil.Notify.error(NbBundle.getMessage(
this.getClass(),
"PhotoRecIngestModule.UnableToCarve", file.getName()),
298 NbBundle.getMessage(
this.getClass(),
"PhotoRecIngestModule.NotEnoughDiskSpace"));
299 return IngestModule.ProcessResult.ERROR;
301 if (this.context.fileIngestIsCancelled() ==
true) {
303 logger.log(Level.INFO,
"PhotoRec cancelled by user");
304 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
305 return IngestModule.ProcessResult.OK;
309 long writestart = System.currentTimeMillis();
310 WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.get(this.jobId);
311 tempFilePath = Paths.get(paths.getTempDirPath().toString(), file.getName());
312 ContentUtils.writeToFile(file, tempFilePath.toFile(), context::fileIngestIsCancelled);
314 if (this.context.fileIngestIsCancelled() ==
true) {
316 logger.log(Level.INFO,
"PhotoRec cancelled by user");
317 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
318 return IngestModule.ProcessResult.OK;
322 Path outputDirPath = Paths.get(paths.getOutputDirPath().toString(), file.getName());
323 Files.createDirectory(outputDirPath);
324 File log =
new File(Paths.get(outputDirPath.toString(), LOG_FILE).toString());
327 ProcessBuilder processAndSettings =
new ProcessBuilder(
328 executableFile.toString(),
330 outputDirPath.toAbsolutePath().toString() + File.separator + PHOTOREC_RESULTS_BASE,
332 tempFilePath.toFile().toString());
334 processAndSettings.command().add(this.optionsString);
337 processAndSettings.environment().put(
"__COMPAT_LAYER",
"RunAsInvoker");
338 processAndSettings.redirectErrorStream(
true);
339 processAndSettings.redirectOutput(Redirect.appendTo(log));
341 FileIngestModuleProcessTerminator terminator =
new FileIngestModuleProcessTerminator(this.context,
true);
342 int exitValue = ExecUtil.execute(processAndSettings, terminator);
344 if (this.context.fileIngestIsCancelled() ==
true) {
346 cleanup(outputDirPath, tempFilePath);
347 logger.log(Level.INFO,
"PhotoRec cancelled by user");
348 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
349 return IngestModule.ProcessResult.OK;
350 }
else if (terminator.getTerminationCode() == ProcTerminationCode.TIME_OUT) {
351 cleanup(outputDirPath, tempFilePath);
352 String msg = NbBundle.getMessage(this.getClass(),
"PhotoRecIngestModule.processTerminated") + file.getName();
353 MessageNotifyUtil.Notify.error(NbBundle.getMessage(
this.getClass(),
"PhotoRecIngestModule.moduleError"), msg);
354 logger.log(Level.SEVERE, msg);
355 return IngestModule.ProcessResult.ERROR;
356 }
else if (0 != exitValue) {
358 cleanup(outputDirPath, tempFilePath);
359 totals.totalItemsWithErrors.incrementAndGet();
360 logger.log(Level.SEVERE,
"PhotoRec carver returned error exit value = {0} when scanning {1}",
361 new Object[]{exitValue, file.getName()});
362 MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.error.exitValue",
363 new Object[]{exitValue, file.getName()}));
364 return IngestModule.ProcessResult.ERROR;
368 java.io.File oldAuditFile =
new java.io.File(Paths.get(outputDirPath.toString(), PHOTOREC_RESULTS_EXTENDED, PHOTOREC_REPORT).toString());
369 java.io.File newAuditFile =
new java.io.File(Paths.get(outputDirPath.toString(), PHOTOREC_REPORT).toString());
370 oldAuditFile.renameTo(newAuditFile);
372 if (this.context.fileIngestIsCancelled() ==
true) {
374 logger.log(Level.INFO,
"PhotoRec cancelled by user");
375 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
376 return IngestModule.ProcessResult.OK;
378 Path pathToRemove = Paths.get(outputDirPath.toAbsolutePath().toString());
379 try (DirectoryStream<Path> stream = Files.newDirectoryStream(pathToRemove)) {
380 for (Path entry : stream) {
381 if (Files.isDirectory(entry)) {
382 FileUtil.deleteDir(
new File(entry.toString()));
386 long writedelta = (System.currentTimeMillis() - writestart);
387 totals.totalWritetime.addAndGet(writedelta);
390 long calcstart = System.currentTimeMillis();
391 PhotoRecCarverOutputParser parser =
new PhotoRecCarverOutputParser(outputDirPath);
392 if (this.context.fileIngestIsCancelled() ==
true) {
394 logger.log(Level.INFO,
"PhotoRec cancelled by user");
395 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
396 return IngestModule.ProcessResult.OK;
398 List<LayoutFile> carvedItems = parser.parse(newAuditFile, file, context);
399 long calcdelta = (System.currentTimeMillis() - calcstart);
400 totals.totalParsetime.addAndGet(calcdelta);
401 if (carvedItems != null && !carvedItems.isEmpty()) {
402 totals.totalItemsRecovered.addAndGet(carvedItems.size());
403 context.addFilesToJob(
new ArrayList<>(carvedItems));
407 List<AbstractFile> virtualParentDirs = getVirtualDirectoryParents(carvedItems);
408 for (AbstractFile virtualDir : virtualParentDirs) {
411 }
catch (TskCoreException ex) {
412 logger.log(Level.WARNING,
"Error collecting carved file parent directories", ex);
416 }
catch (ReadContentInputStreamException ex) {
417 totals.totalItemsWithErrors.incrementAndGet();
418 logger.log(Level.WARNING, String.format(
"Error reading file '%s' (id=%d) with the PhotoRec carver.", file.getName(), file.getId()), ex);
419 MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.error.msg", file.getName()));
420 return IngestModule.ProcessResult.ERROR;
421 }
catch (IOException ex) {
422 totals.totalItemsWithErrors.incrementAndGet();
423 logger.log(Level.SEVERE, String.format(
"Error writing or processing file '%s' (id=%d) to '%s' with the PhotoRec carver.", file.getName(), file.getId(), tempFilePath), ex);
424 MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.error.msg", file.getName()));
425 return IngestModule.ProcessResult.ERROR;
427 if (null != tempFilePath && Files.exists(tempFilePath)) {
429 tempFilePath.toFile().delete();
432 return IngestModule.ProcessResult.OK;
448 private List<AbstractFile> getVirtualDirectoryParents(List<LayoutFile> layoutFiles)
throws TskCoreException {
450 Set<Long> processedParentIds =
new HashSet<>();
456 List<AbstractFile> parentFiles =
new ArrayList<>();
457 for (LayoutFile file : layoutFiles) {
458 AbstractFile currentFile = file;
459 while (currentFile.getParentId().isPresent() && !processedParentIds.contains(currentFile.getParentId().get())) {
460 Content parent = currentFile.getParent();
461 processedParentIds.add(parent.getId());
462 if (! (parent instanceof VirtualDirectory)
463 || (currentFile instanceof DataSource)) {
469 currentFile = (AbstractFile)parent;
470 parentFiles.add(currentFile);
476 private void cleanup(Path outputDirPath, Path tempFilePath) {
478 FileUtil.deleteDir(
new File(outputDirPath.toString()));
479 if (null != tempFilePath && Files.exists(tempFilePath)) {
480 tempFilePath.toFile().delete();
484 private static synchronized void postSummary(
long jobId) {
485 IngestJobTotals jobTotals = totalsForIngestJobs.remove(jobId);
487 StringBuilder detailsSb =
new StringBuilder();
489 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
491 detailsSb.append(
"<tr><td>")
492 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.numberOfCarved"))
494 detailsSb.append(
"<td>").append(jobTotals.totalItemsRecovered.get()).append(
"</td></tr>");
496 detailsSb.append(
"<tr><td>")
497 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.numberOfErrors"))
499 detailsSb.append(
"<td>").append(jobTotals.totalItemsWithErrors.get()).append(
"</td></tr>");
501 detailsSb.append(
"<tr><td>")
502 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.totalWritetime"))
503 .append(
"</td><td>").append(jobTotals.totalWritetime.get()).append(
"</td></tr>\n");
504 detailsSb.append(
"<tr><td>")
505 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.totalParsetime"))
506 .append(
"</td><td>").append(jobTotals.totalParsetime.get()).append(
"</td></tr>\n");
507 detailsSb.append(
"</table>");
509 IngestServices.getInstance().postMessage(IngestMessage.createMessage(
510 IngestMessage.MessageType.INFO,
511 PhotoRecCarverIngestModuleFactory.getModuleName(),
512 NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
513 "PhotoRecIngestModule.complete.photoRecResults"),
514 detailsSb.toString()));
522 public void shutDown() {
523 if (this.context != null && refCounter.decrementAndGet(
this.jobId) == 0) {
527 WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.remove(this.jobId);
528 FileUtil.deleteDir(
new File(paths.getTempDirPath().toString()));
530 }
catch (SecurityException ex) {
531 logger.log(Level.SEVERE,
"Error shutting down PhotoRec carver module", ex);
542 this.outputDirPath = outputDirPath;
543 this.tempDirPath = tempDirPath;
546 Path getOutputDirPath() {
547 return this.outputDirPath;
550 Path getTempDirPath() {
551 return this.tempDirPath;
563 synchronized Path createTempOutputDirectoryForCase() throws IngestModule.IngestModuleException {
565 Path path = Paths.get(Case.getCurrentCaseThrows().getTempDirectory(), PHOTOREC_TEMP_SUBDIR);
566 return createOutputDirectoryForCase(path);
567 }
catch (NoCurrentCaseException ex) {
568 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
580 synchronized Path createModuleOutputDirectoryForCase() throws IngestModule.IngestModuleException {
582 Path path = Paths.get(Case.getCurrentCaseThrows().getModuleDirectory(), PhotoRecCarverIngestModuleFactory.getModuleName());
583 return createOutputDirectoryForCase(path);
584 }
catch (NoCurrentCaseException ex) {
585 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
599 private synchronized Path createOutputDirectoryForCase(Path providedPath)
throws IngestModule.IngestModuleException {
600 Path path = providedPath;
602 Files.createDirectory(path);
603 if (UNCPathUtilities.isUNC(path)) {
607 throw new IngestModule.IngestModuleException(Bundle.PhotoRecIngestModule_nonHostnameUNCPathUsed());
609 if (
false == FileUtil.hasReadWriteAccess(path)) {
610 throw new IngestModule.IngestModuleException(
611 Bundle.PhotoRecIngestModule_PermissionsNotSufficient() + SEP + path.toString() + SEP
612 + Bundle.PhotoRecIngestModule_PermissionsNotSufficientSeeReference()
616 }
catch (FileAlreadyExistsException ex) {
618 }
catch (IOException | SecurityException | UnsupportedOperationException ex) {
619 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
633 public static File locateExecutable() throws IngestModule.IngestModuleException {
636 String photorec_linux_directory =
"/usr/bin";
637 if (PlatformUtil.isWindowsOS()) {
638 execName = Paths.get(PHOTOREC_DIRECTORY, PHOTOREC_SUBDIRECTORY, PHOTOREC_EXECUTABLE);
639 exeFile = InstalledFileLocator.getDefault().locate(execName.toString(), PhotoRecCarverFileIngestModule.class.getPackage().getName(),
false);
641 File usrBin =
new File(
"/usr/bin/photorec");
642 File usrLocalBin =
new File(
"/usr/local/bin/photorec");
643 if (usrBin.canExecute() && usrBin.exists() && !usrBin.isDirectory()) {
644 photorec_linux_directory =
"/usr/bin";
645 }
else if (usrLocalBin.canExecute() && usrLocalBin.exists() && !usrLocalBin.isDirectory()) {
646 photorec_linux_directory =
"/usr/local/bin";
648 throw new IngestModule.IngestModuleException(
"Photorec not found");
650 execName = Paths.get(photorec_linux_directory, PHOTOREC_LINUX_EXECUTABLE);
651 exeFile =
new File(execName.toString());
654 if (null == exeFile) {
655 throw new IngestModule.IngestModuleException(Bundle.missingExecutable_message());
658 if (!exeFile.canExecute()) {
659 throw new IngestModule.IngestModuleException(Bundle.cannotRunExecutable_message());
void fireModuleContentEvent(ModuleContentEvent moduleContentEvent)
synchronized static Logger getLogger(String name)
synchronized Path ipToHostName(Path inputPath)
static synchronized IngestServices getInstance()