19 package org.sleuthkit.autopsy.modules.photoreccarver;
22 import java.io.IOException;
23 import java.lang.ProcessBuilder.Redirect;
24 import java.nio.file.DirectoryStream;
25 import java.nio.file.FileAlreadyExistsException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.text.DateFormat;
30 import java.text.SimpleDateFormat;
31 import java.util.ArrayList;
32 import java.util.Arrays;
33 import java.util.Date;
34 import java.util.HashMap;
35 import java.util.List;
37 import java.util.concurrent.ConcurrentHashMap;
38 import java.util.concurrent.atomic.AtomicLong;
39 import java.util.logging.Level;
40 import java.util.stream.Collectors;
41 import org.openide.modules.InstalledFileLocator;
42 import org.openide.util.NbBundle;
64 import org.
sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException;
72 "PhotoRecIngestModule.PermissionsNotSufficient=Insufficient permissions accessing",
73 "PhotoRecIngestModule.PermissionsNotSufficientSeeReference=See 'Shared Drive Authentication' in Autopsy help.",
74 "# {0} - output directory name",
"cannotCreateOutputDir.message=Unable to create output directory: {0}.",
75 "unallocatedSpaceProcessingSettingsError.message=The selected file ingest filter ignores unallocated space. This module carves unallocated space. Please choose a filter which does not ignore unallocated space or disable this module.",
76 "unsupportedOS.message=PhotoRec module is supported on Windows platforms only.",
77 "missingExecutable.message=Unable to locate PhotoRec executable.",
78 "cannotRunExecutable.message=Unable to execute PhotoRec.",
79 "PhotoRecIngestModule.nonHostnameUNCPathUsed=PhotoRec cannot operate with a UNC path containing IP addresses."
83 static final boolean DEFAULT_CONFIG_KEEP_CORRUPTED_FILES =
false;
84 static final PhotoRecCarverIngestJobSettings.ExtensionFilterOption DEFAULT_CONFIG_EXTENSION_FILTER
85 = PhotoRecCarverIngestJobSettings.ExtensionFilterOption.NO_FILTER;
87 static final boolean DEFAULT_CONFIG_INCLUDE_ELSE_EXCLUDE =
false;
89 private static final String PHOTOREC_TEMP_SUBDIR =
"PhotoRec Carver";
90 private static final String PHOTOREC_DIRECTORY =
"photorec_exec";
91 private static final String PHOTOREC_SUBDIRECTORY =
"bin";
92 private static final String PHOTOREC_EXECUTABLE =
"photorec_win.exe";
93 private static final String PHOTOREC_LINUX_EXECUTABLE =
"photorec";
94 private static final String PHOTOREC_RESULTS_BASE =
"results";
95 private static final String PHOTOREC_RESULTS_EXTENDED =
"results.1";
96 private static final String PHOTOREC_REPORT =
"report.xml";
97 private static final String LOG_FILE =
"run_log.txt";
98 private static final String SEP = System.getProperty(
"line.separator");
99 private static final Logger logger =
Logger.
getLogger(PhotoRecCarverFileIngestModule.class.getName());
100 private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs =
new HashMap<>();
102 private static final Map<Long, WorkingPaths> pathsByJob =
new ConcurrentHashMap<>();
104 private Path rootOutputDirPath;
105 private Path rootTempDirPath;
106 private File executableFile;
109 private final PhotoRecCarverIngestJobSettings settings;
110 private String optionsString;
115 private final AtomicLong totalItemsRecovered =
new AtomicLong(0);
116 private final AtomicLong totalItemsWithErrors =
new AtomicLong(0);
117 private final AtomicLong totalWritetime =
new AtomicLong(0);
118 private final AtomicLong totalParsetime =
new AtomicLong(0);
126 PhotoRecCarverFileIngestModule(PhotoRecCarverIngestJobSettings settings) {
127 this.settings = settings;
138 private String getPhotorecOptions(PhotoRecCarverIngestJobSettings settings) {
139 List<String> toRet =
new ArrayList<String>();
141 if (settings.isKeepCorruptedFiles()) {
142 toRet.addAll(Arrays.asList(
"options",
"keep_corrupted_file"));
145 if (settings.getExtensionFilterOption()
146 != PhotoRecCarverIngestJobSettings.ExtensionFilterOption.NO_FILTER) {
149 toRet.add(
"fileopt");
151 String enable =
"enable";
152 String disable =
"disable";
156 String everythingEnable = settings.getExtensionFilterOption()
157 == PhotoRecCarverIngestJobSettings.ExtensionFilterOption.INCLUDE
160 toRet.addAll(Arrays.asList(
"everything", everythingEnable));
162 final String itemEnable = settings.getExtensionFilterOption()
163 == PhotoRecCarverIngestJobSettings.ExtensionFilterOption.INCLUDE
166 settings.getExtensions().forEach((extension) -> {
167 toRet.addAll(Arrays.asList(extension, itemEnable));
172 return String.join(
",", toRet);
175 private static synchronized IngestJobTotals getTotalsForIngestJobs(
long ingestJobId) {
176 IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
177 if (totals == null) {
178 totals =
new PhotoRecCarverFileIngestModule.IngestJobTotals();
179 totalsForIngestJobs.put(ingestJobId, totals);
184 private static synchronized void initTotalsForIngestJob(
long ingestJobId) {
185 IngestJobTotals totals =
new PhotoRecCarverFileIngestModule.IngestJobTotals();
186 totalsForIngestJobs.put(ingestJobId, totals);
194 "# {0} - extensions",
195 "PhotoRecCarverFileIngestModule_startUp_invalidExtensions_description=The following extensions are invalid: {0}",
196 "PhotoRecCarverFileIngestModule_startUp_noExtensionsProvided_description=No extensions provided for PhotoRec to carve."
198 public void startUp(IngestJobContext context)
throws IngestModule.IngestModuleException {
200 if (this.settings.getExtensionFilterOption() != PhotoRecCarverIngestJobSettings.ExtensionFilterOption.NO_FILTER) {
201 if (this.settings.getExtensions().isEmpty()
202 && this.settings.getExtensionFilterOption() == PhotoRecCarverIngestJobSettings.ExtensionFilterOption.INCLUDE) {
204 throw new IngestModule.IngestModuleException(
205 Bundle.PhotoRecCarverFileIngestModule_startUp_noExtensionsProvided_description());
208 List<String> invalidExtensions = this.settings.getExtensions().stream()
209 .filter((ext) -> !PhotoRecCarverFileOptExtensions.isValidExtension(ext))
210 .collect(Collectors.toList());
212 if (!invalidExtensions.isEmpty()) {
213 throw new IngestModule.IngestModuleException(
214 Bundle.PhotoRecCarverFileIngestModule_startUp_invalidExtensions_description(
215 String.join(
",", invalidExtensions)));
219 this.optionsString = getPhotorecOptions(this.settings);
221 this.context = context;
223 this.jobId = this.context.getJobId();
229 if (!this.context.processingUnallocatedSpace()) {
230 throw new IngestModule.IngestModuleException(Bundle.unallocatedSpaceProcessingSettingsError_message());
233 this.rootOutputDirPath = createModuleOutputDirectoryForCase();
234 this.rootTempDirPath = createTempOutputDirectoryForCase();
237 executableFile = locateExecutable();
239 if (PhotoRecCarverFileIngestModule.refCounter.incrementAndGet(
this.jobId) == 1) {
242 DateFormat dateFormat =
new SimpleDateFormat(
"MM-dd-yyyy-HH-mm-ss-SSSS");
243 Date date =
new Date();
244 String folder = this.context.getDataSource().getId() +
"_" + dateFormat.format(date);
245 Path outputDirPath = Paths.get(this.rootOutputDirPath.toAbsolutePath().toString(), folder);
246 Files.createDirectories(outputDirPath);
249 Path tempDirPath = Paths.get(this.rootTempDirPath.toString(), folder);
250 Files.createDirectory(tempDirPath);
253 PhotoRecCarverFileIngestModule.pathsByJob.put(this.jobId,
new WorkingPaths(outputDirPath, tempDirPath));
256 initTotalsForIngestJob(jobId);
257 }
catch (SecurityException | IOException | UnsupportedOperationException ex) {
258 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
267 public IngestModule.ProcessResult process(AbstractFile file) {
269 if (file.getType() != TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) {
270 return IngestModule.ProcessResult.OK;
274 IngestJobTotals totals = getTotalsForIngestJobs(jobId);
276 Path tempFilePath = null;
279 if (null == this.executableFile) {
280 logger.log(Level.SEVERE,
"PhotoRec carver called after failed start up");
281 return IngestModule.ProcessResult.ERROR;
287 long freeDiskSpace = IngestServices.getInstance().getFreeDiskSpace();
288 if ((freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && ((file.getSize() * 1.2) > freeDiskSpace)) {
289 logger.log(Level.SEVERE,
"PhotoRec error processing {0} with {1} Not enough space on primary disk to save unallocated space.",
290 new Object[]{file.getName(), PhotoRecCarverIngestModuleFactory.getModuleName()});
291 MessageNotifyUtil.Notify.error(NbBundle.getMessage(
this.getClass(),
"PhotoRecIngestModule.UnableToCarve", file.getName()),
292 NbBundle.getMessage(
this.getClass(),
"PhotoRecIngestModule.NotEnoughDiskSpace"));
293 return IngestModule.ProcessResult.ERROR;
295 if (this.context.fileIngestIsCancelled() ==
true) {
297 logger.log(Level.INFO,
"PhotoRec cancelled by user");
298 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
299 return IngestModule.ProcessResult.OK;
303 long writestart = System.currentTimeMillis();
304 WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.get(this.jobId);
305 tempFilePath = Paths.get(paths.getTempDirPath().toString(), file.getName());
306 ContentUtils.writeToFile(file, tempFilePath.toFile(), context::fileIngestIsCancelled);
308 if (this.context.fileIngestIsCancelled() ==
true) {
310 logger.log(Level.INFO,
"PhotoRec cancelled by user");
311 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
312 return IngestModule.ProcessResult.OK;
316 Path outputDirPath = Paths.get(paths.getOutputDirPath().toString(), file.getName());
317 Files.createDirectory(outputDirPath);
318 File log =
new File(Paths.get(outputDirPath.toString(), LOG_FILE).toString());
321 ProcessBuilder processAndSettings =
new ProcessBuilder(
322 executableFile.toString(),
324 outputDirPath.toAbsolutePath().toString() + File.separator + PHOTOREC_RESULTS_BASE,
326 tempFilePath.toFile().toString());
328 processAndSettings.command().add(this.optionsString);
331 processAndSettings.environment().put(
"__COMPAT_LAYER",
"RunAsInvoker");
332 processAndSettings.redirectErrorStream(
true);
333 processAndSettings.redirectOutput(Redirect.appendTo(log));
335 FileIngestModuleProcessTerminator terminator =
new FileIngestModuleProcessTerminator(this.context,
true);
336 int exitValue = ExecUtil.execute(processAndSettings, terminator);
338 if (this.context.fileIngestIsCancelled() ==
true) {
340 cleanup(outputDirPath, tempFilePath);
341 logger.log(Level.INFO,
"PhotoRec cancelled by user");
342 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
343 return IngestModule.ProcessResult.OK;
344 }
else if (terminator.getTerminationCode() == ProcTerminationCode.TIME_OUT) {
345 cleanup(outputDirPath, tempFilePath);
346 String msg = NbBundle.getMessage(this.getClass(),
"PhotoRecIngestModule.processTerminated") + file.getName();
347 MessageNotifyUtil.Notify.error(NbBundle.getMessage(
this.getClass(),
"PhotoRecIngestModule.moduleError"), msg);
348 logger.log(Level.SEVERE, msg);
349 return IngestModule.ProcessResult.ERROR;
350 }
else if (0 != exitValue) {
352 cleanup(outputDirPath, tempFilePath);
353 totals.totalItemsWithErrors.incrementAndGet();
354 logger.log(Level.SEVERE,
"PhotoRec carver returned error exit value = {0} when scanning {1}",
355 new Object[]{exitValue, file.getName()});
356 MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.error.exitValue",
357 new Object[]{exitValue, file.getName()}));
358 return IngestModule.ProcessResult.ERROR;
362 java.io.File oldAuditFile =
new java.io.File(Paths.get(outputDirPath.toString(), PHOTOREC_RESULTS_EXTENDED, PHOTOREC_REPORT).toString());
363 java.io.File newAuditFile =
new java.io.File(Paths.get(outputDirPath.toString(), PHOTOREC_REPORT).toString());
364 oldAuditFile.renameTo(newAuditFile);
366 if (this.context.fileIngestIsCancelled() ==
true) {
368 logger.log(Level.INFO,
"PhotoRec cancelled by user");
369 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
370 return IngestModule.ProcessResult.OK;
372 Path pathToRemove = Paths.get(outputDirPath.toAbsolutePath().toString());
373 try (DirectoryStream<Path> stream = Files.newDirectoryStream(pathToRemove)) {
374 for (Path entry : stream) {
375 if (Files.isDirectory(entry)) {
376 FileUtil.deleteDir(
new File(entry.toString()));
380 long writedelta = (System.currentTimeMillis() - writestart);
381 totals.totalWritetime.addAndGet(writedelta);
384 long calcstart = System.currentTimeMillis();
385 PhotoRecCarverOutputParser parser =
new PhotoRecCarverOutputParser(outputDirPath);
386 if (this.context.fileIngestIsCancelled() ==
true) {
388 logger.log(Level.INFO,
"PhotoRec cancelled by user");
389 MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.cancelledByUser"));
390 return IngestModule.ProcessResult.OK;
392 List<LayoutFile> carvedItems = parser.parse(newAuditFile, file, context);
393 long calcdelta = (System.currentTimeMillis() - calcstart);
394 totals.totalParsetime.addAndGet(calcdelta);
395 if (carvedItems != null && !carvedItems.isEmpty()) {
396 totals.totalItemsRecovered.addAndGet(carvedItems.size());
397 context.addFilesToJob(
new ArrayList<>(carvedItems));
400 }
catch (ReadContentInputStreamException ex) {
401 totals.totalItemsWithErrors.incrementAndGet();
402 logger.log(Level.WARNING, String.format(
"Error reading file '%s' (id=%d) with the PhotoRec carver.", file.getName(), file.getId()), ex);
403 MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.error.msg", file.getName()));
404 return IngestModule.ProcessResult.ERROR;
405 }
catch (IOException ex) {
406 totals.totalItemsWithErrors.incrementAndGet();
407 logger.log(Level.SEVERE, String.format(
"Error writing or processing file '%s' (id=%d) to '%s' with the PhotoRec carver.", file.getName(), file.getId(), tempFilePath), ex);
408 MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.error.msg", file.getName()));
409 return IngestModule.ProcessResult.ERROR;
411 if (null != tempFilePath && Files.exists(tempFilePath)) {
413 tempFilePath.toFile().delete();
416 return IngestModule.ProcessResult.OK;
420 private void cleanup(Path outputDirPath, Path tempFilePath) {
422 FileUtil.deleteDir(
new File(outputDirPath.toString()));
423 if (null != tempFilePath && Files.exists(tempFilePath)) {
424 tempFilePath.toFile().delete();
428 private static synchronized void postSummary(
long jobId) {
429 IngestJobTotals jobTotals = totalsForIngestJobs.remove(jobId);
431 StringBuilder detailsSb =
new StringBuilder();
433 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
435 detailsSb.append(
"<tr><td>")
436 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.numberOfCarved"))
438 detailsSb.append(
"<td>").append(jobTotals.totalItemsRecovered.get()).append(
"</td></tr>");
440 detailsSb.append(
"<tr><td>")
441 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.numberOfErrors"))
443 detailsSb.append(
"<td>").append(jobTotals.totalItemsWithErrors.get()).append(
"</td></tr>");
445 detailsSb.append(
"<tr><td>")
446 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.totalWritetime"))
447 .append(
"</td><td>").append(jobTotals.totalWritetime.get()).append(
"</td></tr>\n");
448 detailsSb.append(
"<tr><td>")
449 .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
"PhotoRecIngestModule.complete.totalParsetime"))
450 .append(
"</td><td>").append(jobTotals.totalParsetime.get()).append(
"</td></tr>\n");
451 detailsSb.append(
"</table>");
453 IngestServices.getInstance().postMessage(IngestMessage.createMessage(
454 IngestMessage.MessageType.INFO,
455 PhotoRecCarverIngestModuleFactory.getModuleName(),
456 NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
457 "PhotoRecIngestModule.complete.photoRecResults"),
458 detailsSb.toString()));
466 public void shutDown() {
467 if (this.context != null && refCounter.decrementAndGet(
this.jobId) == 0) {
471 WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.remove(this.jobId);
472 FileUtil.deleteDir(
new File(paths.getTempDirPath().toString()));
474 }
catch (SecurityException ex) {
475 logger.log(Level.SEVERE,
"Error shutting down PhotoRec carver module", ex);
486 this.outputDirPath = outputDirPath;
487 this.tempDirPath = tempDirPath;
490 Path getOutputDirPath() {
491 return this.outputDirPath;
494 Path getTempDirPath() {
495 return this.tempDirPath;
507 synchronized Path createTempOutputDirectoryForCase() throws IngestModule.IngestModuleException {
509 Path path = Paths.get(Case.getCurrentCaseThrows().getTempDirectory(), PHOTOREC_TEMP_SUBDIR);
510 return createOutputDirectoryForCase(path);
511 }
catch (NoCurrentCaseException ex) {
512 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
524 synchronized Path createModuleOutputDirectoryForCase() throws IngestModule.IngestModuleException {
526 Path path = Paths.get(Case.getCurrentCaseThrows().getModuleDirectory(), PhotoRecCarverIngestModuleFactory.getModuleName());
527 return createOutputDirectoryForCase(path);
528 }
catch (NoCurrentCaseException ex) {
529 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
543 private synchronized Path createOutputDirectoryForCase(Path providedPath)
throws IngestModule.IngestModuleException {
544 Path path = providedPath;
546 Files.createDirectory(path);
547 if (UNCPathUtilities.isUNC(path)) {
551 throw new IngestModule.IngestModuleException(Bundle.PhotoRecIngestModule_nonHostnameUNCPathUsed());
553 if (
false == FileUtil.hasReadWriteAccess(path)) {
554 throw new IngestModule.IngestModuleException(
555 Bundle.PhotoRecIngestModule_PermissionsNotSufficient() + SEP + path.toString() + SEP
556 + Bundle.PhotoRecIngestModule_PermissionsNotSufficientSeeReference()
560 }
catch (FileAlreadyExistsException ex) {
562 }
catch (IOException | SecurityException | UnsupportedOperationException ex) {
563 throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
577 public static File locateExecutable() throws IngestModule.IngestModuleException {
580 String photorec_linux_directory =
"/usr/bin";
581 if (PlatformUtil.isWindowsOS()) {
582 execName = Paths.get(PHOTOREC_DIRECTORY, PHOTOREC_SUBDIRECTORY, PHOTOREC_EXECUTABLE);
583 exeFile = InstalledFileLocator.getDefault().locate(execName.toString(), PhotoRecCarverFileIngestModule.class.getPackage().getName(),
false);
585 File usrBin =
new File(
"/usr/bin/photorec");
586 File usrLocalBin =
new File(
"/usr/local/bin/photorec");
587 if (usrBin.canExecute() && usrBin.exists() && !usrBin.isDirectory()) {
588 photorec_linux_directory =
"/usr/bin";
589 }
else if (usrLocalBin.canExecute() && usrLocalBin.exists() && !usrLocalBin.isDirectory()) {
590 photorec_linux_directory =
"/usr/local/bin";
592 throw new IngestModule.IngestModuleException(
"Photorec not found");
594 execName = Paths.get(photorec_linux_directory, PHOTOREC_LINUX_EXECUTABLE);
595 exeFile =
new File(execName.toString());
598 if (null == exeFile) {
599 throw new IngestModule.IngestModuleException(Bundle.missingExecutable_message());
602 if (!exeFile.canExecute()) {
603 throw new IngestModule.IngestModuleException(Bundle.cannotRunExecutable_message());
void fireModuleContentEvent(ModuleContentEvent moduleContentEvent)
synchronized static Logger getLogger(String name)
synchronized Path ipToHostName(Path inputPath)
static synchronized IngestServices getInstance()