19 package org.sleuthkit.autopsy.ingest;
21 import java.io.Serializable;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.Date;
25 import java.util.LinkedHashMap;
26 import java.util.List;
28 import java.util.concurrent.CopyOnWriteArrayList;
29 import java.util.concurrent.LinkedBlockingQueue;
30 import java.util.concurrent.atomic.AtomicLong;
31 import java.util.logging.Level;
32 import java.util.regex.Matcher;
33 import java.util.regex.Pattern;
34 import java.util.stream.Stream;
35 import javax.swing.JOptionPane;
36 import org.netbeans.api.progress.ProgressHandle;
37 import org.openide.util.Cancellable;
38 import org.openide.util.NbBundle;
39 import org.openide.windows.WindowManager;
50 import org.
sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
52 import org.
sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
69 private static final Pattern
JYTHON_REGEX = Pattern.compile(
"org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$");
78 private static final AtomicLong
nextJobId =
new AtomicLong(0L);
79 private final long id;
82 private final List<AbstractFile>
files =
new ArrayList<>();
154 private static final IngestTasksScheduler
taskScheduler = IngestTasksScheduler.getInstance();
220 this.files.addAll(files);
222 this.doUI = runInteractively;
223 this.createTime =
new Date().getTime();
237 private static void addOrdered(
final List<IngestModuleTemplate> dest,
238 final Map<String, IngestModuleTemplate> src,
final Map<String, IngestModuleTemplate> jythonSrc) {
240 final List<IngestModuleTemplate> autopsyModules =
new ArrayList<>();
241 final List<IngestModuleTemplate> thirdPartyModules =
new ArrayList<>();
243 Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> {
244 if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) {
245 autopsyModules.add(templateEntry.getValue());
247 thirdPartyModules.add(templateEntry.getValue());
251 dest.addAll(autopsyModules);
252 dest.addAll(thirdPartyModules);
266 Matcher m = JYTHON_REGEX.matcher(canonicalName);
268 return String.format(
"%s.%s", m.group(1), m.group(2));
283 private static void addModule(Map<String, IngestModuleTemplate> mapping,
286 String className =
template.getModuleFactory().getClass().getCanonicalName();
288 if (jythonName != null) {
289 jythonMapping.put(jythonName,
template);
291 mapping.put(className,
template);
304 Map<String, IngestModuleTemplate> dataSourceModuleTemplates =
new LinkedHashMap<>();
305 Map<String, IngestModuleTemplate> fileModuleTemplates =
new LinkedHashMap<>();
309 Map<String, IngestModuleTemplate> jythonDataSourceModuleTemplates =
new LinkedHashMap<>();
310 Map<String, IngestModuleTemplate> jythonFileModuleTemplates =
new LinkedHashMap<>();
313 if (
template.isDataSourceIngestModuleTemplate()) {
314 addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates,
template);
316 if (
template.isFileIngestModuleTemplate()) {
317 addModule(fileModuleTemplates, jythonFileModuleTemplates,
template);
325 IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
327 dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
330 fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
333 dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
340 addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates);
341 addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates);
346 this.firstStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, firstStageDataSourceModuleTemplates);
347 this.secondStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, secondStageDataSourceModuleTemplates);
354 for (
int i = 0; i < numberOfFileIngestThreads; ++i) {
355 FileIngestPipeline pipeline =
new FileIngestPipeline(
this, fileIngestModuleTemplates);
356 this.fileIngestPipelinesQueue.put(pipeline);
357 this.fileIngestPipelines.add(pipeline);
359 }
catch (InterruptedException ex) {
365 Thread.currentThread().interrupt();
369 this.
addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
370 this.
addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
371 this.
addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
373 logErrorMessage(Level.WARNING,
"Failed to add ingest modules listing to case database", ex);
377 private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase)
throws TskCoreException {
379 ingestModules.add(skCase.addIngestModule(module.getModuleName(),
FactoryClassNameNormalizer.
normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
404 Map<String, IngestModuleTemplate> ingestModuleTemplates, Map<String, IngestModuleTemplate> jythonIngestModuleTemplates, List<String> pipelineConfig) {
405 List<IngestModuleTemplate> templates =
new ArrayList<>();
406 for (String moduleClassName : pipelineConfig) {
407 if (ingestModuleTemplates != null && ingestModuleTemplates.containsKey(moduleClassName)) {
408 templates.add(ingestModuleTemplates.remove(moduleClassName));
409 }
else if (jythonIngestModuleTemplates != null && jythonIngestModuleTemplates.containsKey(moduleClassName)) {
410 templates.add(jythonIngestModuleTemplates.remove(moduleClassName));
430 String getExecutionContext() {
439 Content getDataSource() {
449 boolean shouldProcessUnallocatedSpace() {
458 FilesSet getFileIngestFilter() {
467 boolean hasIngestPipeline() {
480 return (this.firstStageDataSourceIngestPipeline.isEmpty() ==
false);
490 return (this.secondStageDataSourceIngestPipeline.isEmpty() ==
false);
499 if (!this.fileIngestPipelines.isEmpty()) {
500 return !this.fileIngestPipelines.get(0).isEmpty();
510 List<IngestModuleError> start() {
512 if (errors.isEmpty()) {
515 }
catch (TskCoreException | NoCurrentCaseException ex) {
516 logErrorMessage(Level.WARNING,
"Failed to add ingest job info to case database", ex);
536 List<IngestModuleError> errors =
new ArrayList<>();
541 errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
542 errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
549 if (errors.isEmpty()) {
550 for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
551 errors.addAll(pipeline.startUp());
552 if (!errors.isEmpty()) {
558 while (!this.fileIngestPipelinesQueue.isEmpty()) {
559 FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
560 if (startedPipeline.isRunning()) {
561 List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
562 if (!shutDownErrors.isEmpty()) {
588 this.estimatedFilesToProcess = this.dataSource.accept(
new GetFilesCountVisitor());
616 logInfoMessage(
"Scheduling first stage data source and file level analysis tasks");
619 logInfoMessage(
"Scheduling first stage data source level analysis tasks");
622 logInfoMessage(
"Scheduling file level analysis tasks, no first stage data source level analysis configured");
649 logInfoMessage(
"Scheduling second stage data source level analysis tasks");
659 String displayName = NbBundle.getMessage(this.getClass(),
660 "IngestJob.progress.dataSourceIngest.initialDisplayName",
661 this.dataSource.getName());
662 this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
664 public boolean cancel() {
671 DataSourceIngestCancellationPanel panel =
new DataSourceIngestCancellationPanel();
672 String dialogTitle = NbBundle.getMessage(
DataSourceIngestJob.this.getClass(),
"IngestJob.cancellationDialog.title");
673 JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
674 if (panel.cancelAllDataSourceIngestModules()) {
682 this.dataSourceIngestProgress.start();
683 this.dataSourceIngestProgress.switchToIndeterminate();
694 String displayName = NbBundle.getMessage(this.getClass(),
695 "IngestJob.progress.fileIngest.displayName",
696 this.dataSource.getName());
697 this.fileIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
699 public boolean cancel() {
708 this.fileIngestProgress.start();
709 this.fileIngestProgress.switchToDeterminate((
int) this.estimatedFilesToProcess);
721 switch (this.stage) {
743 List<IngestModuleError> errors =
new ArrayList<>();
744 while (!this.fileIngestPipelinesQueue.isEmpty()) {
745 FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
746 if (pipeline.isRunning()) {
747 errors.addAll(pipeline.shutDown());
750 if (!errors.isEmpty()) {
758 if (this.dataSourceIngestProgress != null) {
759 this.dataSourceIngestProgress.finish();
760 this.dataSourceIngestProgress = null;
767 if (this.fileIngestProgress != null) {
768 this.fileIngestProgress.finish();
769 this.fileIngestProgress = null;
795 if (this.dataSourceIngestProgress != null) {
796 this.dataSourceIngestProgress.finish();
797 this.dataSourceIngestProgress = null;
801 if (ingestJob != null) {
802 if (this.cancelled) {
804 ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
805 }
catch (TskCoreException ex) {
806 logErrorMessage(Level.WARNING,
"Failed to update ingest job status in case database", ex);
810 ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
811 }
catch (TskCoreException ex) {
812 logErrorMessage(Level.WARNING,
"Failed to update ingest job status in case database", ex);
816 this.ingestJob.setEndDateTime(
new Date());
817 }
catch (TskCoreException ex) {
818 logErrorMessage(Level.WARNING,
"Failed to set job end date in case database", ex);
821 this.parentJob.dataSourceJobFinished(
this);
830 void process(DataSourceIngestTask task) {
833 if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
834 List<IngestModuleError> errors =
new ArrayList<>();
835 errors.addAll(this.currentDataSourceIngestPipeline.process(task));
836 if (!errors.isEmpty()) {
848 if (null != this.dataSourceIngestProgress) {
849 this.dataSourceIngestProgress.finish();
850 this.dataSourceIngestProgress = null;
856 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
872 void process(FileIngestTask task)
throws InterruptedException {
874 if (!this.isCancelled()) {
875 FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
876 if (!pipeline.isEmpty()) {
877 AbstractFile file = task.getFile();
885 if (this.processedFiles <= this.estimatedFilesToProcess) {
886 this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
888 this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
890 this.filesInProgress.add(file.getName());
897 List<IngestModuleError> errors =
new ArrayList<>();
898 errors.addAll(pipeline.process(task));
899 if (!errors.isEmpty()) {
903 if (this.doUI && !this.cancelled) {
909 this.filesInProgress.remove(file.getName());
910 if (this.filesInProgress.size() > 0) {
911 this.fileIngestProgress.progress(this.filesInProgress.get(0));
913 this.fileIngestProgress.progress(
"");
918 this.fileIngestPipelinesQueue.put(pipeline);
921 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
933 void addFiles(List<AbstractFile> files) {
934 if (DataSourceIngestJob.Stages.FIRST ==
this.stage) {
935 DataSourceIngestJob.taskScheduler.fastTrackFileIngestTasks(
this, files);
937 logErrorMessage(Level.SEVERE,
"Adding files to job during second stage analysis not supported");
955 void updateDataSourceIngestProgressBarDisplayName(String displayName) {
956 if (this.doUI && !this.cancelled) {
958 this.dataSourceIngestProgress.setDisplayName(displayName);
971 void switchDataSourceIngestProgressBarToDeterminate(
int workUnits) {
972 if (this.doUI && !this.cancelled) {
974 if (null != this.dataSourceIngestProgress) {
975 this.dataSourceIngestProgress.switchToDeterminate(workUnits);
986 void switchDataSourceIngestProgressBarToIndeterminate() {
987 if (this.doUI && !this.cancelled) {
989 if (null != this.dataSourceIngestProgress) {
990 this.dataSourceIngestProgress.switchToIndeterminate();
1002 void advanceDataSourceIngestProgressBar(
int workUnits) {
1003 if (this.doUI && !this.cancelled) {
1005 if (null != this.dataSourceIngestProgress) {
1006 this.dataSourceIngestProgress.progress(
"", workUnits);
1018 void advanceDataSourceIngestProgressBar(String currentTask) {
1019 if (this.doUI && !this.cancelled) {
1021 if (null != this.dataSourceIngestProgress) {
1022 this.dataSourceIngestProgress.progress(currentTask);
1036 void advanceDataSourceIngestProgressBar(String currentTask,
int workUnits) {
1037 if (this.doUI && !this.cancelled) {
1039 this.dataSourceIngestProgress.progress(currentTask, workUnits);
1051 boolean currentDataSourceIngestModuleIsCancelled() {
1061 void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
1062 this.currentDataSourceIngestModuleCancelled =
false;
1063 this.cancelledDataSourceIngestModules.add(moduleDisplayName);
1074 this.dataSourceIngestProgress.finish();
1075 this.dataSourceIngestProgress = null;
1086 DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
1087 if (null != this.currentDataSourceIngestPipeline) {
1088 return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
1098 void cancelCurrentDataSourceIngestModule() {
1099 this.currentDataSourceIngestModuleCancelled =
true;
1108 void cancel(IngestJob.CancellationReason reason) {
1109 this.cancelled =
true;
1111 DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(
this);
1115 if (null != dataSourceIngestProgress) {
1116 dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()));
1117 dataSourceIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling"));
1122 if (null != this.fileIngestProgress) {
1123 this.fileIngestProgress.setDisplayName(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.fileIngest.displayName", this.dataSource.getName()));
1124 this.fileIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling"));
1137 void setCurrentFileIngestModule(String moduleName, String taskName) {
1138 this.currentFileIngestModule = moduleName;
1139 this.currentFileIngestTask = taskName;
1148 boolean isCancelled() {
1157 IngestJob.CancellationReason getCancellationReason() {
1168 logger.log(Level.INFO, String.format(
"%s (data source = %s, objId = %d, jobId = %d)", message, dataSource.getName(), dataSource.getId(),
id));
1180 logger.log(level, String.format(
"%s (data source = %s, objId = %d, jobId = %d)", message, dataSource.getName(), dataSource.getId(),
id), throwable);
1191 logger.log(level, String.format(
"%s (data source = %s, objId = %d, jobId = %d)", message, dataSource.getName(), dataSource.getId(),
id));
1201 logErrorMessage(Level.SEVERE, String.format(
"%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable());
1210 Snapshot getSnapshot(
boolean getIngestTasksSnapshot) {
1216 boolean fileIngestRunning =
false;
1217 Date fileIngestStartTime = null;
1219 for (FileIngestPipeline pipeline : this.fileIngestPipelines) {
1220 if (pipeline.isRunning()) {
1221 fileIngestRunning =
true;
1223 Date pipelineStartTime = pipeline.getStartTime();
1224 if (null != pipelineStartTime && (null == fileIngestStartTime || pipelineStartTime.before(fileIngestStartTime))) {
1225 fileIngestStartTime = pipelineStartTime;
1229 long processedFilesCount = 0;
1230 long estimatedFilesToProcessCount = 0;
1231 long snapShotTime =
new Date().getTime();
1232 IngestJobTasksSnapshot tasksSnapshot = null;
1234 if (getIngestTasksSnapshot) {
1238 snapShotTime =
new Date().getTime();
1240 tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(
id);
1244 return new Snapshot(this.dataSource.getName(),
id,
createTime,
1245 getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime,
1247 processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot);
1253 public static final class Snapshot implements Serializable {
1275 Snapshot(String dataSourceName,
long jobId,
long jobStartTime, PipelineModule dataSourceIngestModule,
1276 boolean fileIngestRunning, Date fileIngestStartTime,
1278 long processedFiles,
long estimatedFilesToProcess,
1279 long snapshotTime, IngestJobTasksSnapshot tasksSnapshot) {
1280 this.dataSource = dataSourceName;
1283 this.dataSourceLevelIngestModule = dataSourceIngestModule;
1289 this.cancelledDataSourceModules = cancelledModules;
1293 this.snapShotTime = snapshotTime;
1303 long getSnapshotTime() {
1313 String getDataSource() {
1333 long getJobStartTime() {
1337 DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1341 boolean getFileIngestIsRunning() {
1345 Date getFileIngestStartTime() {
1356 return (
double) processedFiles / ((snapShotTime -
jobStartTime) / 1000);
1364 long getFilesProcessed() {
1374 long getFilesEstimated() {
1378 long getRootQueueSize() {
1379 if (null == this.tasksSnapshot) {
1382 return this.tasksSnapshot.getRootQueueSize();
1385 long getDirQueueSize() {
1386 if (null == this.tasksSnapshot) {
1389 return this.tasksSnapshot.getDirectoryTasksQueueSize();
1392 long getFileQueueSize() {
1393 if (null == this.tasksSnapshot) {
1396 return this.tasksSnapshot.getFileQueueSize();
1399 long getDsQueueSize() {
1400 if (null == this.tasksSnapshot) {
1403 return this.tasksSnapshot.getDsQueueSize();
1406 long getRunningListSize() {
1407 if (null == this.tasksSnapshot) {
1410 return this.tasksSnapshot.getRunningListSize();
1413 boolean isCancelled() {
1422 IngestJob.CancellationReason getCancellationReason() {
1433 List<String> getCancelledDataSourceIngestModules() {
1434 return Collections.unmodifiableList(this.cancelledDataSourceModules);
transient final List< String > cancelledDataSourceModules
void logErrorMessage(Level level, String message)
final List< AbstractFile > files
final boolean fileIngestRunning
transient final boolean jobCancelled
static synchronized IngestManager getInstance()
static void addOrdered(final List< IngestModuleTemplate > dest, final Map< String, IngestModuleTemplate > src, final Map< String, IngestModuleTemplate > jythonSrc)
final long processedFiles
void logErrorMessage(Level level, String message, Throwable throwable)
static List< IngestModuleTemplate > getConfiguredIngestModuleTemplates(Map< String, IngestModuleTemplate > ingestModuleTemplates, Map< String, IngestModuleTemplate > jythonIngestModuleTemplates, List< String > pipelineConfig)
static String getJythonName(String canonicalName)
transient final CancellationReason jobCancellationReason
void logIngestModuleErrors(List< IngestModuleError > errors)
final List< String > cancelledDataSourceIngestModules
DataSourceIngestPipeline firstStageDataSourceIngestPipeline
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
final IngestJobSettings settings
void checkForStageCompleted()
static final long serialVersionUID
final List< IngestModuleInfo > ingestModules
String getExecutionContext()
final Object stageCompletionCheckLock
volatile IngestJobInfo ingestJob
final IngestJobTasksSnapshot tasksSnapshot
void logInfoMessage(String message)
final Object dataSourceIngestPipelineLock
DataSourceIngestPipeline currentDataSourceIngestPipeline
transient final PipelineModule dataSourceLevelIngestModule
final long estimatedFilesToProcess
static void addModule(Map< String, IngestModuleTemplate > mapping, Map< String, IngestModuleTemplate > jythonMapping, IngestModuleTemplate template)
ProgressHandle fileIngestProgress
final Object dataSourceIngestProgressLock
String currentFileIngestTask
boolean hasFileIngestPipeline()
static String AUTOPSY_MODULE_PREFIX
boolean getProcessUnallocatedSpace()
boolean hasSecondStageDataSourceIngestPipeline()
volatile boolean currentDataSourceIngestModuleCancelled
SleuthkitCase getSleuthkitCase()
long estimatedFilesToProcess
void startDataSourceIngestProgressBar()
DataSourceIngestPipeline secondStageDataSourceIngestPipeline
boolean hasFirstStageDataSourceIngestPipeline()
int getNumberOfFileIngestThreads()
final Object fileIngestProgressLock
void addIngestModules(List< IngestModuleTemplate > templates, IngestModuleType type, SleuthkitCase skCase)
List< IngestModuleError > startUpIngestPipelines()
static String getLocalHostName()
synchronized static Logger getLogger(String name)
static final Logger logger
static Case getCurrentCaseThrows()
final List< String > filesInProgress
void startFileIngestProgressBar()
void createIngestPipelines()
static final Pattern JYTHON_REGEX
final LinkedBlockingQueue< FileIngestPipeline > fileIngestPipelinesQueue
final List< FileIngestPipeline > fileIngestPipelines
String currentFileIngestModule
final IngestJob parentJob
ProgressHandle dataSourceIngestProgress
final Date fileIngestStartTime
volatile IngestJob.CancellationReason cancellationReason
volatile boolean cancelled
static final IngestTasksScheduler taskScheduler
static final AtomicLong nextJobId
static String normalize(String canonicalClassName)