19 package org.sleuthkit.autopsy.ingest;
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.List;
28 import java.util.concurrent.ConcurrentHashMap;
29 import java.util.concurrent.atomic.AtomicInteger;
30 import java.util.concurrent.atomic.AtomicLong;
31 import java.util.logging.Level;
32 import org.openide.util.NbBundle;
59 this.displayName = displayName;
76 private final static AtomicLong
nextId =
new AtomicLong(0L);
77 private final long id;
79 private final List<AbstractFile>
files =
new ArrayList<>();
96 this.ingestJobPipelines =
new ConcurrentHashMap<>();
97 this.ingestMode = Mode.BATCH;
98 this.dataSources.addAll(dataSources);
99 incompleteJobsCount =
new AtomicInteger(dataSources.size());
113 this(Arrays.asList(dataSource),
settings);
114 this.files.addAll(files);
123 IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) {
124 this.
id = IngestJob.nextId.getAndIncrement();
125 this.ingestJobPipelines =
new ConcurrentHashMap<>();
126 this.dataSources.add(dataSource);
129 incompleteJobsCount =
new AtomicInteger(1);
149 boolean hasIngestPipeline() {
150 return (!settings.getEnabledIngestModuleTemplates().isEmpty());
158 void addStreamingIngestFiles(List<Long> fileObjIds) {
159 if (ingestJobPipelines.isEmpty()) {
160 logger.log(Level.SEVERE,
"Attempted to add streaming ingest files with no IngestJobPipeline");
164 IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
165 streamingIngestPipeline.addStreamingIngestFiles(fileObjIds);
171 void processStreamingIngestDataSource() {
172 if (ingestJobPipelines.isEmpty()) {
173 logger.log(Level.SEVERE,
"Attempted to start data source ingest with no IngestJobPipeline");
177 IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
178 streamingIngestPipeline.processStreamingIngestDataSource();
187 List<IngestModuleError> start() {
192 if (files.isEmpty()) {
193 for (Content dataSource : dataSources) {
194 IngestJobPipeline ingestJobPipeline =
new IngestJobPipeline(
this, dataSource, settings);
195 this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
198 IngestJobPipeline ingestJobPipeline =
new IngestJobPipeline(
this, dataSources.get(0),
files,
settings);
199 this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
201 incompleteJobsCount.set(ingestJobPipelines.size());
211 List<IngestModuleError> errors =
new ArrayList<>();
212 for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) {
213 errors.addAll(ingestJobPipeline.start());
214 if (errors.isEmpty() ==
false) {
222 if (errors.isEmpty()) {
223 for (IngestJobPipeline dataSourceJob : this.ingestJobPipelines.values()) {
224 IngestManager.getInstance().fireDataSourceAnalysisStarted(
id, dataSourceJob.getId(), dataSourceJob.getDataSource());
227 cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED);
238 Mode getIngestMode() {
268 List<Snapshot> getDataSourceIngestJobSnapshots() {
269 List<Snapshot> snapshots =
new ArrayList<>();
270 this.ingestJobPipelines.values().stream().forEach((dataSourceJob) -> {
271 snapshots.add(dataSourceJob.getSnapshot(
true));
298 cancellationReason = reason;
309 this.ingestJobPipelines.values().stream().forEach((job) -> {
340 void ingestJobPipelineFinished(IngestJobPipeline ingestJobPipeline) {
342 if (!ingestJobPipeline.isCancelled()) {
343 ingestManager.fireDataSourceAnalysisCompleted(
id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
345 IngestManager.getInstance().fireDataSourceAnalysisCancelled(
id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
347 if (incompleteJobsCount.decrementAndGet() == 0) {
348 ingestManager.finishIngestJob(
this);
383 return snapshot.getDataSource();
393 return snapshot.isCancelled();
402 return snapshot.getCancellationReason();
413 return snapshot.getCancelledDataSourceIngestModules();
422 dataSourceModule = null;
423 fileIngestRunning =
false;
424 fileIngestStartTime = null;
425 dataSourceProcessingSnapshots =
new ArrayList<>();
426 for (IngestJobPipeline pipeline : ingestJobPipelines.values()) {
427 Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot);
429 if (null == dataSourceModule) {
430 DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule();
431 if (null != module) {
435 if (snapshot.getFileIngestIsRunning()) {
436 fileIngestRunning =
true;
438 Date childFileIngestStartTime = snapshot.getFileIngestStartTime();
439 if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
440 fileIngestStartTime = childFileIngestStartTime;
473 return new Date(this.fileIngestStartTime.getTime());
501 return Collections.unmodifiableList(this.dataSourceProcessingSnapshots);
514 private final DataSourceIngestPipeline.DataSourcePipelineModule
module;
529 this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();
539 return this.
module.getDisplayName();
549 return this.
module.getProcessingStartTime();
579 if (this.ingestJobPipeline.getCurrentDataSourceIngestModule() == this.
module) {
580 this.ingestJobPipeline.cancelCurrentDataSourceIngestModule();
List< String > getCancelledDataSourceIngestModules()
static synchronized IngestManager getInstance()
CancellationReason(String displayName)
final boolean jobCancelled
boolean fileIngestIsRunning()
final AtomicInteger incompleteJobsCount
static final Logger logger
void cancel(CancellationReason reason)
final IngestJobPipeline ingestJobPipeline
DataSourceIngestModuleHandle runningDataSourceIngestModule()
List< DataSourceProcessingSnapshot > getDataSourceSnapshots()
final IngestJob.CancellationReason jobCancellationReason
CancellationReason getCancellationReason()
final Map< Long, IngestJobPipeline > ingestJobPipelines
DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module)
ProgressSnapshot getSnapshot()
final List< Content > dataSources
INGEST_MODULES_STARTUP_FAILED
final List< AbstractFile > files
final DataSourceIngestPipeline.DataSourcePipelineModule module
static final AtomicLong nextId
DataSourceProcessingSnapshot(Snapshot snapshot)
boolean fileIngestRunning
ProgressSnapshot(boolean getIngestTasksSnapshot)
volatile CancellationReason cancellationReason
synchronized static Logger getLogger(String name)
ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot)
CancellationReason getCancellationReason()
Date fileIngestStartTime()
CancellationReason getCancellationReason()
final IngestJobSettings settings
DataSourceIngestModuleHandle dataSourceModule
final List< DataSourceProcessingSnapshot > dataSourceProcessingSnapshots