Autopsy  4.18.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJob.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.concurrent.ConcurrentHashMap;
29 import java.util.concurrent.atomic.AtomicInteger;
30 import java.util.concurrent.atomic.AtomicLong;
31 import java.util.logging.Level;
32 import org.openide.util.NbBundle;
34 import org.sleuthkit.datamodel.AbstractFile;
35 import org.sleuthkit.datamodel.Content;
36 import org.sleuthkit.datamodel.DataSource;
37 
42 public final class IngestJob {
43 
44  /*
45  * An ingest job can be cancelled for various reasons.
46  */
47  public enum CancellationReason {
48 
49  NOT_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.notCancelled.text")),
50  USER_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.cancelledByUser.text")),
51  INGEST_MODULES_STARTUP_FAILED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.ingestModStartFail.text")),
52  OUT_OF_DISK_SPACE(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.outOfDiskSpace.text")),
53  SERVICES_DOWN(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.servicesDown.text")),
54  CASE_CLOSED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.caseClosed.text"));
55 
56  private final String displayName;
57 
58  private CancellationReason(String displayName) {
59  this.displayName = displayName;
60  }
61 
62  public String getDisplayName() {
63  return displayName;
64  }
65  }
66 
70  enum Mode {
71  BATCH,
72  STREAMING
73  }
74 
75  private static final Logger logger = Logger.getLogger(IngestJob.class.getName());
76  private final static AtomicLong nextId = new AtomicLong(0L);
77  private final long id;
78  private final List<Content> dataSources = new ArrayList<>();
79  private final List<AbstractFile> files = new ArrayList<>();
80  private final Mode ingestMode;
81  private final Map<Long, IngestJobPipeline> ingestJobPipelines;
82  private final AtomicInteger incompleteJobsCount;
83  private final IngestJobSettings settings;
85 
93  IngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
94  this.id = IngestJob.nextId.getAndIncrement();
95  this.settings = settings;
96  this.ingestJobPipelines = new ConcurrentHashMap<>();
97  this.ingestMode = Mode.BATCH;
98  this.dataSources.addAll(dataSources);
99  incompleteJobsCount = new AtomicInteger(dataSources.size());
100  cancellationReason = CancellationReason.NOT_CANCELLED;
101  }
102 
112  IngestJob(Content dataSource, List<AbstractFile> files, IngestJobSettings settings) {
113  this(Arrays.asList(dataSource), settings);
114  this.files.addAll(files);
115  }
116 
123  IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) {
124  this.id = IngestJob.nextId.getAndIncrement();
125  this.ingestJobPipelines = new ConcurrentHashMap<>();
126  this.dataSources.add(dataSource);
127  this.settings = settings;
128  this.ingestMode = ingestMode;
129  incompleteJobsCount = new AtomicInteger(1);
130  cancellationReason = CancellationReason.NOT_CANCELLED;
131  }
132 
138  public long getId() {
139  return this.id;
140  }
141 
149  boolean hasIngestPipeline() {
150  return (!settings.getEnabledIngestModuleTemplates().isEmpty());
151  }
152 
158  void addStreamingIngestFiles(List<Long> fileObjIds) {
159  if (ingestJobPipelines.isEmpty()) {
160  logger.log(Level.SEVERE, "Attempted to add streaming ingest files with no IngestJobPipeline");
161  return;
162  }
163  // Streaming ingest jobs will only have one data source
164  IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
165  streamingIngestPipeline.addStreamingIngestFiles(fileObjIds);
166  }
167 
171  void processStreamingIngestDataSource() {
172  if (ingestJobPipelines.isEmpty()) {
173  logger.log(Level.SEVERE, "Attempted to start data source ingest with no IngestJobPipeline");
174  return;
175  }
176  // Streaming ingest jobs will only have one data source
177  IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
178  streamingIngestPipeline.processStreamingIngestDataSource();
179  }
180 
187  List<IngestModuleError> start() {
188 
189  /*
190  * Set up the pipeline(s)
191  */
192  if (files.isEmpty()) {
193  for (Content dataSource : dataSources) {
194  IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings);
195  this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
196  }
197  } else {
198  IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings);
199  this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
200  }
201  incompleteJobsCount.set(ingestJobPipelines.size());
202 
203  /*
204  * Try to start each data source ingest job. Note that there is an
205  * assumption here that if there is going to be a module startup
206  * failure, it will be for the first ingest job pipeline.
207  *
208  * TODO (RC): Consider separating module start up from pipeline startup
209  * so that no processing is done if this assumption is false.
210  */
211  List<IngestModuleError> errors = new ArrayList<>();
212  for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) {
213  errors.addAll(ingestJobPipeline.start());
214  if (errors.isEmpty() == false) {
215  break;
216  }
217  }
218 
219  /*
220  * Handle start up success or failure.
221  */
222  if (errors.isEmpty()) {
223  for (IngestJobPipeline dataSourceJob : this.ingestJobPipelines.values()) {
224  IngestManager.getInstance().fireDataSourceAnalysisStarted(id, dataSourceJob.getId(), dataSourceJob.getDataSource());
225  }
226  } else {
227  cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED);
228  }
229 
230  return errors;
231  }
232 
238  Mode getIngestMode() {
239  return ingestMode;
240  }
241 
248  return new ProgressSnapshot(true);
249  }
250 
258  public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) {
259  return new ProgressSnapshot(getIngestTasksSnapshot);
260  }
261 
268  List<Snapshot> getDataSourceIngestJobSnapshots() {
269  List<Snapshot> snapshots = new ArrayList<>();
270  this.ingestJobPipelines.values().stream().forEach((dataSourceJob) -> {
271  snapshots.add(dataSourceJob.getSnapshot(true));
272  });
273  return snapshots;
274  }
275 
284  @Deprecated
285  public void cancel() {
287  }
288 
297  public void cancel(CancellationReason reason) {
298  cancellationReason = reason;
299  /*
300  * Cancel the ingest pipelines for each data source. This is done in a
301  * separate thread to avoid a potential deadlock. The deadlock is
302  * possible because this method can be called in a thread that acquires
303  * the ingest manager's ingest jobs list lock and then tries to acquire
304  * the ingest pipeline stage transition lock, while an ingest thread
305  * that has acquired the stage transition lock is trying to acquire the
306  * ingest manager's ingest jobs list lock.
307  */
308  new Thread(() -> {
309  this.ingestJobPipelines.values().stream().forEach((job) -> {
310  job.cancel(reason);
311  });
312  }).start();
313  }
314 
321  return this.cancellationReason;
322  }
323 
330  public boolean isCancelled() {
331  return (CancellationReason.NOT_CANCELLED != this.cancellationReason);
332  }
333 
340  void ingestJobPipelineFinished(IngestJobPipeline ingestJobPipeline) {
341  IngestManager ingestManager = IngestManager.getInstance();
342  if (!ingestJobPipeline.isCancelled()) {
343  ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
344  } else {
345  IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
346  }
347  if (incompleteJobsCount.decrementAndGet() == 0) {
348  ingestManager.finishIngestJob(this);
349  }
350  }
351 
355  public final class ProgressSnapshot {
356 
357  private final List<DataSourceProcessingSnapshot> dataSourceProcessingSnapshots;
359  private boolean fileIngestRunning;
360  private Date fileIngestStartTime;
361  private final boolean jobCancelled;
363 
368  public final class DataSourceProcessingSnapshot {
369 
370  private final Snapshot snapshot;
371 
373  this.snapshot = snapshot;
374  }
375 
382  public String getDataSource() {
383  return snapshot.getDataSource();
384  }
385 
392  public boolean isCancelled() {
393  return snapshot.isCancelled();
394  }
395 
402  return snapshot.getCancellationReason();
403  }
404 
412  public List<String> getCancelledDataSourceIngestModules() {
413  return snapshot.getCancelledDataSourceIngestModules();
414  }
415 
416  }
417 
421  private ProgressSnapshot(boolean getIngestTasksSnapshot) {
422  dataSourceModule = null;
423  fileIngestRunning = false;
424  fileIngestStartTime = null;
425  dataSourceProcessingSnapshots = new ArrayList<>();
426  for (IngestJobPipeline pipeline : ingestJobPipelines.values()) {
427  Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot);
428  dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
429  if (null == dataSourceModule) {
430  DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule();
431  if (null != module) {
432  dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module);
433  }
434  }
435  if (snapshot.getFileIngestIsRunning()) {
436  fileIngestRunning = true;
437  }
438  Date childFileIngestStartTime = snapshot.getFileIngestStartTime();
439  if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
440  fileIngestStartTime = childFileIngestStartTime;
441  }
442  }
443  this.jobCancelled = isCancelled();
445  }
446 
454  return this.dataSourceModule;
455  }
456 
463  public boolean fileIngestIsRunning() {
464  return this.fileIngestRunning;
465  }
466 
472  public Date fileIngestStartTime() {
473  return new Date(this.fileIngestStartTime.getTime());
474  }
475 
482  public boolean isCancelled() {
483  return this.jobCancelled;
484  }
485 
492  return this.jobCancellationReason;
493  }
494 
500  public List<DataSourceProcessingSnapshot> getDataSourceSnapshots() {
501  return Collections.unmodifiableList(this.dataSourceProcessingSnapshots);
502  }
503 
504  }
505 
511  public static class DataSourceIngestModuleHandle {
512 
513  private final IngestJobPipeline ingestJobPipeline;
514  private final DataSourceIngestPipeline.DataSourcePipelineModule module;
515  private final boolean cancelled;
516 
526  private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) {
527  this.ingestJobPipeline = ingestJobPipeline;
528  this.module = module;
529  this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();
530  }
531 
538  public String displayName() {
539  return this.module.getDisplayName();
540  }
541 
548  public Date startTime() {
549  return this.module.getProcessingStartTime();
550  }
551 
558  public boolean isCancelled() {
559  return this.cancelled;
560  }
561 
567  public void cancel() {
579  if (this.ingestJobPipeline.getCurrentDataSourceIngestModule() == this.module) {
580  this.ingestJobPipeline.cancelCurrentDataSourceIngestModule();
581  }
582  }
583 
584  }
585 
586 }
static synchronized IngestManager getInstance()
final AtomicInteger incompleteJobsCount
Definition: IngestJob.java:82
void cancel(CancellationReason reason)
Definition: IngestJob.java:297
DataSourceIngestModuleHandle runningDataSourceIngestModule()
Definition: IngestJob.java:453
List< DataSourceProcessingSnapshot > getDataSourceSnapshots()
Definition: IngestJob.java:500
final IngestJob.CancellationReason jobCancellationReason
Definition: IngestJob.java:362
CancellationReason getCancellationReason()
Definition: IngestJob.java:320
final Map< Long, IngestJobPipeline > ingestJobPipelines
Definition: IngestJob.java:81
DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module)
Definition: IngestJob.java:526
final List< Content > dataSources
Definition: IngestJob.java:78
final List< AbstractFile > files
Definition: IngestJob.java:79
final DataSourceIngestPipeline.DataSourcePipelineModule module
Definition: IngestJob.java:514
static final AtomicLong nextId
Definition: IngestJob.java:76
ProgressSnapshot(boolean getIngestTasksSnapshot)
Definition: IngestJob.java:421
volatile CancellationReason cancellationReason
Definition: IngestJob.java:84
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot)
Definition: IngestJob.java:258
final IngestJobSettings settings
Definition: IngestJob.java:83
final List< DataSourceProcessingSnapshot > dataSourceProcessingSnapshots
Definition: IngestJob.java:357

Copyright © 2012-2021 Basis Technology. Generated on: Thu Jul 8 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.