Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJob.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.List;
25 import java.util.concurrent.atomic.AtomicLong;
26 import java.util.logging.Level;
27 import org.openide.util.NbBundle;
28 import org.python.google.common.collect.ImmutableList;
30 import org.sleuthkit.datamodel.AbstractFile;
31 import org.sleuthkit.datamodel.AnalysisResult;
32 import org.sleuthkit.datamodel.DataArtifact;
33 import org.sleuthkit.datamodel.DataSource;
34 
39 public final class IngestJob {
40 
44  public enum CancellationReason {
45 
46  NOT_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.notCancelled.text")),
47  USER_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.cancelledByUser.text")),
48  INGEST_MODULES_STARTUP_FAILED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.ingestModStartFail.text")),
49  OUT_OF_DISK_SPACE(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.outOfDiskSpace.text")),
50  SERVICES_DOWN(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.servicesDown.text")),
51  CASE_CLOSED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.caseClosed.text"));
52 
53  private final String displayName;
54 
55  private CancellationReason(String displayName) {
56  this.displayName = displayName;
57  }
58 
59  public String getDisplayName() {
60  return displayName;
61  }
62  }
63 
67  enum Mode {
68  BATCH,
69  STREAMING
70  }
71 
72  private static final Logger logger = Logger.getLogger(IngestJob.class.getName());
73  private final static AtomicLong nextId = new AtomicLong(0L);
74  private final long id;
75  private final DataSource dataSource;
76  private final List<AbstractFile> files = new ArrayList<>();
77  private final Mode ingestMode;
78  private final IngestJobSettings settings;
79  private volatile IngestJobExecutor ingestModuleExecutor;
81 
92  IngestJob(DataSource dataSource, List<AbstractFile> files, IngestJobSettings settings) {
93  this(dataSource, Mode.BATCH, settings);
94  this.files.addAll(files);
95  }
96 
106  IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) {
107  id = IngestJob.nextId.getAndIncrement();
108  this.dataSource = dataSource;
109  this.settings = settings;
110  this.ingestMode = ingestMode;
111  cancellationReason = CancellationReason.NOT_CANCELLED;
112  }
113 
120  public long getId() {
121  return this.id;
122  }
123 
129  DataSource getDataSource() {
130  return dataSource;
131  }
132 
140  List<AbstractFile> getFiles() {
141  return ImmutableList.copyOf(files);
142  }
143 
149  IngestJobSettings getSettings() {
150  return settings;
151  }
152 
159  boolean hasIngestPipeline() {
160  return (!settings.getEnabledIngestModuleTemplates().isEmpty());
161  }
162 
169  void addStreamedFiles(List<Long> fileObjIds) {
170  if (ingestMode == Mode.STREAMING) {
171  if (ingestModuleExecutor != null) {
172  ingestModuleExecutor.addStreamedFiles(fileObjIds);
173  } else {
174  logger.log(Level.SEVERE, "Attempted to add streamed files with no ingest pipeline");
175  }
176  } else {
177  logger.log(Level.SEVERE, "Attempted to add streamed files to batch ingest job");
178  }
179  }
180 
187  void addDataArtifacts(List<DataArtifact> dataArtifacts) {
188  ingestModuleExecutor.addDataArtifacts(dataArtifacts);
189  }
190 
197  void addAnalysisResults(List<AnalysisResult> results) {
198  ingestModuleExecutor.addAnalysisResults(results);
199  }
200 
205  void addStreamedDataSource() {
206  if (ingestMode == Mode.STREAMING) {
207  if (ingestModuleExecutor != null) {
208  ingestModuleExecutor.addStreamedDataSource();
209  } else {
210  logger.log(Level.SEVERE, "Attempted to start data source analaysis with no ingest pipeline");
211  }
212  } else {
213  logger.log(Level.SEVERE, "Attempted to add streamed ingest files to batch ingest job");
214  }
215  }
216 
227  synchronized List<IngestModuleError> start() throws InterruptedException {
228  if (ingestModuleExecutor != null) {
229  logger.log(Level.SEVERE, "Attempt to start ingest job that has already been started");
230  return Collections.emptyList();
231  }
232 
233  ingestModuleExecutor = new IngestJobExecutor(this);
234  List<IngestModuleError> errors = new ArrayList<>();
235  errors.addAll(ingestModuleExecutor.startUp());
236  if (errors.isEmpty()) {
237  IngestManager.getInstance().fireDataSourceAnalysisStarted(id, ingestModuleExecutor.getDataSource());
238  } else {
239  cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED);
240  }
241  return errors;
242  }
243 
249  Mode getIngestMode() {
250  return ingestMode;
251  }
252 
259  return getSnapshot(true);
260  }
261 
270  public ProgressSnapshot getSnapshot(boolean includeIngestTasksSnapshot) {
271  ProgressSnapshot snapshot = null;
272  if (ingestModuleExecutor != null) {
273  return new ProgressSnapshot(includeIngestTasksSnapshot);
274  }
275  return snapshot;
276  }
277 
283  IngestJobProgressSnapshot getDiagnosticStatsSnapshot() {
284  IngestJobProgressSnapshot snapshot = null;
285  if (ingestModuleExecutor != null) {
286  snapshot = ingestModuleExecutor.getIngestJobProgressSnapshot(true);
287  }
288  return snapshot;
289  }
290 
299  @Deprecated
300  public void cancel() {
302  }
303 
312  public void cancel(CancellationReason reason) {
313  cancellationReason = reason;
314  /*
315  * Cancels the running of the ingest module pipelines. This is done in a
316  * separate thread to avoid a potential deadlock. The deadlock is
317  * possible because this method can be called in a thread that acquires
318  * the ingest manager's ingest jobs list lock and then tries to acquire
319  * the ingest pipeline stage transition lock, while an ingest thread
320  * that has acquired the stage transition lock is trying to acquire the
321  * ingest manager's ingest jobs list lock.
322  */
323  new Thread(() -> {
324  if (ingestModuleExecutor != null) {
325  ingestModuleExecutor.cancel(reason);
326  }
327  }).start();
328  }
329 
336  return cancellationReason;
337  }
338 
345  public boolean isCancelled() {
346  return (CancellationReason.NOT_CANCELLED != cancellationReason);
347  }
348 
353  void notifyIngestPipelinesShutDown() {
354  IngestManager ingestManager = IngestManager.getInstance();
355  if (!ingestModuleExecutor.isCancelled()) {
356  ingestManager.fireDataSourceAnalysisCompleted(id, dataSource);
357  } else {
358  IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, dataSource);
359  }
360  ingestManager.finishIngestJob(this);
361  }
362 
366  public final class ProgressSnapshot {
367 
369  private final boolean jobCancellationRequested;
371 
379  public final class DataSourceProcessingSnapshot {
380 
382 
391  this.snapshot = snapshot;
392  }
393 
400  public String getDataSource() {
401  return snapshot.getDataSource();
402  }
403 
410  public boolean fileIngestIsRunning() {
411  return snapshot.getFileIngestIsRunning();
412  }
413 
419  public Date fileIngestStartTime() {
420  return new Date(snapshot.getFileIngestStartTime().getTime());
421  }
422 
429  DataSourceIngestPipeline.DataSourcePipelineModule getDataSourceLevelIngestModule() {
430  return snapshot.getDataSourceLevelIngestModule();
431  }
432 
439  public boolean isCancelled() {
440  return snapshot.isCancelled();
441  }
442 
449  return snapshot.getCancellationReason();
450  }
451 
459  public List<String> getCancelledDataSourceIngestModules() {
460  return snapshot.getCancelledDataSourceIngestModules();
461  }
462 
463  }
464 
472  private ProgressSnapshot(boolean includeIngestTasksSnapshot) {
473  IngestJobProgressSnapshot snapshot = ingestModuleExecutor.getIngestJobProgressSnapshot(includeIngestTasksSnapshot);
474  dataSourceProcessingSnapshot = new DataSourceProcessingSnapshot(snapshot);
475  jobCancellationRequested = IngestJob.this.isCancelled();
476  jobCancellationReason = IngestJob.this.getCancellationReason();
477  }
478 
487  DataSourceIngestModuleHandle moduleHandle = null;
488  DataSourceIngestPipeline.DataSourcePipelineModule module = dataSourceProcessingSnapshot.getDataSourceLevelIngestModule();
489  if (module != null) {
490  moduleHandle = new DataSourceIngestModuleHandle(ingestModuleExecutor, module);
491  }
492  return moduleHandle;
493  }
494 
501  public boolean fileIngestIsRunning() {
502  return dataSourceProcessingSnapshot.fileIngestIsRunning();
503  }
504 
510  public Date fileIngestStartTime() {
511  return new Date(dataSourceProcessingSnapshot.fileIngestStartTime().getTime());
512  }
513 
520  public boolean isCancelled() {
522  }
523 
531  return jobCancellationReason;
532  }
533 
542  }
543 
544  }
545 
551  public static class DataSourceIngestModuleHandle {
552 
553  private final IngestJobExecutor ingestJobExecutor;
554  private final DataSourceIngestPipeline.DataSourcePipelineModule module;
555  private final boolean cancelled;
556 
566  private DataSourceIngestModuleHandle(IngestJobExecutor ingestJobExecutor, DataSourceIngestPipeline.DataSourcePipelineModule module) {
567  this.ingestJobExecutor = ingestJobExecutor;
568  this.module = module;
569  this.cancelled = ingestJobExecutor.currentDataSourceIngestModuleIsCancelled();
570  }
571 
578  public String displayName() {
579  return module.getDisplayName();
580  }
581 
588  public Date startTime() {
589  return module.getProcessingStartTime();
590  }
591 
598  public boolean isCancelled() {
599  return cancelled;
600  }
601 
607  public void cancel() {
608  /*
609  * Note that this cancellation mechanism has a race condition. This
610  * could perhaps be solved by adding a cancel() API to the
611  * IngestModule interface.
612  */
613  if (ingestJobExecutor.getCurrentDataSourceIngestModule() == module) {
614  ingestJobExecutor.cancelCurrentDataSourceIngestModule();
615  }
616  }
617 
618  }
619 
620 }
final DataSourceProcessingSnapshot dataSourceProcessingSnapshot
Definition: IngestJob.java:368
static synchronized IngestManager getInstance()
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
DataSourceIngestModuleHandle(IngestJobExecutor ingestJobExecutor, DataSourceIngestPipeline.DataSourcePipelineModule module)
Definition: IngestJob.java:566
void cancel(CancellationReason reason)
Definition: IngestJob.java:312
DataSourceIngestModuleHandle runningDataSourceIngestModule()
Definition: IngestJob.java:486
CancellationReason getCancellationReason()
Definition: IngestJob.java:335
final List< AbstractFile > files
Definition: IngestJob.java:76
ProgressSnapshot(boolean includeIngestTasksSnapshot)
Definition: IngestJob.java:472
final DataSourceIngestPipeline.DataSourcePipelineModule module
Definition: IngestJob.java:554
ProgressSnapshot getSnapshot(boolean includeIngestTasksSnapshot)
Definition: IngestJob.java:270
static final AtomicLong nextId
Definition: IngestJob.java:73
volatile CancellationReason cancellationReason
Definition: IngestJob.java:80
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
volatile IngestJobExecutor ingestModuleExecutor
Definition: IngestJob.java:79
DataSourceProcessingSnapshot getDataSourceProcessingSnapshot()
Definition: IngestJob.java:540
final IngestJobSettings settings
Definition: IngestJob.java:78

Copyright © 2012-2022 Basis Technology. Generated on: Tue Dec 6 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.