Autopsy  4.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
DataSourceIngestJob.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.concurrent.CopyOnWriteArrayList;
28 import java.util.concurrent.LinkedBlockingQueue;
29 import java.util.concurrent.atomic.AtomicLong;
30 import java.util.logging.Level;
31 import javax.swing.JOptionPane;
32 import org.netbeans.api.progress.ProgressHandle;
33 import org.netbeans.api.progress.ProgressHandleFactory;
34 import org.openide.util.Cancellable;
35 import org.openide.util.NbBundle;
37 import org.sleuthkit.datamodel.AbstractFile;
38 import org.sleuthkit.datamodel.Content;
39 
44 final class DataSourceIngestJob {
45 
46  private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
47 
52  private final IngestJob parentJob;
53  private static final AtomicLong nextJobId = new AtomicLong(0L);
54  private final long id;
55  private final IngestJobSettings settings;
56  private final Content dataSource;
57 
61  private static enum Stages {
62 
80  FINALIZATION
81  };
82  private volatile Stages stage = DataSourceIngestJob.Stages.INITIALIZATION;
83  private final Object stageCompletionCheckLock = new Object();
84 
93  private final Object dataSourceIngestPipelineLock = new Object();
94  private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
95  private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
96  private DataSourceIngestPipeline currentDataSourceIngestPipeline;
97 
105  private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue = new LinkedBlockingQueue<>();
106  private final List<FileIngestPipeline> fileIngestPipelines = new ArrayList<>();
107 
119  private volatile boolean currentDataSourceIngestModuleCancelled;
120  private volatile boolean cancelled;
121  private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED;
122  private final Object cancellationStateMonitor = new Object();
123  private final List<String> cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>();
124 
129  private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
130 
135  private final boolean doUI;
136 
141  private final Object dataSourceIngestProgressLock = new Object();
142  private ProgressHandle dataSourceIngestProgress;
143 
148  private final Object fileIngestProgressLock = new Object();
149  private final List<String> filesInProgress = new ArrayList<>();
150  private long estimatedFilesToProcess;
151  private long processedFiles;
152  private ProgressHandle fileIngestProgress;
153  private String currentFileIngestModule = "";
154  private String currentFileIngestTask = "";
155 
159  private final long createTime;
160 
172  DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) {
173  this.parentJob = parentJob;
174  this.id = DataSourceIngestJob.nextJobId.getAndIncrement();
175  this.dataSource = dataSource;
176  this.settings = settings;
177  this.doUI = runInteractively;
178  this.createTime = new Date().getTime();
179  this.createIngestPipelines();
180  }
181 
185  private void createIngestPipelines() {
186  List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();
187 
191  Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new HashMap<>();
192  Map<String, IngestModuleTemplate> fileModuleTemplates = new HashMap<>();
193  for (IngestModuleTemplate template : ingestModuleTemplates) {
194  if (template.isDataSourceIngestModuleTemplate()) {
195  dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
196  }
197  if (template.isFileIngestModuleTemplate()) {
198  fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
199  }
200  }
201 
206  IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
207  List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
208  List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
209  List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
210 
216  for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) {
217  firstStageDataSourceModuleTemplates.add(template);
218  }
219  for (IngestModuleTemplate template : fileModuleTemplates.values()) {
220  fileIngestModuleTemplates.add(template);
221  }
222 
226  this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates);
227  this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates);
228 
232  try {
233  int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
234  for (int i = 0; i < numberOfFileIngestThreads; ++i) {
235  FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates);
236  this.fileIngestPipelinesQueue.put(pipeline);
237  this.fileIngestPipelines.add(pipeline);
238  }
239  } catch (InterruptedException ex) {
245  Thread.currentThread().interrupt();
246  }
247  }
248 
264  private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
265  List<IngestModuleTemplate> templates = new ArrayList<>();
266  for (String moduleClassName : pipelineConfig) {
267  if (ingestModuleTemplates.containsKey(moduleClassName)) {
268  templates.add(ingestModuleTemplates.remove(moduleClassName));
269  }
270  }
271  return templates;
272  }
273 
279  long getId() {
280  return this.id;
281  }
282 
288  String getExecutionContext() {
289  return this.settings.getExecutionContext();
290  }
291 
297  Content getDataSource() {
298  return this.dataSource;
299  }
300 
307  boolean shouldProcessUnallocatedSpace() {
308  return this.settings.getProcessUnallocatedSpace();
309  }
310 
316  boolean hasIngestPipeline() {
317  return this.hasFirstStageDataSourceIngestPipeline()
318  || this.hasFileIngestPipeline()
319  || this.hasSecondStageDataSourceIngestPipeline();
320  }
321 
328  private boolean hasFirstStageDataSourceIngestPipeline() {
329  return (this.firstStageDataSourceIngestPipeline.isEmpty() == false);
330  }
331 
338  private boolean hasSecondStageDataSourceIngestPipeline() {
339  return (this.secondStageDataSourceIngestPipeline.isEmpty() == false);
340  }
341 
347  private boolean hasFileIngestPipeline() {
348  if (!this.fileIngestPipelines.isEmpty()) {
349  return !this.fileIngestPipelines.get(0).isEmpty();
350  }
351  return false;
352  }
353 
359  List<IngestModuleError> start() {
360  List<IngestModuleError> errors = startUpIngestPipelines();
361  if (errors.isEmpty()) {
362  if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) {
363  logger.log(Level.INFO, "Starting first stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
364  this.startFirstStage();
365  } else if (this.hasSecondStageDataSourceIngestPipeline()) {
366  logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1}), no first stage configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
367  this.startSecondStage();
368  }
369  }
370  return errors;
371  }
372 
379  private List<IngestModuleError> startUpIngestPipelines() {
380  List<IngestModuleError> errors = new ArrayList<>();
381 
382  // Start up the first stage data source ingest pipeline.
383  errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
384 
385  // Start up the second stage data source ingest pipeline.
386  errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
387 
388  // Start up the file ingest pipelines (one per file ingest thread).
389  for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
390  errors.addAll(pipeline.startUp());
391  if (!errors.isEmpty()) {
392  // If there are start up errors, the ingest job will not proceed
393  // and the errors will ultimately be reported to the user for
394  // possible remedy so shut down the pipelines now that an
395  // attempt has been made to start up the data source ingest
396  // pipeline and at least one copy of the file ingest pipeline.
397  // pipeline. There is no need to complete starting up all of the
398  // file ingest pipeline copies since any additional start up
399  // errors are likely redundant.
400  while (!this.fileIngestPipelinesQueue.isEmpty()) {
401  pipeline = this.fileIngestPipelinesQueue.poll();
402  if (pipeline.isRunning()) {
403  List<IngestModuleError> shutDownErrors = pipeline.shutDown();
404  if (!shutDownErrors.isEmpty()) {
405  logIngestModuleErrors(shutDownErrors);
406  }
407  }
408  }
409  break;
410  }
411  }
412 
413  logIngestModuleErrors(errors);
414  return errors;
415  }
416 
420  private void startFirstStage() {
421  this.stage = DataSourceIngestJob.Stages.FIRST;
422 
423  if (this.hasFileIngestPipeline()) {
424  synchronized (this.fileIngestProgressLock) {
425  this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor());
426  }
427  }
428 
429  if (this.doUI) {
433  if (this.hasFirstStageDataSourceIngestPipeline()) {
434  this.startDataSourceIngestProgressBar();
435  }
436  if (this.hasFileIngestPipeline()) {
437  this.startFileIngestProgressBar();
438  }
439  }
440 
445  synchronized (this.dataSourceIngestPipelineLock) {
446  this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
447  }
448 
452  if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
453  logger.log(Level.INFO, "Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
454  DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this);
455  } else if (this.hasFirstStageDataSourceIngestPipeline()) {
456  logger.log(Level.INFO, "Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
457  DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
458  } else {
459  logger.log(Level.INFO, "Scheduling file level analysis tasks for {0} (jobId={1}), no first stage data source level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
460  DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this);
461 
470  this.checkForStageCompleted();
471  }
472  }
473 
477  private void startSecondStage() {
478  logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
479  this.stage = DataSourceIngestJob.Stages.SECOND;
480  if (this.doUI) {
481  this.startDataSourceIngestProgressBar();
482  }
483  synchronized (this.dataSourceIngestPipelineLock) {
484  this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
485  }
486  logger.log(Level.INFO, "Scheduling second stage data source level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
487  DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
488  }
489 
493  private void startDataSourceIngestProgressBar() {
494  if (this.doUI) {
495  synchronized (this.dataSourceIngestProgressLock) {
496  String displayName = NbBundle.getMessage(this.getClass(),
497  "IngestJob.progress.dataSourceIngest.initialDisplayName",
498  this.dataSource.getName());
499  this.dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
500  @Override
501  public boolean cancel() {
502  // If this method is called, the user has already pressed
503  // the cancel button on the progress bar and the OK button
504  // of a cancelation confirmation dialog supplied by
505  // NetBeans. What remains to be done is to find out whether
506  // the user wants to cancel only the currently executing
507  // data source ingest module or the entire ingest job.
508  DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel();
509  String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title");
510  JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
511  if (panel.cancelAllDataSourceIngestModules()) {
512  DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
513  } else {
514  DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule();
515  }
516  return true;
517  }
518  });
519  this.dataSourceIngestProgress.start();
520  this.dataSourceIngestProgress.switchToIndeterminate();
521  }
522  }
523  }
524 
528  private void startFileIngestProgressBar() {
529  if (this.doUI) {
530  synchronized (this.fileIngestProgressLock) {
531  String displayName = NbBundle.getMessage(this.getClass(),
532  "IngestJob.progress.fileIngest.displayName",
533  this.dataSource.getName());
534  this.fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
535  @Override
536  public boolean cancel() {
537  // If this method is called, the user has already pressed
538  // the cancel button on the progress bar and the OK button
539  // of a cancelation confirmation dialog supplied by
540  // NetBeans.
541  DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
542  return true;
543  }
544  });
545  this.fileIngestProgress.start();
546  this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess);
547  }
548  }
549  }
550 
555  private void checkForStageCompleted() {
556  synchronized (this.stageCompletionCheckLock) {
557  if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) {
558  switch (this.stage) {
559  case FIRST:
560  this.finishFirstStage();
561  break;
562  case SECOND:
563  this.finish();
564  break;
565  }
566  }
567  }
568  }
569 
574  private void finishFirstStage() {
575  logger.log(Level.INFO, "Finished first stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
576 
577  // Shut down the file ingest pipelines. Note that no shut down is
578  // required for the data source ingest pipeline because data source
579  // ingest modules do not have a shutdown() method.
580  List<IngestModuleError> errors = new ArrayList<>();
581  while (!this.fileIngestPipelinesQueue.isEmpty()) {
582  FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
583  if (pipeline.isRunning()) {
584  errors.addAll(pipeline.shutDown());
585  }
586  }
587  if (!errors.isEmpty()) {
588  logIngestModuleErrors(errors);
589  }
590 
591  if (this.doUI) {
592  // Finish the first stage data source ingest progress bar, if it hasn't
593  // already been finished.
594  synchronized (this.dataSourceIngestProgressLock) {
595  if (this.dataSourceIngestProgress != null) {
596  this.dataSourceIngestProgress.finish();
597  this.dataSourceIngestProgress = null;
598  }
599  }
600 
601  // Finish the file ingest progress bar, if it hasn't already
602  // been finished.
603  synchronized (this.fileIngestProgressLock) {
604  if (this.fileIngestProgress != null) {
605  this.fileIngestProgress.finish();
606  this.fileIngestProgress = null;
607  }
608  }
609  }
610 
614  if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
615  this.startSecondStage();
616  } else {
617  this.finish();
618  }
619  }
620 
624  private void finish() {
625  logger.log(Level.INFO, "Finished analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
626  this.stage = DataSourceIngestJob.Stages.FINALIZATION;
627 
628  if (this.doUI) {
629  // Finish the second stage data source ingest progress bar, if it hasn't
630  // already been finished.
631  synchronized (this.dataSourceIngestProgressLock) {
632  if (this.dataSourceIngestProgress != null) {
633  this.dataSourceIngestProgress.finish();
634  this.dataSourceIngestProgress = null;
635  }
636  }
637  }
638 
639  this.parentJob.dataSourceJobFinished(this);
640  }
641 
648  void process(DataSourceIngestTask task) {
649  try {
650  synchronized (this.dataSourceIngestPipelineLock) {
651  if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
652  List<IngestModuleError> errors = new ArrayList<>();
653  errors.addAll(this.currentDataSourceIngestPipeline.process(task));
654  if (!errors.isEmpty()) {
655  logIngestModuleErrors(errors);
656  }
657  }
658  }
659 
660  if (this.doUI) {
665  synchronized (this.dataSourceIngestProgressLock) {
666  if (null != this.dataSourceIngestProgress) {
667  this.dataSourceIngestProgress.finish();
668  this.dataSourceIngestProgress = null;
669  }
670  }
671  }
672 
673  } finally {
674  DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
675  this.checkForStageCompleted();
676  }
677  }
678 
690  void process(FileIngestTask task) throws InterruptedException {
691  try {
692  if (!this.isCancelled()) {
693  FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
694  if (!pipeline.isEmpty()) {
695  AbstractFile file = task.getFile();
696 
697  synchronized (this.fileIngestProgressLock) {
698  ++this.processedFiles;
699  if (this.doUI) {
703  if (this.processedFiles <= this.estimatedFilesToProcess) {
704  this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
705  } else {
706  this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
707  }
708  this.filesInProgress.add(file.getName());
709  }
710  }
711 
715  List<IngestModuleError> errors = new ArrayList<>();
716  errors.addAll(pipeline.process(task));
717  if (!errors.isEmpty()) {
718  logIngestModuleErrors(errors);
719  }
720 
721  if (this.doUI && !this.cancelled) {
722  synchronized (this.fileIngestProgressLock) {
727  this.filesInProgress.remove(file.getName());
728  if (this.filesInProgress.size() > 0) {
729  this.fileIngestProgress.progress(this.filesInProgress.get(0));
730  } else {
731  this.fileIngestProgress.progress("");
732  }
733  }
734  }
735  }
736  this.fileIngestPipelinesQueue.put(pipeline);
737  }
738  } finally {
739  DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
740  this.checkForStageCompleted();
741  }
742  }
743 
751  void addFiles(List<AbstractFile> files) {
752  if (DataSourceIngestJob.Stages.FIRST == this.stage) {
753  for (AbstractFile file : files) {
754  DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file);
755  }
756  } else {
757  DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS
758  }
759 
766  this.checkForStageCompleted();
767  }
768 
775  void updateDataSourceIngestProgressBarDisplayName(String displayName) {
776  if (this.doUI && !this.cancelled) {
777  synchronized (this.dataSourceIngestProgressLock) {
778  this.dataSourceIngestProgress.setDisplayName(displayName);
779  }
780  }
781  }
782 
791  void switchDataSourceIngestProgressBarToDeterminate(int workUnits) {
792  if (this.doUI && !this.cancelled) {
793  synchronized (this.dataSourceIngestProgressLock) {
794  if (null != this.dataSourceIngestProgress) {
795  this.dataSourceIngestProgress.switchToDeterminate(workUnits);
796  }
797  }
798  }
799  }
800 
806  void switchDataSourceIngestProgressBarToIndeterminate() {
807  if (this.doUI && !this.cancelled) {
808  synchronized (this.dataSourceIngestProgressLock) {
809  if (null != this.dataSourceIngestProgress) {
810  this.dataSourceIngestProgress.switchToIndeterminate();
811  }
812  }
813  }
814  }
815 
822  void advanceDataSourceIngestProgressBar(int workUnits) {
823  if (this.doUI && !this.cancelled) {
824  synchronized (this.dataSourceIngestProgressLock) {
825  if (null != this.dataSourceIngestProgress) {
826  this.dataSourceIngestProgress.progress("", workUnits);
827  }
828  }
829  }
830  }
831 
838  void advanceDataSourceIngestProgressBar(String currentTask) {
839  if (this.doUI && !this.cancelled) {
840  synchronized (this.dataSourceIngestProgressLock) {
841  if (null != this.dataSourceIngestProgress) {
842  this.dataSourceIngestProgress.progress(currentTask);
843  }
844  }
845  }
846  }
847 
856  void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) {
857  if (this.doUI && !this.cancelled) {
858  synchronized (this.fileIngestProgressLock) {
859  this.dataSourceIngestProgress.progress(currentTask, workUnits);
860  }
861  }
862  }
863 
871  boolean currentDataSourceIngestModuleIsCancelled() {
872  return this.currentDataSourceIngestModuleCancelled;
873  }
874 
881  void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
882  this.currentDataSourceIngestModuleCancelled = false;
883  this.cancelledDataSourceIngestModules.add(moduleDisplayName);
884 
885  if (this.doUI) {
893  synchronized (this.dataSourceIngestProgressLock) {
894  this.dataSourceIngestProgress.finish();
895  this.dataSourceIngestProgress = null;
896  this.startDataSourceIngestProgressBar();
897  }
898  }
899  }
900 
906  DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
907  if (null != this.currentDataSourceIngestPipeline) {
908  return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
909  } else {
910  return null;
911  }
912  }
913 
918  void cancelCurrentDataSourceIngestModule() {
919  this.currentDataSourceIngestModuleCancelled = true;
920  }
921 
928  void cancel(IngestJob.CancellationReason reason) {
929  if (this.doUI) {
934  synchronized (this.dataSourceIngestProgressLock) {
935  if (dataSourceIngestProgress != null) {
936  final String displayName = NbBundle.getMessage(this.getClass(),
937  "IngestJob.progress.dataSourceIngest.initialDisplayName",
938  dataSource.getName());
939  dataSourceIngestProgress.setDisplayName(
940  NbBundle.getMessage(this.getClass(),
941  "IngestJob.progress.cancelling",
942  displayName));
943  }
944  }
945 
950  synchronized (this.fileIngestProgressLock) {
951  if (this.fileIngestProgress != null) {
952  final String displayName = NbBundle.getMessage(this.getClass(),
953  "IngestJob.progress.fileIngest.displayName",
954  this.dataSource.getName());
955  this.fileIngestProgress.setDisplayName(
956  NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
957  displayName));
958  if (!this.currentFileIngestModule.isEmpty() && !this.currentFileIngestTask.isEmpty()) {
959  this.fileIngestProgress.progress(NbBundle.getMessage(this.getClass(),
960  "IngestJob.progress.fileIngest.cancelMessage",
961  this.currentFileIngestModule, this.currentFileIngestTask));
962  }
963 
964  }
965  }
966  }
967 
968  /*
969  * If the work is not already done, show this job as cancelled for the
970  * given reason.
971  */
972  if (Stages.FINALIZATION != stage) {
973  synchronized (cancellationStateMonitor) {
974  /*
975  * These fields are volatile for reading, synchronized on the
976  * monitor here for writing.
977  */
978  this.cancelled = true;
979  this.cancellationReason = reason;
980  }
981  }
982 
987  DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this);
988  this.checkForStageCompleted();
989  }
990 
998  void setCurrentFileIngestModule(String moduleName, String taskName) {
999  this.currentFileIngestModule = moduleName;
1000  this.currentFileIngestTask = taskName;
1001  }
1002 
1009  boolean isCancelled() {
1010  return this.cancelled;
1011  }
1012 
1018  IngestJob.CancellationReason getCancellationReason() {
1019  return this.cancellationReason;
1020  }
1021 
1027  private void logIngestModuleErrors(List<IngestModuleError> errors) {
1028  for (IngestModuleError error : errors) {
1029  DataSourceIngestJob.logger.log(Level.SEVERE, String.format("%s experienced an error analyzing %s (jobId=%d)", error.getModuleDisplayName(), dataSource.getName(), this.id), error.getModuleError()); //NON-NLS
1030  }
1031  }
1032 
1038  Snapshot getSnapshot(boolean getIngestTasksSnapshot) {
1039  return new Snapshot(getIngestTasksSnapshot);
1040  }
1041 
1045  final class Snapshot {
1046 
1047  private final String dataSource;
1048  private final long jobId;
1049  private final long jobStartTime;
1050  private final long snapShotTime;
1051  private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
1052  private boolean fileIngestRunning;
1053  private Date fileIngestStartTime;
1054  private final long processedFiles;
1055  private final long estimatedFilesToProcess;
1056  private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
1057  private final boolean jobCancelled;
1058  private final IngestJob.CancellationReason jobCancellationReason;
1059  private final List<String> cancelledDataSourceModules;
1060 
1065  Snapshot(boolean getIngestTasksSnapshot) {
1066  this.dataSource = DataSourceIngestJob.this.dataSource.getName();
1067  this.jobId = DataSourceIngestJob.this.id;
1068  this.jobStartTime = DataSourceIngestJob.this.createTime;
1069  this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
1070 
1076  for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
1077  if (pipeline.isRunning()) {
1078  this.fileIngestRunning = true;
1079  }
1080  Date pipelineStartTime = pipeline.getStartTime();
1081  if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(this.fileIngestStartTime))) {
1082  this.fileIngestStartTime = pipelineStartTime;
1083  }
1084  }
1085 
1086  this.jobCancelled = cancelled;
1087  this.jobCancellationReason = cancellationReason;
1088  this.cancelledDataSourceModules = new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
1089 
1090  if (getIngestTasksSnapshot) {
1091  synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
1092  this.processedFiles = DataSourceIngestJob.this.processedFiles;
1093  this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
1094  this.snapShotTime = new Date().getTime();
1095  }
1096  this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
1097 
1098  } else {
1099  this.processedFiles = 0;
1100  this.estimatedFilesToProcess = 0;
1101  this.snapShotTime = new Date().getTime();
1102  this.tasksSnapshot = null;
1103  }
1104  }
1105 
1112  long getSnapshotTime() {
1113  return snapShotTime;
1114  }
1115 
1122  String getDataSource() {
1123  return dataSource;
1124  }
1125 
1132  long getJobId() {
1133  return this.jobId;
1134  }
1135 
1142  long getJobStartTime() {
1143  return jobStartTime;
1144  }
1145 
1146  DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1147  return this.dataSourceLevelIngestModule;
1148  }
1149 
1150  boolean fileIngestIsRunning() {
1151  return this.fileIngestRunning;
1152  }
1153 
1154  Date fileIngestStartTime() {
1155  return this.fileIngestStartTime;
1156  }
1157 
1164  double getSpeed() {
1165  return (double) processedFiles / ((snapShotTime - jobStartTime) / 1000);
1166  }
1167 
1173  long getFilesProcessed() {
1174  return processedFiles;
1175  }
1176 
1183  long getFilesEstimated() {
1184  return estimatedFilesToProcess;
1185  }
1186 
1187  long getRootQueueSize() {
1188  if (null == this.tasksSnapshot) {
1189  return 0;
1190  }
1191  return this.tasksSnapshot.getRootQueueSize();
1192  }
1193 
1194  long getDirQueueSize() {
1195  if (null == this.tasksSnapshot) {
1196  return 0;
1197  }
1198  return this.tasksSnapshot.getDirectoryTasksQueueSize();
1199  }
1200 
1201  long getFileQueueSize() {
1202  if (null == this.tasksSnapshot) {
1203  return 0;
1204  }
1205  return this.tasksSnapshot.getFileQueueSize();
1206  }
1207 
1208  long getDsQueueSize() {
1209  if (null == this.tasksSnapshot) {
1210  return 0;
1211  }
1212  return this.tasksSnapshot.getDsQueueSize();
1213  }
1214 
1215  long getRunningListSize() {
1216  if (null == this.tasksSnapshot) {
1217  return 0;
1218  }
1219  return this.tasksSnapshot.getRunningListSize();
1220  }
1221 
1222  boolean isCancelled() {
1223  return this.jobCancelled;
1224  }
1225 
1231  IngestJob.CancellationReason getCancellationReason() {
1232  return this.jobCancellationReason;
1233  }
1234 
1242  List<String> getCancelledDataSourceIngestModules() {
1243  return Collections.unmodifiableList(this.cancelledDataSourceModules);
1244  }
1245 
1246  }
1247 
1248 }

Copyright © 2012-2015 Basis Technology. Generated on: Wed Apr 6 2016
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.