19 package org.sleuthkit.autopsy.ingest;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashSet;
25 import java.util.LinkedHashMap;
26 import java.util.List;
29 import java.util.concurrent.CopyOnWriteArrayList;
30 import java.util.concurrent.LinkedBlockingQueue;
31 import java.util.concurrent.atomic.AtomicLong;
32 import java.util.logging.Level;
33 import java.util.regex.Matcher;
34 import java.util.regex.Pattern;
35 import java.util.stream.Stream;
36 import javax.annotation.concurrent.GuardedBy;
37 import javax.swing.JOptionPane;
38 import org.netbeans.api.progress.ProgressHandle;
39 import org.openide.util.Cancellable;
40 import org.openide.util.NbBundle;
41 import org.openide.windows.WindowManager;
51 import org.
sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
53 import org.
sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
63 final class IngestJobPipeline {
65 private static String AUTOPSY_MODULE_PREFIX =
"org.sleuthkit.autopsy";
67 private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName());
70 private static final Pattern JYTHON_REGEX = Pattern.compile(
"org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$");
78 private final IngestJob parentJob;
79 private static final AtomicLong nextJobId =
new AtomicLong(0L);
80 private final long id;
81 private final IngestJobSettings settings;
82 private Content dataSource = null;
83 private final IngestJob.Mode ingestMode;
84 private final List<AbstractFile> files =
new ArrayList<>();
115 private final Object stageCompletionCheckLock =
new Object();
125 private final Object dataSourceIngestPipelineLock =
new Object();
126 private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
127 private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
128 private DataSourceIngestPipeline currentDataSourceIngestPipeline;
137 private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue =
new LinkedBlockingQueue<>();
138 private final List<FileIngestPipeline> fileIngestPipelines =
new ArrayList<>();
151 private volatile boolean currentDataSourceIngestModuleCancelled;
152 private final List<String> cancelledDataSourceIngestModules =
new CopyOnWriteArrayList<>();
153 private volatile boolean cancelled;
160 private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
166 private final boolean doUI;
172 private final Object dataSourceIngestProgressLock =
new Object();
173 private ProgressHandle dataSourceIngestProgress;
179 private final Object fileIngestProgressLock =
new Object();
180 private final List<String> filesInProgress =
new ArrayList<>();
181 private long estimatedFilesToProcess;
182 private long processedFiles;
183 private ProgressHandle fileIngestProgress;
184 private String currentFileIngestModule =
"";
185 private String currentFileIngestTask =
"";
186 private final List<IngestModuleInfo> ingestModules =
new ArrayList<>();
187 private volatile IngestJobInfo ingestJob;
192 private final long createTime;
200 private final Object threadRegistrationLock =
new Object();
201 @GuardedBy(
"threadRegistrationLock")
202 private final Set<Thread> pausedIngestThreads = new HashSet<>();
214 this(parentJob, dataSource, Collections.emptyList(), settings);
229 this.parentJob = parentJob;
230 this.
id = IngestJobPipeline.nextJobId.getAndIncrement();
231 this.dataSource = dataSource;
232 this.files.addAll(files);
233 this.ingestMode = parentJob.getIngestMode();
234 this.settings = settings;
236 this.createTime =
new Date().getTime();
238 this.createIngestPipelines();
251 private static void addOrdered(
final List<IngestModuleTemplate> dest,
252 final Map<String, IngestModuleTemplate> src,
final Map<String, IngestModuleTemplate> jythonSrc) {
254 final List<IngestModuleTemplate> autopsyModules =
new ArrayList<>();
255 final List<IngestModuleTemplate> thirdPartyModules =
new ArrayList<>();
257 Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> {
258 if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) {
259 autopsyModules.add(templateEntry.getValue());
261 thirdPartyModules.add(templateEntry.getValue());
265 dest.addAll(autopsyModules);
266 dest.addAll(thirdPartyModules);
279 private static String getJythonName(String canonicalName) {
280 Matcher m = JYTHON_REGEX.matcher(canonicalName);
282 return String.format(
"%s.%s", m.group(1), m.group(2));
297 private static void addModule(Map<String, IngestModuleTemplate> mapping,
298 Map<String, IngestModuleTemplate> jythonMapping, IngestModuleTemplate
template) {
300 String className =
template.getModuleFactory().getClass().getCanonicalName();
301 String jythonName = getJythonName(className);
302 if (jythonName != null) {
303 jythonMapping.put(jythonName,
template);
305 mapping.put(className,
template);
312 private void createIngestPipelines() {
318 Map<String, IngestModuleTemplate> dataSourceModuleTemplates =
new LinkedHashMap<>();
319 Map<String, IngestModuleTemplate> fileModuleTemplates =
new LinkedHashMap<>();
322 Map<String, IngestModuleTemplate> jythonDataSourceModuleTemplates =
new LinkedHashMap<>();
323 Map<String, IngestModuleTemplate> jythonFileModuleTemplates =
new LinkedHashMap<>();
325 for (IngestModuleTemplate
template : ingestModuleTemplates) {
326 if (
template.isDataSourceIngestModuleTemplate()) {
327 addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates,
template);
329 if (
template.isFileIngestModuleTemplate()) {
330 addModule(fileModuleTemplates, jythonFileModuleTemplates,
template);
338 IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
339 List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(
340 dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
342 List<IngestModuleTemplate> fileIngestModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(
343 fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
345 List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(
346 dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
353 addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates);
354 addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates);
359 this.firstStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, firstStageDataSourceModuleTemplates);
360 this.secondStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, secondStageDataSourceModuleTemplates);
366 int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
367 for (
int i = 0; i < numberOfFileIngestThreads; ++i) {
368 FileIngestPipeline pipeline =
new FileIngestPipeline(
this, fileIngestModuleTemplates);
369 this.fileIngestPipelinesQueue.put(pipeline);
370 this.fileIngestPipelines.add(pipeline);
372 }
catch (InterruptedException ex) {
378 Thread.currentThread().interrupt();
381 SleuthkitCase skCase = Case.getCurrentCaseThrows().getSleuthkitCase();
382 this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
383 this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
384 this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
385 }
catch (TskCoreException | NoCurrentCaseException ex) {
386 logErrorMessage(Level.WARNING,
"Failed to add ingest modules listing to case database", ex);
390 private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase)
throws TskCoreException {
391 for (IngestModuleTemplate module : templates) {
392 ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
416 private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(
417 Map<String, IngestModuleTemplate> ingestModuleTemplates, Map<String, IngestModuleTemplate> jythonIngestModuleTemplates, List<String> pipelineConfig) {
418 List<IngestModuleTemplate> templates =
new ArrayList<>();
419 for (String moduleClassName : pipelineConfig) {
420 if (ingestModuleTemplates != null && ingestModuleTemplates.containsKey(moduleClassName)) {
421 templates.add(ingestModuleTemplates.remove(moduleClassName));
422 }
else if (jythonIngestModuleTemplates != null && jythonIngestModuleTemplates.containsKey(moduleClassName)) {
423 templates.add(jythonIngestModuleTemplates.remove(moduleClassName));
443 String getExecutionContext() {
452 Content getDataSource() {
453 return this.dataSource;
462 boolean shouldProcessUnallocatedSpace() {
471 FilesSet getFileIngestFilter() {
480 boolean hasIngestPipeline() {
481 return this.hasFirstStageDataSourceIngestPipeline()
482 || this.hasFileIngestPipeline()
483 || this.hasSecondStageDataSourceIngestPipeline();
492 private boolean hasFirstStageDataSourceIngestPipeline() {
493 return (this.firstStageDataSourceIngestPipeline.isEmpty() ==
false);
502 private boolean hasSecondStageDataSourceIngestPipeline() {
503 return (this.secondStageDataSourceIngestPipeline.isEmpty() ==
false);
511 private boolean hasFileIngestPipeline() {
512 if (!this.fileIngestPipelines.isEmpty()) {
513 return !this.fileIngestPipelines.get(0).isEmpty();
523 List<IngestModuleError> start() {
524 if (dataSource == null) {
526 throw new IllegalStateException(
"Ingest started before setting data source");
528 List<IngestModuleError> errors = startUpIngestPipelines();
529 if (errors.isEmpty()) {
531 this.ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules,
new Date(this.createTime),
new Date(0), IngestJobStatusType.STARTED,
"");
532 }
catch (TskCoreException | NoCurrentCaseException ex) {
533 logErrorMessage(Level.WARNING,
"Failed to add ingest job info to case database", ex);
536 if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) {
537 if (ingestMode == IngestJob.Mode.BATCH) {
538 logInfoMessage(
"Starting first stage analysis");
539 this.startFirstStage();
541 logInfoMessage(
"Preparing for first stage analysis");
542 this.startFileIngestStreaming();
544 }
else if (this.hasSecondStageDataSourceIngestPipeline()) {
545 logInfoMessage(
"Starting second stage analysis");
546 this.startSecondStage();
558 private List<IngestModuleError> startUpIngestPipelines() {
559 List<IngestModuleError> errors =
new ArrayList<>();
564 errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
565 errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
572 if (errors.isEmpty()) {
573 for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
574 errors.addAll(pipeline.startUp());
575 if (!errors.isEmpty()) {
581 while (!this.fileIngestPipelinesQueue.isEmpty()) {
582 FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
583 if (startedPipeline.isRunning()) {
584 List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
585 if (!shutDownErrors.isEmpty()) {
591 logIngestModuleErrors(shutDownErrors);
605 private void startFirstStage() {
608 if (this.hasFileIngestPipeline()) {
609 synchronized (this.fileIngestProgressLock) {
610 this.estimatedFilesToProcess = this.dataSource.accept(
new GetFilesCountVisitor());
618 if (this.hasFirstStageDataSourceIngestPipeline()) {
619 this.startDataSourceIngestProgressBar();
621 if (this.hasFileIngestPipeline()) {
622 this.startFileIngestProgressBar();
630 synchronized (this.dataSourceIngestPipelineLock) {
631 this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
637 if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
638 logInfoMessage(
"Scheduling first stage data source and file level analysis tasks");
639 IngestJobPipeline.taskScheduler.scheduleIngestTasks(
this);
640 }
else if (this.hasFirstStageDataSourceIngestPipeline()) {
641 logInfoMessage(
"Scheduling first stage data source level analysis tasks");
642 IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(
this);
644 logInfoMessage(
"Scheduling file level analysis tasks, no first stage data source level analysis configured");
645 IngestJobPipeline.taskScheduler.scheduleFileIngestTasks(
this, this.files);
655 this.checkForStageCompleted();
663 private void startFileIngestStreaming() {
664 synchronized (this.stageCompletionCheckLock) {
668 if (this.hasFileIngestPipeline()) {
669 synchronized (this.fileIngestProgressLock) {
670 this.estimatedFilesToProcess = 0;
675 if (this.hasFileIngestPipeline()) {
676 this.startFileIngestProgressBar();
680 logInfoMessage(
"Waiting for streaming files");
687 private void startDataSourceIngestStreaming() {
691 synchronized (fileIngestProgressLock) {
692 if (null != this.fileIngestProgress) {
693 estimatedFilesToProcess = dataSource.accept(
new GetFilesCountVisitor());
694 fileIngestProgress.switchToDeterminate((
int) estimatedFilesToProcess);
702 if (this.hasFirstStageDataSourceIngestPipeline()) {
703 this.startDataSourceIngestProgressBar();
711 synchronized (this.dataSourceIngestPipelineLock) {
712 this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
715 logInfoMessage(
"Scheduling first stage data source level analysis tasks");
716 synchronized (this.stageCompletionCheckLock) {
718 IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(
this);
725 private void startSecondStage() {
726 logInfoMessage(
"Starting second stage analysis");
729 this.startDataSourceIngestProgressBar();
731 synchronized (this.dataSourceIngestPipelineLock) {
732 this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
734 logInfoMessage(
"Scheduling second stage data source level analysis tasks");
735 IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(
this);
741 private void startDataSourceIngestProgressBar() {
743 synchronized (this.dataSourceIngestProgressLock) {
744 String displayName = NbBundle.getMessage(this.getClass(),
745 "IngestJob.progress.dataSourceIngest.initialDisplayName",
746 this.dataSource.getName());
747 this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
749 public boolean cancel() {
756 DataSourceIngestCancellationPanel panel =
new DataSourceIngestCancellationPanel();
757 String dialogTitle = NbBundle.getMessage(IngestJobPipeline.this.getClass(),
"IngestJob.cancellationDialog.title");
758 JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
759 if (panel.cancelAllDataSourceIngestModules()) {
760 IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
762 IngestJobPipeline.this.cancelCurrentDataSourceIngestModule();
767 this.dataSourceIngestProgress.start();
768 this.dataSourceIngestProgress.switchToIndeterminate();
776 private void startFileIngestProgressBar() {
778 synchronized (this.fileIngestProgressLock) {
779 String displayName = NbBundle.getMessage(this.getClass(),
780 "IngestJob.progress.fileIngest.displayName",
781 this.dataSource.getName());
782 this.fileIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
784 public boolean cancel() {
789 IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
793 this.fileIngestProgress.start();
794 this.fileIngestProgress.switchToDeterminate((
int) this.estimatedFilesToProcess);
803 private void checkForStageCompleted() {
804 if (ingestMode == IngestJob.Mode.BATCH) {
805 checkForStageCompletedBatch();
807 checkForStageCompletedStreaming();
815 private void checkForStageCompletedBatch() {
816 synchronized (this.stageCompletionCheckLock) {
817 if (IngestJobPipeline.taskScheduler.currentTasksAreCompleted(
this)) {
818 switch (this.stage) {
819 case FIRST_STAGE_FILES_AND_DATASOURCE:
820 this.finishFirstStage();
834 private void checkForStageCompletedStreaming() {
835 synchronized (this.stageCompletionCheckLock) {
836 if (IngestJobPipeline.taskScheduler.currentTasksAreCompleted(
this)) {
837 switch (this.stage) {
838 case FIRST_STAGE_FILES_ONLY:
841 case FIRST_STAGE_FILES_AND_DATASOURCE:
843 this.finishFirstStage();
857 private void finishFirstStage() {
858 logInfoMessage(
"Finished first stage analysis");
863 List<IngestModuleError> errors =
new ArrayList<>();
864 while (!this.fileIngestPipelinesQueue.isEmpty()) {
865 FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
866 if (pipeline.isRunning()) {
867 errors.addAll(pipeline.shutDown());
870 if (!errors.isEmpty()) {
871 logIngestModuleErrors(errors);
877 synchronized (this.dataSourceIngestProgressLock) {
878 if (this.dataSourceIngestProgress != null) {
879 this.dataSourceIngestProgress.finish();
880 this.dataSourceIngestProgress = null;
886 synchronized (this.fileIngestProgressLock) {
887 if (this.fileIngestProgress != null) {
888 this.fileIngestProgress.finish();
889 this.fileIngestProgress = null;
897 if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
898 this.startSecondStage();
907 private void finish() {
908 logInfoMessage(
"Finished analysis");
914 synchronized (this.dataSourceIngestProgressLock) {
915 if (this.dataSourceIngestProgress != null) {
916 this.dataSourceIngestProgress.finish();
917 this.dataSourceIngestProgress = null;
921 if (ingestJob != null) {
922 if (this.cancelled) {
924 ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
925 }
catch (TskCoreException ex) {
926 logErrorMessage(Level.WARNING,
"Failed to update ingest job status in case database", ex);
930 ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
931 }
catch (TskCoreException ex) {
932 logErrorMessage(Level.WARNING,
"Failed to update ingest job status in case database", ex);
936 this.ingestJob.setEndDateTime(
new Date());
937 }
catch (TskCoreException ex) {
938 logErrorMessage(Level.WARNING,
"Failed to set job end date in case database", ex);
941 this.parentJob.ingestJobPipelineFinished(
this);
950 void process(DataSourceIngestTask task) {
952 synchronized (this.dataSourceIngestPipelineLock) {
953 if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
954 List<IngestModuleError> errors =
new ArrayList<>();
955 errors.addAll(this.currentDataSourceIngestPipeline.performTask(task));
956 if (!errors.isEmpty()) {
957 logIngestModuleErrors(errors);
967 synchronized (this.dataSourceIngestProgressLock) {
968 if (null != this.dataSourceIngestProgress) {
969 this.dataSourceIngestProgress.finish();
970 this.dataSourceIngestProgress = null;
976 IngestJobPipeline.taskScheduler.notifyTaskCompleted(task);
977 this.checkForStageCompleted();
992 void process(FileIngestTask task)
throws InterruptedException {
994 if (!this.isCancelled()) {
995 FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
996 if (!pipeline.isEmpty()) {
999 file = task.getFile();
1000 }
catch (TskCoreException ex) {
1003 List<IngestModuleError> errors =
new ArrayList<>();
1004 errors.add(
new IngestModuleError(
"Ingest Job Pipeline", ex));
1005 logIngestModuleErrors(errors);
1006 this.fileIngestPipelinesQueue.put(pipeline);
1010 synchronized (this.fileIngestProgressLock) {
1011 ++this.processedFiles;
1016 if (this.processedFiles <= this.estimatedFilesToProcess) {
1017 this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
1019 this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
1021 this.filesInProgress.add(file.getName());
1028 List<IngestModuleError> errors =
new ArrayList<>();
1029 errors.addAll(pipeline.performTask(task));
1030 if (!errors.isEmpty()) {
1031 logIngestModuleErrors(errors, file);
1034 if (this.doUI && !this.cancelled) {
1035 synchronized (this.fileIngestProgressLock) {
1040 this.filesInProgress.remove(file.getName());
1041 if (this.filesInProgress.size() > 0) {
1042 this.fileIngestProgress.progress(this.filesInProgress.get(0));
1044 this.fileIngestProgress.progress(
"");
1049 this.fileIngestPipelinesQueue.put(pipeline);
1052 IngestJobPipeline.taskScheduler.notifyTaskCompleted(task);
1053 this.checkForStageCompleted();
1063 void addStreamingIngestFiles(List<Long> fileObjIds) {
1066 if (!hasFileIngestPipeline()) {
1070 if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY)) {
1071 IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(
this, fileObjIds);
1073 logErrorMessage(Level.SEVERE,
"Adding streaming files to job during stage " + stage.toString() +
" not supported");
1081 void processStreamingIngestDataSource() {
1082 startDataSourceIngestStreaming();
1083 checkForStageCompleted();
1093 void addFiles(List<AbstractFile> files) {
1094 if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY)
1095 || stage.equals(Stages.FIRST_STAGE_FILES_AND_DATASOURCE)) {
1096 IngestJobPipeline.taskScheduler.fastTrackFileIngestTasks(
this, files);
1098 logErrorMessage(Level.SEVERE,
"Adding files to job during second stage analysis not supported");
1107 this.checkForStageCompleted();
1116 void updateDataSourceIngestProgressBarDisplayName(String displayName) {
1117 if (this.doUI && !this.cancelled) {
1118 synchronized (this.dataSourceIngestProgressLock) {
1119 this.dataSourceIngestProgress.setDisplayName(displayName);
1132 void switchDataSourceIngestProgressBarToDeterminate(
int workUnits) {
1133 if (this.doUI && !this.cancelled) {
1134 synchronized (this.dataSourceIngestProgressLock) {
1135 if (null != this.dataSourceIngestProgress) {
1136 this.dataSourceIngestProgress.switchToDeterminate(workUnits);
1147 void switchDataSourceIngestProgressBarToIndeterminate() {
1148 if (this.doUI && !this.cancelled) {
1149 synchronized (this.dataSourceIngestProgressLock) {
1150 if (null != this.dataSourceIngestProgress) {
1151 this.dataSourceIngestProgress.switchToIndeterminate();
1163 void advanceDataSourceIngestProgressBar(
int workUnits) {
1164 if (this.doUI && !this.cancelled) {
1165 synchronized (this.dataSourceIngestProgressLock) {
1166 if (null != this.dataSourceIngestProgress) {
1167 this.dataSourceIngestProgress.progress(
"", workUnits);
1179 void advanceDataSourceIngestProgressBar(String currentTask) {
1180 if (this.doUI && !this.cancelled) {
1181 synchronized (this.dataSourceIngestProgressLock) {
1182 if (null != this.dataSourceIngestProgress) {
1183 this.dataSourceIngestProgress.progress(currentTask);
1197 void advanceDataSourceIngestProgressBar(String currentTask,
int workUnits) {
1198 if (this.doUI && !this.cancelled) {
1199 synchronized (this.fileIngestProgressLock) {
1200 this.dataSourceIngestProgress.progress(currentTask, workUnits);
1212 boolean currentDataSourceIngestModuleIsCancelled() {
1213 return this.currentDataSourceIngestModuleCancelled;
1222 void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
1223 this.currentDataSourceIngestModuleCancelled =
false;
1224 this.cancelledDataSourceIngestModules.add(moduleDisplayName);
1234 synchronized (this.dataSourceIngestProgressLock) {
1235 this.dataSourceIngestProgress.finish();
1236 this.dataSourceIngestProgress = null;
1237 this.startDataSourceIngestProgressBar();
1247 DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() {
1248 if (null != currentDataSourceIngestPipeline) {
1249 return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule();
1259 void cancelCurrentDataSourceIngestModule() {
1260 this.currentDataSourceIngestModuleCancelled =
true;
1269 void cancel(IngestJob.CancellationReason reason) {
1270 this.cancelled =
true;
1271 this.cancellationReason = reason;
1272 IngestJobPipeline.taskScheduler.cancelPendingTasksForIngestJob(
this);
1275 synchronized (this.dataSourceIngestProgressLock) {
1276 if (null != dataSourceIngestProgress) {
1277 dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName()));
1278 dataSourceIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling"));
1282 synchronized (this.fileIngestProgressLock) {
1283 if (null != this.fileIngestProgress) {
1284 this.fileIngestProgress.setDisplayName(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.fileIngest.displayName", this.dataSource.getName()));
1285 this.fileIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling"));
1290 synchronized (threadRegistrationLock) {
1291 for (Thread thread : pausedIngestThreads) {
1294 pausedIngestThreads.clear();
1298 checkForStageCompleted();
1308 void setCurrentFileIngestModule(String moduleName, String taskName) {
1309 this.currentFileIngestModule = moduleName;
1310 this.currentFileIngestTask = taskName;
1319 boolean isCancelled() {
1320 return this.cancelled;
1328 IngestJob.CancellationReason getCancellationReason() {
1329 return this.cancellationReason;
1338 private void logInfoMessage(String message) {
1339 logger.log(Level.INFO, String.format(
"%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message,
this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId()));
1350 private void logErrorMessage(Level level, String message, Throwable throwable) {
1351 logger.log(level, String.format(
"%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message,
this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId()), throwable);
1361 private void logErrorMessage(Level level, String message) {
1362 logger.log(level, String.format(
"%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message,
this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId()));
1370 private void logIngestModuleErrors(List<IngestModuleError> errors) {
1371 for (IngestModuleError error : errors) {
1372 logErrorMessage(Level.SEVERE, String.format(
"%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable());
1382 private void logIngestModuleErrors(List<IngestModuleError> errors, AbstractFile file) {
1383 for (IngestModuleError error : errors) {
1384 logErrorMessage(Level.SEVERE, String.format(
"%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable());
1393 Snapshot getSnapshot(
boolean getIngestTasksSnapshot) {
1399 boolean fileIngestRunning =
false;
1400 Date fileIngestStartTime = null;
1402 for (FileIngestPipeline pipeline : this.fileIngestPipelines) {
1403 if (pipeline.isRunning()) {
1404 fileIngestRunning =
true;
1406 Date pipelineStartTime = pipeline.getStartTime();
1407 if (null != pipelineStartTime && (null == fileIngestStartTime || pipelineStartTime.before(fileIngestStartTime))) {
1408 fileIngestStartTime = pipelineStartTime;
1412 long processedFilesCount = 0;
1413 long estimatedFilesToProcessCount = 0;
1414 long snapShotTime =
new Date().getTime();
1415 IngestJobTasksSnapshot tasksSnapshot = null;
1417 if (getIngestTasksSnapshot) {
1418 synchronized (fileIngestProgressLock) {
1419 processedFilesCount = this.processedFiles;
1420 estimatedFilesToProcessCount = this.estimatedFilesToProcess;
1421 snapShotTime =
new Date().getTime();
1423 tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(
id);
1427 return new Snapshot(this.dataSource.getName(), id, createTime,
1428 getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime,
1429 cancelled, cancellationReason, cancelledDataSourceIngestModules,
1430 processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot);
1439 void registerPausedIngestThread(Thread thread) {
1440 synchronized (threadRegistrationLock) {
1441 pausedIngestThreads.add(thread);
1451 void unregisterPausedIngestThread(Thread thread) {
1452 synchronized (threadRegistrationLock) {
1453 pausedIngestThreads.remove(thread);
static boolean runningWithGUI
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
String getExecutionContext()
FIRST_STAGE_FILES_AND_DATASOURCE
boolean getProcessUnallocatedSpace()