Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestTasksScheduler.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2012-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.io.Serializable;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Comparator;
25 import java.util.Deque;
26 import java.util.Iterator;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Queue;
30 import java.util.TreeSet;
31 import java.util.concurrent.BlockingDeque;
32 import java.util.concurrent.LinkedBlockingDeque;
33 import java.util.logging.Level;
34 import java.util.regex.Matcher;
35 import java.util.regex.Pattern;
36 import javax.annotation.concurrent.GuardedBy;
37 import javax.annotation.concurrent.ThreadSafe;
40 import org.sleuthkit.datamodel.AbstractFile;
41 import org.sleuthkit.datamodel.AnalysisResult;
42 import org.sleuthkit.datamodel.Blackboard;
43 import org.sleuthkit.datamodel.Content;
44 import org.sleuthkit.datamodel.DataArtifact;
45 import org.sleuthkit.datamodel.DataSource;
46 import org.sleuthkit.datamodel.FileSystem;
47 import org.sleuthkit.datamodel.TskCoreException;
48 import org.sleuthkit.datamodel.TskData;
49 
54 @ThreadSafe
55 final class IngestTasksScheduler {
56 
57  private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
58  private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName());
59  @GuardedBy("IngestTasksScheduler.this")
60  private static IngestTasksScheduler instance;
61  private final IngestTaskTrackingQueue dataSourceIngestTasksQueue;
62  @GuardedBy("this")
63  private final TreeSet<FileIngestTask> topLevelFileIngestTasksQueue;
64  @GuardedBy("this")
65  private final Deque<FileIngestTask> batchedFileIngestTasksQueue;
66  @GuardedBy("this")
67  private final Queue<FileIngestTask> streamedFileIngestTasksQueue;
68  private final IngestTaskTrackingQueue fileIngestTasksQueue;
69  private final IngestTaskTrackingQueue artifactIngestTasksQueue;
70  private final IngestTaskTrackingQueue resultIngestTasksQueue;
71 
77  synchronized static IngestTasksScheduler getInstance() {
78  if (IngestTasksScheduler.instance == null) {
79  IngestTasksScheduler.instance = new IngestTasksScheduler();
80  }
81  return IngestTasksScheduler.instance;
82  }
83 
89  private IngestTasksScheduler() {
90  dataSourceIngestTasksQueue = new IngestTaskTrackingQueue();
91  topLevelFileIngestTasksQueue = new TreeSet<>(new RootDirectoryTaskComparator());
92  batchedFileIngestTasksQueue = new LinkedList<>();
93  fileIngestTasksQueue = new IngestTaskTrackingQueue();
94  streamedFileIngestTasksQueue = new LinkedList<>();
95  artifactIngestTasksQueue = new IngestTaskTrackingQueue();
96  resultIngestTasksQueue = new IngestTaskTrackingQueue();
97  }
98 
105  BlockingIngestTaskQueue getDataSourceIngestTaskQueue() {
106  return dataSourceIngestTasksQueue;
107  }
108 
115  BlockingIngestTaskQueue getFileIngestTaskQueue() {
116  return fileIngestTasksQueue;
117  }
118 
125  BlockingIngestTaskQueue getDataArtifactIngestTaskQueue() {
126  return artifactIngestTasksQueue;
127  }
128 
135  BlockingIngestTaskQueue getAnalysisResultIngestTaskQueue() {
136  return resultIngestTasksQueue;
137  }
138 
149  synchronized void scheduleDataSourceIngestTask(IngestJobExecutor executor) {
150  if (!executor.isCancelled()) {
151  DataSourceIngestTask task = new DataSourceIngestTask(executor);
152  try {
153  dataSourceIngestTasksQueue.putLast(task);
154  } catch (InterruptedException ex) {
155  IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (ingest job ID={%d)", executor.getIngestJobId()), ex);
156  Thread.currentThread().interrupt();
157  }
158  }
159  }
160 
175  synchronized void scheduleFileIngestTasks(IngestJobExecutor executor, Collection<AbstractFile> files) {
176  if (!executor.isCancelled()) {
177  Collection<AbstractFile> candidateFiles;
178  if (files.isEmpty()) {
179  candidateFiles = getTopLevelFiles(executor.getDataSource());
180  } else {
181  candidateFiles = files;
182  }
183  for (AbstractFile file : candidateFiles) {
184  FileIngestTask task = new FileIngestTask(executor, file);
185  if (IngestTasksScheduler.shouldEnqueueFileTask(task)) {
186  topLevelFileIngestTasksQueue.add(task);
187  }
188  }
189  refillFileIngestTasksQueue();
190  }
191  }
192 
204  synchronized void scheduleStreamedFileIngestTasks(IngestJobExecutor executor, List<Long> fileIds) {
205  if (!executor.isCancelled()) {
206  for (long id : fileIds) {
207  /*
208  * Create the file ingest task. Note that we do not do the
209  * shouldEnqueueFileTask() check here in order to delay querying
210  * the case database to construct the AbstractFile object. The
211  * file filter will be applied before the file task makes it to
212  * the task queue consumed by the file ingest threads.
213  */
214  FileIngestTask task = new FileIngestTask(executor, id);
215  streamedFileIngestTasksQueue.add(task);
216  }
217  refillFileIngestTasksQueue();
218  }
219  }
220 
235  synchronized void scheduleHighPriorityFileIngestTasks(IngestJobExecutor executor, Collection<AbstractFile> files) {
236  if (!executor.isCancelled()) {
237  /*
238  * Put the files directly into the queue for the file ingest
239  * threads, if they pass the file filter for the job. The files are
240  * added to the queue for the ingest threads BEFORE the other queued
241  * tasks because the use case for this method is scheduling new
242  * carved or derived files from a high priority task that is already
243  * in progress.
244  */
245  for (AbstractFile file : files) {
246  FileIngestTask fileTask = new FileIngestTask(executor, file);
247  if (shouldEnqueueFileTask(fileTask)) {
248  try {
249  fileIngestTasksQueue.putFirst(fileTask);
250  } catch (InterruptedException ex) {
251  DataSource dataSource = executor.getDataSource();
252  logger.log(Level.WARNING, String.format("Interrupted while enqueuing file tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS
253  Thread.currentThread().interrupt();
254  return;
255  }
256  }
257  }
258  }
259  }
260 
272  synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor executor) {
273  if (!executor.isCancelled()) {
274  Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard();
275  try {
276  List<DataArtifact> artifacts = blackboard.getDataArtifacts(executor.getDataSource().getId(), null);
277  scheduleDataArtifactIngestTasks(executor, artifacts);
278  } catch (TskCoreException ex) {
279  DataSource dataSource = executor.getDataSource();
280  logger.log(Level.SEVERE, String.format("Failed to retrieve data artifacts for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS
281  }
282  }
283  }
284 
296  synchronized void scheduleAnalysisResultIngestTasks(IngestJobExecutor executor) {
297  if (!executor.isCancelled()) {
298  Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard();
299  try {
300  List<AnalysisResult> results = blackboard.getAnalysisResults(executor.getDataSource().getId(), null);
301  scheduleAnalysisResultIngestTasks(executor, results);
302  } catch (TskCoreException ex) {
303  DataSource dataSource = executor.getDataSource();
304  logger.log(Level.SEVERE, String.format("Failed to retrieve analysis results for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS
305  }
306  }
307  }
308 
323  synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor executor, List<DataArtifact> artifacts) {
324  if (!executor.isCancelled()) {
325  for (DataArtifact artifact : artifacts) {
326  DataArtifactIngestTask task = new DataArtifactIngestTask(executor, artifact);
327  try {
328  artifactIngestTasksQueue.putLast(task);
329  } catch (InterruptedException ex) {
330  DataSource dataSource = executor.getDataSource();
331  logger.log(Level.WARNING, String.format("Interrupted while enqueuing data artifact tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS
332  Thread.currentThread().interrupt();
333  break;
334  }
335  }
336  }
337  }
338 
353  synchronized void scheduleAnalysisResultIngestTasks(IngestJobExecutor executor, List<AnalysisResult> results) {
354  if (!executor.isCancelled()) {
355  for (AnalysisResult result : results) {
356  AnalysisResultIngestTask task = new AnalysisResultIngestTask(executor, result);
357  try {
358  resultIngestTasksQueue.putLast(task);
359  } catch (InterruptedException ex) {
360  DataSource dataSource = executor.getDataSource();
361  logger.log(Level.WARNING, String.format("Interrupted while enqueuing analysis results tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS
362  Thread.currentThread().interrupt();
363  break;
364  }
365  }
366  }
367  }
368 
375  synchronized void notifyTaskCompleted(DataSourceIngestTask task) {
376  dataSourceIngestTasksQueue.taskCompleted(task);
377  }
378 
385  synchronized void notifyTaskCompleted(FileIngestTask task) {
386  fileIngestTasksQueue.taskCompleted(task);
387  refillFileIngestTasksQueue();
388  }
389 
396  synchronized void notifyTaskCompleted(DataArtifactIngestTask task) {
397  artifactIngestTasksQueue.taskCompleted(task);
398  }
399 
406  synchronized void notifyTaskCompleted(AnalysisResultIngestTask task) {
407  resultIngestTasksQueue.taskCompleted(task);
408  }
409 
418  synchronized boolean currentTasksAreCompleted(Long ingestJobId) {
419  return !(dataSourceIngestTasksQueue.hasTasksForJob(ingestJobId)
420  || hasTasksForJob(topLevelFileIngestTasksQueue, ingestJobId)
421  || hasTasksForJob(batchedFileIngestTasksQueue, ingestJobId)
422  || hasTasksForJob(streamedFileIngestTasksQueue, ingestJobId)
423  || fileIngestTasksQueue.hasTasksForJob(ingestJobId)
424  || artifactIngestTasksQueue.hasTasksForJob(ingestJobId)
425  || resultIngestTasksQueue.hasTasksForJob(ingestJobId));
426  }
427 
447  synchronized void cancelPendingFileTasksForIngestJob(long ingestJobId) {
448  removeTasksForJob(topLevelFileIngestTasksQueue, ingestJobId);
449  removeTasksForJob(batchedFileIngestTasksQueue, ingestJobId);
450  removeTasksForJob(streamedFileIngestTasksQueue, ingestJobId);
451  }
452 
461  private static List<AbstractFile> getTopLevelFiles(Content dataSource) {
462  List<AbstractFile> topLevelFiles = new ArrayList<>();
463  Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
464  if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
465  /*
466  * The data source is itself a file to be processed.
467  */
468  topLevelFiles.add((AbstractFile) dataSource);
469  } else {
470  for (AbstractFile root : rootObjects) {
471  List<Content> children;
472  try {
473  children = root.getChildren();
474  if (children.isEmpty()) {
475  /*
476  * Add the root object itself, it could be an
477  * unallocated space file, or a child of a volume or an
478  * image.
479  */
480  topLevelFiles.add(root);
481  } else {
482  /*
483  * The root object is a file system root directory, get
484  * the files within it.
485  */
486  for (Content child : children) {
487  if (child instanceof AbstractFile) {
488  topLevelFiles.add((AbstractFile) child);
489  }
490  }
491  }
492  } catch (TskCoreException ex) {
493  logger.log(Level.SEVERE, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS
494  }
495  }
496  }
497  return topLevelFiles;
498  }
499 
507  synchronized private void refillFileIngestTasksQueue() {
508  try {
509  takeFromStreamingFileTasksQueue();
510  takeFromBatchTasksQueues();
511  } catch (InterruptedException ex) {
512  IngestTasksScheduler.logger.log(Level.INFO, "Ingest tasks scheduler interrupted while blocked adding a task to the file level ingest task queue", ex);
513  Thread.currentThread().interrupt();
514  }
515  }
516 
523  synchronized private void takeFromStreamingFileTasksQueue() throws InterruptedException {
524  while (fileIngestTasksQueue.isEmpty()) {
525  int taskCount = 0;
526  while (taskCount < IngestManager.getInstance().getNumberOfFileIngestThreads()) {
527  final FileIngestTask streamingTask = streamedFileIngestTasksQueue.poll();
528  if (streamingTask == null) {
529  return; // No streaming tasks are queued right now
530  }
531  if (shouldEnqueueFileTask(streamingTask)) {
532  fileIngestTasksQueue.putLast(streamingTask);
533  taskCount++;
534  }
535  }
536  }
537  }
538 
564  synchronized private void takeFromBatchTasksQueues() throws InterruptedException {
565 
566  while (fileIngestTasksQueue.isEmpty()) {
567  /*
568  * If the batched file task queue is empty, move the highest
569  * priority top level file task into it.
570  */
571  if (batchedFileIngestTasksQueue.isEmpty()) {
572  final FileIngestTask topLevelTask = topLevelFileIngestTasksQueue.pollFirst();
573  if (topLevelTask != null) {
574  batchedFileIngestTasksQueue.addLast(topLevelTask);
575  }
576  }
577 
578  /*
579  * Try to move the next task from the batched file tasks queue into
580  * the queue for the file ingest threads.
581  */
582  final FileIngestTask nextTask = batchedFileIngestTasksQueue.pollFirst();
583  if (nextTask == null) {
584  return;
585  }
586  if (shouldEnqueueFileTask(nextTask)) {
587  fileIngestTasksQueue.putLast(nextTask);
588  }
589 
590  /*
591  * If the task that was just queued for the file ingest threads has
592  * children, queue tasks for the children as well.
593  */
594  AbstractFile file = null;
595  try {
596  file = nextTask.getFile();
597  List<Content> children = file.getChildren();
598  for (Content child : children) {
599  if (child instanceof AbstractFile) {
600  AbstractFile childFile = (AbstractFile) child;
601  FileIngestTask childTask = new FileIngestTask(nextTask.getIngestJobExecutor(), childFile);
602  if (childFile.hasChildren()) {
603  batchedFileIngestTasksQueue.add(childTask);
604  } else if (shouldEnqueueFileTask(childTask)) {
605  fileIngestTasksQueue.putLast(childTask);
606  }
607  }
608  }
609  } catch (TskCoreException ex) {
610  if (file != null) {
611  logger.log(Level.SEVERE, String.format("Error getting the children of %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS
612  } else {
613  logger.log(Level.SEVERE, "Error loading file with object ID = {0}", nextTask.getFileId()); //NON-NLS
614  }
615  }
616  }
617  }
618 
629  private static boolean shouldEnqueueFileTask(final FileIngestTask task) {
630  AbstractFile file;
631  try {
632  file = task.getFile();
633  } catch (TskCoreException ex) {
634  logger.log(Level.SEVERE, "Error loading file with ID {0}", task.getFileId());
635  return false;
636  }
637 
638  /*
639  * Skip the task if the file is actually the pseudo-file for the parent
640  * or current directory.
641  */
642  String fileName = file.getName();
643 
644  if (fileName.equals(".") || fileName.equals("..")) {
645  return false;
646  }
647 
648  /*
649  * Ensures that all directories, files which are members of the ingest
650  * file filter, and unallocated blocks (when processUnallocated is
651  * enabled) all continue to be processed. AbstractFiles which do not
652  * meet one of these criteria will be skipped.
653  *
654  * An additional check to see if unallocated space should be processed
655  * is part of the FilesSet.fileIsMemberOf() method.
656  *
657  * This code may need to be updated when
658  * TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS comes into use by Autopsy.
659  */
660  if (!file.isDir() && !shouldBeCarved(task) && !fileAcceptedByFilter(task)) {
661  return false;
662  }
663 
664  /*
665  * Skip the task if the file is one of a select group of special, large
666  * NTFS or FAT file system files.
667  */
668  if (file instanceof org.sleuthkit.datamodel.File) {
669  final org.sleuthkit.datamodel.File f = (org.sleuthkit.datamodel.File) file;
670 
671  /*
672  * Get the type of the file system, if any, that owns the file.
673  */
674  TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP;
675  try {
676  FileSystem fs = f.getFileSystem();
677  if (fs != null) {
678  fsType = fs.getFsType();
679  }
680  } catch (TskCoreException ex) {
681  logger.log(Level.SEVERE, "Error querying file system for " + f, ex); //NON-NLS
682  }
683 
684  /*
685  * If the file system is not NTFS or FAT, don't skip the file.
686  */
687  if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
688  return true;
689  }
690 
691  /*
692  * Find out whether the file is in a root directory.
693  */
694  boolean isInRootDir = false;
695  try {
696  AbstractFile parent = f.getParentDirectory();
697  if (parent == null) {
698  isInRootDir = true;
699  } else {
700  isInRootDir = parent.isRoot();
701  }
702  } catch (TskCoreException ex) {
703  logger.log(Level.WARNING, "Error querying parent directory for" + f.getName(), ex); //NON-NLS
704  }
705 
706  /*
707  * If the file is in the root directory of an NTFS or FAT file
708  * system, check its meta-address and check its name for the '$'
709  * character and a ':' character (not a default attribute).
710  */
711  if (isInRootDir && f.getMetaAddr() < 32) {
712  String name = f.getName();
713  if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) {
714  return false;
715  }
716  }
717  }
718 
719  return true;
720  }
721 
729  private static boolean shouldBeCarved(final FileIngestTask task) {
730  try {
731  AbstractFile file = task.getFile();
732  return task.getIngestJobExecutor().shouldProcessUnallocatedSpace() && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS);
733  } catch (TskCoreException ex) {
734  return false;
735  }
736  }
737 
746  private static boolean fileAcceptedByFilter(final FileIngestTask task) {
747  try {
748  AbstractFile file = task.getFile();
749  return !(task.getIngestJobExecutor().getFileIngestFilter().fileIsMemberOf(file) == null);
750  } catch (TskCoreException ex) {
751  return false;
752  }
753  }
754 
764  synchronized private static boolean hasTasksForJob(Collection<? extends IngestTask> tasks, long ingestJobId) {
765  for (IngestTask task : tasks) {
766  if (task.getIngestJobExecutor().getIngestJobId() == ingestJobId) {
767  return true;
768  }
769  }
770  return false;
771  }
772 
780  private static void removeTasksForJob(Collection<? extends IngestTask> tasks, long ingestJobId) {
781  Iterator<? extends IngestTask> iterator = tasks.iterator();
782  while (iterator.hasNext()) {
783  IngestTask task = iterator.next();
784  if (task.getIngestJobExecutor().getIngestJobId() == ingestJobId) {
785  iterator.remove();
786  }
787  }
788  }
789 
799  private static int countTasksForJob(Collection<? extends IngestTask> tasks, long ingestJobId) {
800  int count = 0;
801  for (IngestTask task : tasks) {
802  if (task.getIngestJobExecutor().getIngestJobId() == ingestJobId) {
803  count++;
804  }
805  }
806  return count;
807  }
808 
817  synchronized IngestTasksSnapshot getTasksSnapshotForJob(long ingestJobId) {
818  return new IngestTasksSnapshot(
819  ingestJobId,
820  dataSourceIngestTasksQueue.countQueuedTasksForJob(ingestJobId),
821  countTasksForJob(topLevelFileIngestTasksQueue, ingestJobId),
822  countTasksForJob(batchedFileIngestTasksQueue, ingestJobId),
823  fileIngestTasksQueue.countQueuedTasksForJob(ingestJobId),
824  countTasksForJob(streamedFileIngestTasksQueue, ingestJobId),
825  artifactIngestTasksQueue.countQueuedTasksForJob(ingestJobId),
826  resultIngestTasksQueue.countQueuedTasksForJob(ingestJobId),
827  dataSourceIngestTasksQueue.countRunningTasksForJob(ingestJobId) + fileIngestTasksQueue.countRunningTasksForJob(ingestJobId) + artifactIngestTasksQueue.countRunningTasksForJob(ingestJobId) + resultIngestTasksQueue.countRunningTasksForJob(ingestJobId)
828  );
829  }
830 
835  private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
836 
837  @Override
838  public int compare(FileIngestTask q1, FileIngestTask q2) {
839  /*
840  * In practice the case where one or both calls to getFile() fails
841  * should never occur since such tasks would not be added to the
842  * queue.
843  */
844  AbstractFile file1 = null;
845  AbstractFile file2 = null;
846  try {
847  file1 = q1.getFile();
848  } catch (TskCoreException ex) {
849  /*
850  * Do nothing - the exception has been logged elsewhere
851  */
852  }
853 
854  try {
855  file2 = q2.getFile();
856  } catch (TskCoreException ex) {
857  /*
858  * Do nothing - the exception has been logged elsewhere
859  */
860  }
861 
862  if (file1 == null) {
863  if (file2 == null) {
864  return (int) (q2.getFileId() - q1.getFileId());
865  } else {
866  return 1;
867  }
868  } else if (file2 == null) {
869  return -1;
870  }
871 
872  AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(file1);
873  AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(file2);
874  if (p1 == p2) {
875  return (int) (file2.getId() - file1.getId());
876  } else {
877  return p2.ordinal() - p1.ordinal();
878  }
879  }
880 
885  private static class AbstractFilePriority {
886 
888  }
889 
890  enum Priority {
891 
892  LAST, LOW, MEDIUM, HIGH
893  }
894 
895  static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
896 
897  static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
898 
899  static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
900 
901  static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
902 
903  /*
904  * Prioritize root directory folders based on the assumption that we
905  * are looking for user content. Other types of investigations may
906  * want different priorities.
907  */
908  static {
909  // these files have no structure, so they go last
910  //unalloc files are handled as virtual files in getPriority()
911  //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
912  //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
913  LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
914  LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
915  // orphan files are often corrupt and windows does not typically have
916  // user content, so put them towards the bottom
917  LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
918  LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
919  // all other files go into the medium category too
920  MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
921  // user content is top priority
922  HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
923  HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
924  HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
925  HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE));
926  }
927 
935  static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) {
936  if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
937  //quickly filter out unstructured content
938  //non-fs virtual files and dirs, such as representing unalloc space
939  return AbstractFilePriority.Priority.LAST;
940  }
941  //determine the fs files priority by name
942  final String path = abstractFile.getName();
943  if (path == null) {
944  return AbstractFilePriority.Priority.MEDIUM;
945  }
946  for (Pattern p : HIGH_PRI_PATHS) {
947  Matcher m = p.matcher(path);
948  if (m.find()) {
949  return AbstractFilePriority.Priority.HIGH;
950  }
951  }
952  for (Pattern p : MEDIUM_PRI_PATHS) {
953  Matcher m = p.matcher(path);
954  if (m.find()) {
955  return AbstractFilePriority.Priority.MEDIUM;
956  }
957  }
958  for (Pattern p : LOW_PRI_PATHS) {
959  Matcher m = p.matcher(path);
960  if (m.find()) {
961  return AbstractFilePriority.Priority.LOW;
962  }
963  }
964  for (Pattern p : LAST_PRI_PATHS) {
965  Matcher m = p.matcher(path);
966  if (m.find()) {
967  return AbstractFilePriority.Priority.LAST;
968  }
969  }
970  //default is medium
971  return AbstractFilePriority.Priority.MEDIUM;
972  }
973  }
974  }
975 
980  @ThreadSafe
981  private class IngestTaskTrackingQueue implements BlockingIngestTaskQueue {
982 
983  private final BlockingDeque<IngestTask> taskQueue = new LinkedBlockingDeque<>();
984  @GuardedBy("this")
985  private final List<IngestTask> queuedTasks = new LinkedList<>();
986  @GuardedBy("this")
987  private final List<IngestTask> tasksInProgress = new LinkedList<>();
988 
999  void putFirst(IngestTask task) throws InterruptedException {
1000  synchronized (this) {
1001  this.queuedTasks.add(task);
1002  }
1003  try {
1004  this.taskQueue.putFirst(task);
1005  } catch (InterruptedException ex) {
1006  synchronized (this) {
1007  this.queuedTasks.remove(task);
1008  }
1009  throw ex;
1010  }
1011  }
1012 
1023  void putLast(IngestTask task) throws InterruptedException {
1024  synchronized (this) {
1025  this.queuedTasks.add(task);
1026  }
1027  try {
1028  this.taskQueue.putLast(task);
1029  } catch (InterruptedException ex) {
1030  synchronized (this) {
1031  this.queuedTasks.remove(task);
1032  }
1033  throw ex;
1034  }
1035  }
1036 
1047  @Override
1048  public IngestTask getNextTask() throws InterruptedException {
1049  IngestTask task = taskQueue.takeFirst();
1050  synchronized (this) {
1051  this.queuedTasks.remove(task);
1052  this.tasksInProgress.add(task);
1053  }
1054  return task;
1055  }
1056 
1062  boolean isEmpty() {
1063  synchronized (this) {
1064  return this.queuedTasks.isEmpty();
1065  }
1066  }
1067 
1074  void taskCompleted(IngestTask task) {
1075  synchronized (this) {
1076  this.tasksInProgress.remove(task);
1077  }
1078  }
1079 
1088  boolean hasTasksForJob(long ingestJobId) {
1089  synchronized (this) {
1090  return IngestTasksScheduler.hasTasksForJob(queuedTasks, ingestJobId) || IngestTasksScheduler.hasTasksForJob(tasksInProgress, ingestJobId);
1091  }
1092  }
1093 
1101  int countQueuedTasksForJob(long ingestJobId) {
1102  synchronized (this) {
1103  return IngestTasksScheduler.countTasksForJob(queuedTasks, ingestJobId);
1104  }
1105  }
1106 
1114  int countRunningTasksForJob(long ingestJobId) {
1115  synchronized (this) {
1116  return IngestTasksScheduler.countTasksForJob(tasksInProgress, ingestJobId);
1117  }
1118  }
1119 
1120  }
1121 
1126  static final class IngestTasksSnapshot implements Serializable {
1127 
1128  private static final long serialVersionUID = 1L;
1129  private final long ingestJobId;
1130  private final long dataSourceQueueSize;
1131  private final long rootQueueSize;
1132  private final long dirQueueSize;
1133  private final long fileQueueSize;
1134  private final long inProgressListSize;
1135  private final long streamedFileQueueSize;
1136  private final long artifactsQueueSize;
1137  private final long resultsQueueSize;
1138 
1160  IngestTasksSnapshot(long ingestJobId, long dataSourceQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long streamedFileQueueSize, long artifactsQueueSize, long resultsQueueSize, long inProgressListSize) {
1161  this.ingestJobId = ingestJobId;
1162  this.dataSourceQueueSize = dataSourceQueueSize;
1163  this.rootQueueSize = rootQueueSize;
1164  this.dirQueueSize = dirQueueSize;
1165  this.fileQueueSize = fileQueueSize;
1166  this.streamedFileQueueSize = streamedFileQueueSize;
1167  this.artifactsQueueSize = artifactsQueueSize;
1168  this.resultsQueueSize = resultsQueueSize;
1169  this.inProgressListSize = inProgressListSize;
1170  }
1171 
1178  long getIngestJobId() {
1179  return ingestJobId;
1180  }
1181 
1188  long getRootQueueSize() {
1189  return rootQueueSize;
1190  }
1191 
1198  long getDirQueueSize() {
1199  return dirQueueSize;
1200  }
1201 
1208  long getFileQueueSize() {
1209  return fileQueueSize;
1210  }
1211 
1218  long getStreamedFilesQueueSize() {
1219  return streamedFileQueueSize;
1220  }
1221 
1228  long getDataSourceQueueSize() {
1229  return dataSourceQueueSize;
1230  }
1231 
1238  long getArtifactsQueueSize() {
1239  return artifactsQueueSize;
1240  }
1241 
1248  long getResultsQueueSize() {
1249  return resultsQueueSize;
1250  }
1251 
1258  long getProgressListSize() {
1259  return inProgressListSize;
1260  }
1261 
1262  }
1263 
1264 }

Copyright © 2012-2022 Basis Technology. Generated on: Thu Sep 29 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.