Autopsy  4.19.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
FileIngestPipeline.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Date;
23 import java.util.List;
24 import java.util.Optional;
25 import java.util.logging.Level;
26 import org.openide.util.NbBundle;
30 import org.sleuthkit.datamodel.AbstractFile;
31 import org.sleuthkit.datamodel.SleuthkitCase;
32 import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
33 import org.sleuthkit.datamodel.TskCoreException;
34 
39 @NbBundle.Messages({
40  "FileIngestPipeline_SaveResults_Activity=Saving Results"
41 })
42 final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
43 
44  private static final int FILE_BATCH_SIZE = 500;
45  private static final String SAVE_RESULTS_ACTIVITY = Bundle.FileIngestPipeline_SaveResults_Activity();
46  private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName());
47  private static final IngestManager ingestManager = IngestManager.getInstance();
48  private final IngestJobPipeline ingestJobPipeline;
49  private final List<AbstractFile> fileBatch;
50 
59  FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
60  super(ingestJobPipeline, moduleTemplates);
61  this.ingestJobPipeline = ingestJobPipeline;
62  fileBatch = new ArrayList<>();
63  }
64 
65  @Override
66  Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
67  Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> module = Optional.empty();
68  if (template.isFileIngestModuleTemplate()) {
69  FileIngestModule ingestModule = template.createFileIngestModule();
70  module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName()));
71  }
72  return module;
73  }
74 
75  @Override
76  void prepareForTask(FileIngestTask task) throws IngestTaskPipelineException {
77  }
78 
79  @Override
80  void cleanUpAfterTask(FileIngestTask task) throws IngestTaskPipelineException {
81  try {
82  ingestManager.setIngestTaskProgress(task, SAVE_RESULTS_ACTIVITY);
83  AbstractFile file = task.getFile();
84  file.close();
85  cacheFileForBatchUpdate(file);
86  } catch (TskCoreException ex) {
87  throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
88  } finally {
89  ingestManager.setIngestTaskProgressCompleted(task);
90  }
91  }
92 
93  @Override
94  List<IngestModuleError> shutDown() {
95  List<IngestModuleError> errors = new ArrayList<>();
96  Date start = new Date();
97  try {
98  updateBatchedFiles();
99  } catch (IngestTaskPipelineException ex) {
100  errors.add(new IngestModuleError(SAVE_RESULTS_ACTIVITY, ex));
101  }
102  Date finish = new Date();
103  ingestManager.incrementModuleRunTime(SAVE_RESULTS_ACTIVITY, finish.getTime() - start.getTime());
104  errors.addAll(super.shutDown());
105  return errors;
106  }
107 
118  private void cacheFileForBatchUpdate(AbstractFile file) throws IngestTaskPipelineException {
119  /*
120  * Only one file ingest thread at a time will try to access the file
121  * cache. The synchronization here is to ensure visibility of the files
122  * in all of the threads that share the cache, rather than to prevent
123  * simultaneous access in multiple threads.
124  */
125  synchronized (fileBatch) {
126  fileBatch.add(file);
127  if (fileBatch.size() >= FILE_BATCH_SIZE) {
128  updateBatchedFiles();
129  }
130  }
131  }
132 
139  private void updateBatchedFiles() throws IngestTaskPipelineException {
140  /*
141  * Only one file ingest thread at a time will try to access the file
142  * cache. The synchronization here is to ensure visibility of the files
143  * in all of the threads that share the cache, rather than to prevent
144  * simultaneous access in multiple threads.
145  */
146  synchronized (fileBatch) {
147  CaseDbTransaction transaction = null;
148  try {
149  if (!ingestJobPipeline.isCancelled()) {
150  Case currentCase = Case.getCurrentCaseThrows();
151  SleuthkitCase caseDb = currentCase.getSleuthkitCase();
152  transaction = caseDb.beginTransaction();
153  for (AbstractFile file : fileBatch) {
154  file.save(transaction);
155  }
156  transaction.commit();
157  for (AbstractFile file : fileBatch) {
158  IngestManager.getInstance().fireFileIngestDone(file);
159  }
160  }
161  } catch (NoCurrentCaseException | TskCoreException ex) {
162  if (transaction != null) {
163  try {
164  transaction.rollback();
165  } catch (TskCoreException ex1) {
166  logger.log(Level.SEVERE, "Error rolling back transaction after failure to save updated properties for cached files from tasks", ex1);
167  }
168  }
169  throw new IngestTaskPipelineException("Failed to save updated properties for cached files from tasks", ex); //NON-NLS
170  } finally {
171  fileBatch.clear();
172  }
173  }
174  }
175 
180  static final class FileIngestPipelineModule extends IngestTaskPipeline.PipelineModule<FileIngestTask> {
181 
182  private final FileIngestModule module;
183 
192  FileIngestPipelineModule(FileIngestModule module, String displayName) {
193  super(module, displayName);
194  this.module = module;
195  }
196 
197  @Override
198  void executeTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException {
199  AbstractFile file = null;
200  try {
201  file = task.getFile();
202  } catch (TskCoreException ex) {
203  throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
204  }
205  ingestManager.setIngestTaskProgress(task, getDisplayName());
206  module.process(file);
207  }
208 
209  }
210 
211 }

Copyright © 2012-2021 Basis Technology. Generated on: Thu Sep 30 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.