Autopsy  4.21.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
FileIngestPipeline.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Date;
23 import java.util.List;
24 import java.util.Optional;
25 import java.util.logging.Level;
26 import org.openide.util.NbBundle;
30 import org.sleuthkit.datamodel.AbstractFile;
31 import org.sleuthkit.datamodel.SleuthkitCase;
32 import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
33 import org.sleuthkit.datamodel.TskCoreException;
34 
39 @NbBundle.Messages({
40  "FileIngestPipeline_SaveResults_Activity=Saving Results"
41 })
42 final class FileIngestPipeline extends IngestPipeline<FileIngestTask> {
43 
44  private static final int FILE_BATCH_SIZE = 500;
45  private static final String SAVE_RESULTS_ACTIVITY = Bundle.FileIngestPipeline_SaveResults_Activity();
46  private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName());
47  private static final IngestManager ingestManager = IngestManager.getInstance();
48  private final IngestJobExecutor ingestJobExecutor;
49  private final List<AbstractFile> fileBatch;
50 
61  FileIngestPipeline(IngestJobExecutor ingestJobExecutor, List<IngestModuleTemplate> moduleTemplates) {
62  super(ingestJobExecutor, moduleTemplates);
63  this.ingestJobExecutor = ingestJobExecutor;
64  fileBatch = new ArrayList<>();
65  }
66 
67  @Override
68  Optional<IngestPipeline.PipelineModule<FileIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
69  Optional<IngestPipeline.PipelineModule<FileIngestTask>> module = Optional.empty();
70  if (template.isFileIngestModuleTemplate()) {
71  FileIngestModule ingestModule = template.createFileIngestModule();
72  module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName()));
73  }
74  return module;
75  }
76 
77  @Override
78  void prepareForTask(FileIngestTask task) throws IngestPipelineException {
79  }
80 
81  @Override
82  void cleanUpAfterTask(FileIngestTask task) throws IngestPipelineException {
83  try {
84  ingestManager.setIngestTaskProgress(task, SAVE_RESULTS_ACTIVITY);
85  AbstractFile file = task.getFile();
86  file.close();
87  cacheFileForBatchUpdate(file);
88  } catch (TskCoreException ex) {
89  throw new IngestPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
90  } finally {
91  ingestManager.setIngestTaskProgressCompleted(task);
92  }
93  }
94 
95  @Override
96  List<IngestModuleError> shutDown() {
97  List<IngestModuleError> errors = new ArrayList<>();
98  Date start = new Date();
99  try {
100  updateBatchedFiles();
101  } catch (IngestPipelineException ex) {
102  errors.add(new IngestModuleError(SAVE_RESULTS_ACTIVITY, ex));
103  }
104  Date finish = new Date();
105  ingestManager.incrementModuleRunTime(SAVE_RESULTS_ACTIVITY, finish.getTime() - start.getTime());
106  errors.addAll(super.shutDown());
107  return errors;
108  }
109 
120  private void cacheFileForBatchUpdate(AbstractFile file) throws IngestPipelineException {
121  /*
122  * Only one file ingest thread at a time will try to access the file
123  * cache. The synchronization here is to ensure visibility of the files
124  * in all of the threads that share the cache, rather than to prevent
125  * simultaneous access in multiple threads.
126  */
127  synchronized (fileBatch) {
128  fileBatch.add(file);
129  if (fileBatch.size() >= FILE_BATCH_SIZE) {
130  updateBatchedFiles();
131  }
132  }
133  }
134 
141  private void updateBatchedFiles() throws IngestPipelineException {
142  /*
143  * Only one file ingest thread at a time will try to access the file
144  * cache. The synchronization here is to ensure visibility of the files
145  * in all of the threads that share the cache, rather than to prevent
146  * simultaneous access in multiple threads.
147  */
148  synchronized (fileBatch) {
149  CaseDbTransaction transaction = null;
150  try {
151  if (!ingestJobExecutor.isCancelled()) {
152  Case currentCase = Case.getCurrentCaseThrows();
153  SleuthkitCase caseDb = currentCase.getSleuthkitCase();
154  transaction = caseDb.beginTransaction();
155  for (AbstractFile file : fileBatch) {
156  file.save(transaction);
157  }
158  transaction.commit();
159  for (AbstractFile file : fileBatch) {
160  IngestManager.getInstance().fireFileIngestDone(file);
161  }
162  }
163  } catch (NoCurrentCaseException | TskCoreException ex) {
164  if (transaction != null) {
165  try {
166  transaction.rollback();
167  } catch (TskCoreException ex1) {
168  logger.log(Level.SEVERE, "Error rolling back transaction after failure to save updated properties for cached files from tasks", ex1);
169  }
170  }
171  throw new IngestPipelineException("Failed to save updated properties for cached files from tasks", ex); //NON-NLS
172  } finally {
173  fileBatch.clear();
174  }
175  }
176  }
177 
182  static final class FileIngestPipelineModule extends IngestPipeline.PipelineModule<FileIngestTask> {
183 
184  private final FileIngestModule module;
185 
194  FileIngestPipelineModule(FileIngestModule module, String displayName) {
195  super(module, displayName);
196  this.module = module;
197  }
198 
199  @Override
200  void process(IngestJobExecutor ingestJobExecutor, FileIngestTask task) throws IngestModuleException {
201  AbstractFile file = null;
202  try {
203  file = task.getFile();
204  } catch (TskCoreException ex) {
205  throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
206  }
207  ingestManager.setIngestTaskProgress(task, getDisplayName());
208  module.process(file);
209  }
210 
211  }
212 
213 }

Copyright © 2012-2022 Basis Technology. Generated on: Tue Feb 6 2024
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.