Autopsy  4.17.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
CentralRepoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2011-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.ingestmodule;
20 
21 import java.util.Arrays;
22 import java.util.Collection;
23 import java.util.List;
24 import java.util.logging.Level;
25 import java.util.stream.Collectors;
26 import org.openide.util.NbBundle.Messages;
48 import org.sleuthkit.datamodel.AbstractFile;
49 import org.sleuthkit.datamodel.Blackboard;
50 import org.sleuthkit.datamodel.BlackboardArtifact;
51 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT;
52 import org.sleuthkit.datamodel.BlackboardAttribute;
53 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
54 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
55 import org.sleuthkit.datamodel.HashUtility;
56 import org.sleuthkit.datamodel.TskCoreException;
57 import org.sleuthkit.datamodel.TskData;
59 
64 @Messages({"CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
65  "CentralRepoIngestModule.prevCaseComment.text=Previous Case: "})
66 final class CentralRepoIngestModule implements FileIngestModule {
67 
68  private static final String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
69 
70  static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = false;
71  static final boolean DEFAULT_FLAG_PREVIOUS_DEVICES = false;
72  static final boolean DEFAULT_CREATE_CR_PROPERTIES = true;
73 
74  private final static Logger logger = Logger.getLogger(CentralRepoIngestModule.class.getName());
75  private final IngestServices services = IngestServices.getInstance();
76  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
77  private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
78  private long jobId;
79  private CorrelationCase eamCase;
80  private CorrelationDataSource eamDataSource;
81  private CorrelationAttributeInstance.Type filesType;
82  private final boolean flagTaggedNotableItems;
83  private final boolean flagPreviouslySeenDevices;
84  private Blackboard blackboard;
85  private final boolean createCorrelationProperties;
86 
92  CentralRepoIngestModule(IngestSettings settings) {
93  flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
94  flagPreviouslySeenDevices = settings.isFlagPreviousDevices();
95  createCorrelationProperties = settings.shouldCreateCorrelationProperties();
96  }
97 
98  @Override
99  public ProcessResult process(AbstractFile abstractFile) {
100  if (CentralRepository.isEnabled() == false) {
101  /*
102  * Not signaling an error for now. This is a workaround for the way
103  * all newly didscovered ingest modules are automatically anabled.
104  *
105  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
106  */
107  return ProcessResult.OK;
108  }
109 
110  try {
111  blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
112  } catch (NoCurrentCaseException ex) {
113  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
114  return ProcessResult.ERROR;
115  }
116 
118  return ProcessResult.OK;
119  }
120 
121  if (abstractFile.getKnown() == TskData.FileKnown.KNOWN) {
122  return ProcessResult.OK;
123  }
124 
125  CentralRepository dbManager;
126  try {
127  dbManager = CentralRepository.getInstance();
128  } catch (CentralRepoException ex) {
129  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
130  return ProcessResult.ERROR;
131  }
132 
133  // only continue if we are correlating filesType
134  if (!filesType.isEnabled()) {
135  return ProcessResult.OK;
136  }
137 
138  // get the hash because we're going to correlate it
139  String md5 = abstractFile.getMd5Hash();
140  if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
141  return ProcessResult.OK;
142  }
143 
144  /*
145  * Search the central repo to see if this file was previously marked as
146  * being bad. Create artifact if it was.
147  */
148  if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
149  try {
150  TimingMetric timingMetric = HealthMonitor.getTimingMetric("Central Repository: Notable artifact query");
151  List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
152  HealthMonitor.submitTimingMetric(timingMetric);
153  if (!caseDisplayNamesList.isEmpty()) {
154  postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
155  }
156  } catch (CentralRepoException ex) {
157  logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS
158  return ProcessResult.ERROR;
160  logger.log(Level.INFO, "Error searching database for artifact.", ex); // NON-NLS
161  return ProcessResult.ERROR;
162  }
163  }
164 
165  // insert this file into the central repository
166  if (createCorrelationProperties) {
167  try {
169  filesType,
170  md5,
171  eamCase,
172  eamDataSource,
173  abstractFile.getParentPath() + abstractFile.getName(),
174  null,
175  TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
176  ,
177  abstractFile.getId());
178  dbManager.addAttributeInstanceBulk(cefi);
179  } catch (CentralRepoException ex) {
180  logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
181  return ProcessResult.ERROR;
183  logger.log(Level.INFO, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
184  return ProcessResult.ERROR;
185  }
186  }
187  return ProcessResult.OK;
188  }
189 
190  @Override
191  public void shutDown() {
193 
194  if ((CentralRepository.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) {
195  return;
196  }
197  CentralRepository dbManager;
198  try {
199  dbManager = CentralRepository.getInstance();
200  } catch (CentralRepoException ex) {
201  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
202  return;
203  }
204  try {
205  dbManager.commitAttributeInstancesBulk();
206  } catch (CentralRepoException ex) {
207  logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
208  }
209  try {
210  Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamDataSource);
211  logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS
212  } catch (CentralRepoException ex) {
213  logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
214  }
215 
216  // TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk.
217  refCounter.decrementAndGet(jobId);
218  }
219 
220  // see ArtifactManagerTimeTester for details
221  @Messages({
222  "CentralRepoIngestModule.notfyBubble.title=Central Repository Not Initialized",
223  "CentralRepoIngestModule.errorMessage.isNotEnabled=Central repository settings are not initialized, cannot run Central Repository ingest module."
224  })
225  @Override
226  public void startUp(IngestJobContext context) throws IngestModuleException {
228 
229  /*
230  * Tell the IngestEventsListener to flag notable items based on the
231  * current module's configuration. This is a work around for the lack of
232  * an artifacts pipeline. Note that this can be changed by another
233  * module instance. All modules are affected by the value. While not
234  * ideal, this will be good enough until a better solution can be
235  * posited.
236  *
237  * Note: Flagging cannot be disabled if any other instances of the
238  * Central Repository module are running. This restriction is to prevent
239  * missing results in the case where the first module is flagging
240  * notable items, and the proceeding module (with flagging disabled)
241  * causes the first to stop flagging.
242  */
244  IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems);
245  }
247  IngestEventsListener.setFlagSeenDevices(flagPreviouslySeenDevices);
248  }
250  IngestEventsListener.setCreateCrProperties(createCorrelationProperties);
251  }
252 
253  if (CentralRepository.isEnabled() == false) {
254  /*
255  * Not throwing the customary exception for now. This is a
256  * workaround for the way all newly didscovered ingest modules are
257  * automatically anabled.
258  *
259  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
260  */
262  if (1L == warningMsgRefCounter.incrementAndGet(jobId)) {
263  MessageNotifyUtil.Notify.warn(Bundle.CentralRepoIngestModule_notfyBubble_title(), Bundle.CentralRepoIngestModule_errorMessage_isNotEnabled());
264  }
265  }
266  return;
267  }
268  Case autopsyCase;
269  try {
270  autopsyCase = Case.getCurrentCaseThrows();
271  } catch (NoCurrentCaseException ex) {
272  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
273  throw new IngestModuleException("Exception while getting open case.", ex);
274  }
275 
276  // Don't allow sqlite central repo databases to be used for multi user cases
277  if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE)
279  logger.log(Level.SEVERE, "Cannot run Central Repository ingest module on a multi-user case with a SQLite central repository.");
280  throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS
281  }
282  jobId = context.getJobId();
283 
284  CentralRepository centralRepoDb;
285  try {
286  centralRepoDb = CentralRepository.getInstance();
287  } catch (CentralRepoException ex) {
288  logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS
289  throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS
290  }
291 
292  try {
294  } catch (CentralRepoException ex) {
295  logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
296  throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
297  }
298 
299  try {
300  eamCase = centralRepoDb.getCase(autopsyCase);
301  } catch (CentralRepoException ex) {
302  throw new IngestModuleException("Unable to get case from central repository database ", ex);
303  }
304 
305  try {
306  eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource());
307  } catch (CentralRepoException ex) {
308  logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS
309  throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS
310  }
311  // TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter
312  // if we are the first thread / module for this job, then make sure the case
313  // and image exist in the DB before we associate artifacts with it.
314  if (refCounter.incrementAndGet(jobId)
315  == 1) {
316  // ensure we have this data source in the EAM DB
317  try {
318  if (null == centralRepoDb.getDataSource(eamCase, eamDataSource.getDataSourceObjectID())) {
319  centralRepoDb.newDataSource(eamDataSource);
320  }
321  } catch (CentralRepoException ex) {
322  logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
323  throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS
324  }
325 
326  }
327  }
328 
335  private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames) {
336 
337  Collection<BlackboardAttribute> attributes = Arrays.asList(
338  new BlackboardAttribute(
339  TSK_SET_NAME, MODULE_NAME,
340  Bundle.CentralRepoIngestModule_prevTaggedSet_text()),
341  new BlackboardAttribute(
342  TSK_COMMENT, MODULE_NAME,
343  Bundle.CentralRepoIngestModule_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))));
344  try {
345 
346  // Create artifact if it doesn't already exist.
347  if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_FILE_HIT, attributes)) {
348  BlackboardArtifact tifArtifact = abstractFile.newArtifact(TSK_INTERESTING_FILE_HIT);
349  tifArtifact.addAttributes(attributes);
350  try {
351  // index the artifact for keyword search
352  blackboard.postArtifact(tifArtifact, MODULE_NAME);
353  } catch (Blackboard.BlackboardException ex) {
354  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
355  }
356  // send inbox message
357  sendBadFileInboxMessage(tifArtifact, abstractFile.getName(), abstractFile.getMd5Hash(), caseDisplayNames);
358  }
359  } catch (TskCoreException ex) {
360  logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
361  } catch (IllegalStateException ex) {
362  logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
363  }
364  }
365 
366  @Messages({
367  "CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br>",
368  "CentralRepoIngestModel_name_header=Name:<br>",
369  "CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br>",
370  "# {0} - Name of file that is Notable",
371  "CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0}"
372  })
373 
383  private void sendBadFileInboxMessage(BlackboardArtifact artifact, String name, String md5Hash, List<String> caseDisplayNames) {
384  StringBuilder detailsSb = new StringBuilder(1024);
385 
386  detailsSb.append(Bundle.CentralRepoIngestModule_notable_message_header()).append(Bundle.CentralRepoIngestModel_name_header());
387  detailsSb.append(name).append(Bundle.CentralRepoIngestModel_previous_case_header());
388  for (String str : caseDisplayNames) {
389  detailsSb.append(str).append("<br>");
390  }
391  detailsSb.append("</html>");
393  Bundle.CentralRepoIngestModule_postToBB_knownBadMsg(name),
394  detailsSb.toString(),
395  name + md5Hash,
396  artifact));
397  }
398 }
void addAttributeInstanceBulk(CorrelationAttributeInstance eamArtifact)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
static TimingMetric getTimingMetric(String name)
void postMessage(final IngestMessage message)
CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource)
static void submitTimingMetric(TimingMetric metric)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId)
Long getCountArtifactInstancesByCaseDataSource(CorrelationDataSource correlationDataSource)
static void warn(String title, String message)
static synchronized IngestServices getInstance()

Copyright © 2012-2021 Basis Technology. Generated on: Tue Jan 19 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.