Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
CentralRepoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2011-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.ingestmodule;
20 
21 import java.util.ArrayList;
22 import java.util.Collection;
25 import java.util.List;
26 import java.util.logging.Level;
27 import java.util.stream.Collectors;
28 import org.openide.util.NbBundle.Messages;
47 import org.sleuthkit.datamodel.AbstractFile;
48 import org.sleuthkit.datamodel.BlackboardArtifact;
49 import org.sleuthkit.datamodel.BlackboardAttribute;
50 import org.sleuthkit.datamodel.HashUtility;
51 import org.sleuthkit.datamodel.TskCoreException;
52 import org.sleuthkit.datamodel.TskData;
56 import org.sleuthkit.datamodel.SleuthkitCase;
57 
62 @Messages({"CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
63  "CentralRepoIngestModule.prevCaseComment.text=Previous Case: "})
64 final class CentralRepoIngestModule implements FileIngestModule {
65 
66  static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true;
67  static final boolean DEFAULT_FLAG_PREVIOUS_DEVICES = true;
68  static final boolean DEFAULT_CREATE_CR_PROPERTIES = true;
69 
70  private final static Logger logger = Logger.getLogger(CentralRepoIngestModule.class.getName());
71  private final IngestServices services = IngestServices.getInstance();
72  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
73  private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
74  private long jobId;
75  private CorrelationCase eamCase;
76  private CorrelationDataSource eamDataSource;
77  private Blackboard blackboard;
78  private CorrelationAttributeInstance.Type filesType;
79  private final boolean flagTaggedNotableItems;
80  private final boolean flagPreviouslySeenDevices;
81  private final boolean createCorrelationProperties;
82 
88  CentralRepoIngestModule(IngestSettings settings) {
89  flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
90  flagPreviouslySeenDevices = settings.isFlagPreviousDevices();
91  createCorrelationProperties = settings.shouldCreateCorrelationProperties();
92  }
93 
94  @Override
95  public ProcessResult process(AbstractFile abstractFile) {
96  if (EamDb.isEnabled() == false) {
97  /*
98  * Not signaling an error for now. This is a workaround for the way
99  * all newly didscovered ingest modules are automatically anabled.
100  *
101  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
102  */
103  return ProcessResult.OK;
104  }
105 
106  try {
108  } catch (NoCurrentCaseException ex) {
109  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
110  return ProcessResult.ERROR;
111  }
112 
113  if (!EamArtifactUtil.isSupportedAbstractFileType(abstractFile)) {
114  return ProcessResult.OK;
115  }
116 
117  if (abstractFile.getKnown() == TskData.FileKnown.KNOWN) {
118  return ProcessResult.OK;
119  }
120 
121  EamDb dbManager;
122  try {
123  dbManager = EamDb.getInstance();
124  } catch (EamDbException ex) {
125  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
126  return ProcessResult.ERROR;
127  }
128 
129  // only continue if we are correlating filesType
130  if (!filesType.isEnabled()) {
131  return ProcessResult.OK;
132  }
133 
134  // get the hash because we're going to correlate it
135  String md5 = abstractFile.getMd5Hash();
136  if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) {
137  return ProcessResult.OK;
138  }
139 
140  /*
141  * Search the central repo to see if this file was previously marked as
142  * being bad. Create artifact if it was.
143  */
144  if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
145  try {
146  TimingMetric timingMetric = HealthMonitor.getTimingMetric("Correlation Engine: Notable artifact query");
147  List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
148  HealthMonitor.submitTimingMetric(timingMetric);
149  if (!caseDisplayNamesList.isEmpty()) {
150  postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
151  }
152  } catch (EamDbException ex) {
153  logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS
154  return ProcessResult.ERROR;
156  logger.log(Level.INFO, "Error searching database for artifact.", ex); // NON-NLS
157  return ProcessResult.ERROR;
158  }
159  }
160 
161  // insert this file into the central repository
162  if (createCorrelationProperties) {
163  try {
165  filesType,
166  md5,
167  eamCase,
168  eamDataSource,
169  abstractFile.getParentPath() + abstractFile.getName(),
170  null,
171  TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
172  ,
173  abstractFile.getId());
174  dbManager.addAttributeInstanceBulk(cefi);
175  } catch (EamDbException ex) {
176  logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
177  return ProcessResult.ERROR;
179  logger.log(Level.INFO, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
180  return ProcessResult.ERROR;
181  }
182  }
183  return ProcessResult.OK;
184  }
185 
186  @Override
187  public void shutDown() {
189 
190  if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) {
191  return;
192  }
193  EamDb dbManager;
194  try {
195  dbManager = EamDb.getInstance();
196  } catch (EamDbException ex) {
197  logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
198  return;
199  }
200  try {
201  dbManager.commitAttributeInstancesBulk();
202  } catch (EamDbException ex) {
203  logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
204  }
205  try {
206  Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamDataSource);
207  logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS
208  } catch (EamDbException ex) {
209  logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
210  }
211 
212  // TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk.
213  refCounter.decrementAndGet(jobId);
214  }
215 
216  // see ArtifactManagerTimeTester for details
217  @Messages({
218  "CentralRepoIngestModule.notfyBubble.title=Central Repository Not Initialized",
219  "CentralRepoIngestModule.errorMessage.isNotEnabled=Central repository settings are not initialized, cannot run Correlation Engine ingest module."
220  })
221  @Override
222  public void startUp(IngestJobContext context) throws IngestModuleException {
224 
225  /*
226  * Tell the IngestEventsListener to flag notable items based on the
227  * current module's configuration. This is a work around for the lack of
228  * an artifacts pipeline. Note that this can be changed by another
229  * module instance. All modules are affected by the value. While not
230  * ideal, this will be good enough until a better solution can be
231  * posited.
232  *
233  * Note: Flagging cannot be disabled if any other instances of the
234  * Correlation Engine module are running. This restriction is to prevent
235  * missing results in the case where the first module is flagging
236  * notable items, and the proceeding module (with flagging disabled)
237  * causes the first to stop flagging.
238  */
240  IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems);
241  }
243  IngestEventsListener.setFlagSeenDevices(flagPreviouslySeenDevices);
244  }
246  IngestEventsListener.setCreateCrProperties(createCorrelationProperties);
247  }
248 
249  if (EamDb.isEnabled() == false) {
250  /*
251  * Not throwing the customary exception for now. This is a
252  * workaround for the way all newly didscovered ingest modules are
253  * automatically anabled.
254  *
255  * TODO (JIRA-2731): Add isEnabled API for ingest modules.
256  */
258  if (1L == warningMsgRefCounter.incrementAndGet(jobId)) {
259  MessageNotifyUtil.Notify.warn(Bundle.CentralRepoIngestModule_notfyBubble_title(), Bundle.CentralRepoIngestModule_errorMessage_isNotEnabled());
260  }
261  }
262  return;
263  }
264  Case autopsyCase;
265  try {
266  autopsyCase = Case.getCurrentCaseThrows();
267  } catch (NoCurrentCaseException ex) {
268  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
269  throw new IngestModuleException("Exception while getting open case.", ex);
270  }
271 
272  // Don't allow sqlite central repo databases to be used for multi user cases
273  if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE)
275  logger.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository.");
276  throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS
277  }
278  jobId = context.getJobId();
279 
280  EamDb centralRepoDb;
281  try {
282  centralRepoDb = EamDb.getInstance();
283  } catch (EamDbException ex) {
284  logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS
285  throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS
286  }
287 
288  try {
290  } catch (EamDbException ex) {
291  logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
292  throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
293  }
294 
295  try {
296  eamCase = centralRepoDb.getCase(autopsyCase);
297  } catch (EamDbException ex) {
298  throw new IngestModuleException("Unable to get case from central repository database ", ex);
299  }
300  if (eamCase == null) {
301  // ensure we have this case defined in the EAM DB
302  try {
303  eamCase = centralRepoDb.newCase(autopsyCase);
304  } catch (EamDbException ex) {
305  logger.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS
306  throw new IngestModuleException("Error creating new case in ingest module start up.", ex); // NON-NLS
307  }
308  }
309 
310  try {
311  eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource());
312  } catch (EamDbException ex) {
313  logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS
314  throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS
315  }
316  // TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter
317  // if we are the first thread / module for this job, then make sure the case
318  // and image exist in the DB before we associate artifacts with it.
319  if (refCounter.incrementAndGet(jobId)
320  == 1) {
321  // ensure we have this data source in the EAM DB
322  try {
323  if (null == centralRepoDb.getDataSource(eamCase, eamDataSource.getDataSourceObjectID())) {
324  centralRepoDb.newDataSource(eamDataSource);
325  }
326  } catch (EamDbException ex) {
327  logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
328  throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS
329  }
330 
331  }
332  }
333 
340  private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames) {
341 
342  try {
343  String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
344 
345  Collection<BlackboardAttribute> attributes = new ArrayList<>();
346  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME,
347  Bundle.CentralRepoIngestModule_prevTaggedSet_text()));
348  attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME,
349  Bundle.CentralRepoIngestModule_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(",", "", ""))));
350 
351  SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
352  org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard();
353  // Create artifact if it doesn't already exist.
354  if (!tskBlackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) {
355  BlackboardArtifact tifArtifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT);
356  tifArtifact.addAttributes(attributes);
357 
358  try {
359  // index the artifact for keyword search
360  blackboard.indexArtifact(tifArtifact);
361  } catch (Blackboard.BlackboardException ex) {
362  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS
363  }
364 
365  // send inbox message
366  sendBadFileInboxMessage(tifArtifact, abstractFile.getName(), abstractFile.getMd5Hash());
367 
368  // fire event to notify UI of this new artifact
369  services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT));
370  }
371  } catch (TskCoreException ex) {
372  logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
373  } catch (IllegalStateException ex) {
374  logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
375  } catch (NoCurrentCaseException ex) {
376  logger.log(Level.SEVERE, "Exception while getting open case.", ex); // NON-NLS
377  }
378  }
379 
388  @Messages({"CentralRepoIngestModule.postToBB.fileName=File Name",
389  "CentralRepoIngestModule.postToBB.md5Hash=MD5 Hash",
390  "CentralRepoIngestModule.postToBB.hashSetSource=Source of Hash",
391  "CentralRepoIngestModule.postToBB.eamHit=Central Repository",
392  "# {0} - Name of file that is Notable",
393  "CentralRepoIngestModule.postToBB.knownBadMsg=Notable: {0}"})
394  public void sendBadFileInboxMessage(BlackboardArtifact artifact, String name, String md5Hash) {
395  StringBuilder detailsSb = new StringBuilder();
396  //details
397  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
398  //hit
399  detailsSb.append("<tr>"); //NON-NLS
400  detailsSb.append("<th>") //NON-NLS
401  .append(Bundle.CentralRepoIngestModule_postToBB_fileName())
402  .append("</th>"); //NON-NLS
403  detailsSb.append("<td>") //NON-NLS
404  .append(name)
405  .append("</td>"); //NON-NLS
406  detailsSb.append("</tr>"); //NON-NLS
407 
408  detailsSb.append("<tr>"); //NON-NLS
409  detailsSb.append("<th>") //NON-NLS
410  .append(Bundle.CentralRepoIngestModule_postToBB_md5Hash())
411  .append("</th>"); //NON-NLS
412  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
413  detailsSb.append("</tr>"); //NON-NLS
414 
415  detailsSb.append("<tr>"); //NON-NLS
416  detailsSb.append("<th>") //NON-NLS
417  .append(Bundle.CentralRepoIngestModule_postToBB_hashSetSource())
418  .append("</th>"); //NON-NLS
419  detailsSb.append("<td>").append(Bundle.CentralRepoIngestModule_postToBB_eamHit()).append("</td>"); //NON-NLS
420  detailsSb.append("</tr>"); //NON-NLS
421 
422  detailsSb.append("</table>"); //NON-NLS
423 
425  Bundle.CentralRepoIngestModule_postToBB_knownBadMsg(name),
426  detailsSb.toString(),
427  name + md5Hash,
428  artifact));
429  }
430 }
CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
CorrelationCase newCase(CorrelationCase eamCase)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
static TimingMetric getTimingMetric(String name)
void addAttributeInstanceBulk(CorrelationAttributeInstance eamArtifact)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
Long getCountArtifactInstancesByCaseDataSource(CorrelationDataSource correlationDataSource)
static void submitTimingMetric(TimingMetric metric)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:58
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static void warn(String title, String message)
static synchronized IngestServices getInstance()

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.