Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJobSettings.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2017 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.FileOutputStream;
24 import java.io.IOException;
25 import java.nio.file.Files;
26 import java.nio.file.Path;
27 import java.nio.file.Paths;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.HashSet;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Objects;
34 import java.util.logging.Level;
35 import org.openide.util.NbBundle;
36 import org.openide.util.io.NbObjectInputStream;
37 import org.openide.util.io.NbObjectOutputStream;
38 import org.python.util.PythonObjectInputStream;
44 
51 public class IngestJobSettings {
52 
53  private static final String ENABLED_MODULES_KEY = "Enabled_Ingest_Modules"; //NON-NLS
54  private static final String DISABLED_MODULES_KEY = "Disabled_Ingest_Modules"; //NON-NLS
55  private static final String LAST_FILE_INGEST_FILTER_KEY = "Last_File_Ingest_Filter"; //NON-NLS
56  private static final String MODULE_SETTINGS_FOLDER = "IngestModuleSettings"; //NON-NLS
57  private static final String MODULE_SETTINGS_FOLDER_PATH = Paths.get(PlatformUtil.getUserConfigDirectory(), IngestJobSettings.MODULE_SETTINGS_FOLDER).toAbsolutePath().toString();
58  private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
59  private static final Logger LOGGER = Logger.getLogger(IngestJobSettings.class.getName());
61  private String executionContext;
62  private final IngestType ingestType;
63  private String moduleSettingsFolderPath;
64  private static final CharSequence pythonModuleSettingsPrefixCS = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); //NON-NLS
65  private final List<IngestModuleTemplate> moduleTemplates;
66  private final List<String> warnings;
67 
75  FilesSet getFileIngestFilter() {
76  if (fileIngestFilter == null) {
77  fileIngestFilter = FilesSetsManager.getDefaultFilter();
78  }
79  return fileIngestFilter;
80  }
81 
88  void setFileIngestFilter(FilesSet fileIngestFilter) {
89  this.fileIngestFilter = fileIngestFilter;
90  }
91 
95  public enum IngestType {
96 
108  FILES_ONLY
109  }
110 
119  public IngestJobSettings(String executionContext) {
120  this.executionContext = executionContext;
121  this.ingestType = IngestType.ALL_MODULES;
122  this.moduleTemplates = new ArrayList<>();
123  this.warnings = new ArrayList<>();
125  this.load();
126  }
127 
137  public IngestJobSettings(String context, IngestType ingestType) {
138  this.ingestType = ingestType;
139 
140  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
141  this.executionContext = context;
142  } else {
143  this.executionContext = context + "." + this.ingestType.name();
144  }
145 
146  this.moduleTemplates = new ArrayList<>();
147 
148  this.warnings = new ArrayList<>();
150  this.load();
151  }
152 
156  public void save() {
157  this.store();
158  }
159 
167  void saveAs(String executionContext) {
168  this.executionContext = executionContext;
170  this.store();
171  }
172 
181  String getExecutionContext() {
182  return this.executionContext;
183  }
184 
191  public List<String> getWarnings() {
192  List<String> warningMessages = new ArrayList<>(this.warnings);
193  this.warnings.clear();
194  return warningMessages;
195  }
196 
202  List<IngestModuleTemplate> getIngestModuleTemplates() {
203  return Collections.unmodifiableList(this.moduleTemplates);
204  }
205 
212  List<IngestModuleTemplate> getEnabledIngestModuleTemplates() {
213  List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
214  for (IngestModuleTemplate moduleTemplate : this.moduleTemplates) {
215  if (moduleTemplate.isEnabled()) {
216  enabledModuleTemplates.add(moduleTemplate);
217  }
218  }
219  return enabledModuleTemplates;
220  }
221 
227  void setIngestModuleTemplates(List<IngestModuleTemplate> moduleTemplates) {
228  this.moduleTemplates.clear();
229  this.moduleTemplates.addAll(moduleTemplates);
230  }
231 
239  boolean getProcessUnallocatedSpace() {
240  /*
241  * Used to be a simple flag but the processUnallocated checkbox was
242  * changed to a skip unallocated. This was due to the FileIngestFilters
243  * needing a default value which did not skip unallocated files. This
244  * method exists to maintain existing functionality.
245  */
246  boolean processUnallocated = true;
247  if (!Objects.isNull(this.fileIngestFilter)) {
248  processUnallocated = (this.fileIngestFilter.ingoresUnallocatedSpace() == false);
249  }
250  return processUnallocated;
251  }
252 
259  return Paths.get(IngestJobSettings.MODULE_SETTINGS_FOLDER_PATH, executionContext);
260  }
261 
270  static Path getSavedModuleSettingsFolder(String context) {
271  return Paths.get(IngestJobSettings.MODULE_SETTINGS_FOLDER_PATH, context);
272  }
273 
279  try {
280  Path folder = getSavedModuleSettingsFolder();
281  Files.createDirectories(folder);
282  this.moduleSettingsFolderPath = folder.toAbsolutePath().toString();
283  } catch (IOException | SecurityException ex) {
284  LOGGER.log(Level.SEVERE, "Failed to create ingest module settings directory " + this.moduleSettingsFolderPath, ex); //NON-NLS
285  this.warnings.add(NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.createModuleSettingsFolder.warning")); //NON-NLS
286  }
287  }
288 
292  private void load() {
297  List<IngestModuleFactory> moduleFactories = new ArrayList<>();
298  List<IngestModuleFactory> allModuleFactories = IngestModuleFactoryLoader.getIngestModuleFactories();
299  HashSet<String> loadedModuleNames = new HashSet<>();
300 
301  // Add modules that are going to be used for this ingest depending on type.
302  for (IngestModuleFactory moduleFactory : allModuleFactories) {
303  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
304  moduleFactories.add(moduleFactory);
305  } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) {
306  moduleFactories.add(moduleFactory);
307  } else if (this.ingestType.equals(IngestType.FILES_ONLY) && moduleFactory.isFileIngestModuleFactory()) {
308  moduleFactories.add(moduleFactory);
309  }
310  }
311 
312  for (IngestModuleFactory moduleFactory : moduleFactories) {
313  loadedModuleNames.add(moduleFactory.getModuleDisplayName());
314  }
315 
320  HashSet<String> enabledModuleNames = getModulesNamesFromSetting(executionContext, IngestJobSettings.ENABLED_MODULES_KEY, makeCommaSeparatedValuesList(loadedModuleNames));
321  HashSet<String> disabledModuleNames = getModulesNamesFromSetting(executionContext, IngestJobSettings.DISABLED_MODULES_KEY, ""); //NON-NLS
322 
326  List<String> missingModuleNames = new ArrayList<>();
327  for (String moduleName : enabledModuleNames) {
328  if (!loadedModuleNames.contains(moduleName)) {
329  missingModuleNames.add(moduleName);
330  }
331  }
332  for (String moduleName : disabledModuleNames) {
333  if (!loadedModuleNames.contains(moduleName)) {
334  missingModuleNames.add(moduleName);
335  }
336  }
337  for (String moduleName : missingModuleNames) {
338  enabledModuleNames.remove(moduleName);
339  disabledModuleNames.remove(moduleName);
340  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.missingModule.warning", moduleName); //NON-NLS
341  LOGGER.log(Level.WARNING, warning);
342  this.warnings.add(warning);
343  }
344 
349  for (IngestModuleFactory moduleFactory : moduleFactories) {
350  IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, loadModuleSettings(moduleFactory));
351  String moduleName = moduleTemplate.getModuleName();
352  if (enabledModuleNames.contains(moduleName)) {
353  moduleTemplate.setEnabled(true);
354  } else if (disabledModuleNames.contains(moduleName)) {
355  moduleTemplate.setEnabled(false);
356  } else {
357  // The module factory was loaded, but the module name does not
358  // appear in the enabled/disabled module settings. Treat the
359  // module as a new module and enable it by default.
360  moduleTemplate.setEnabled(true);
361  enabledModuleNames.add(moduleName);
362  }
363  this.moduleTemplates.add(moduleTemplate);
364  }
365 
372 
376  if (ModuleSettings.settingExists(this.executionContext, IngestJobSettings.LAST_FILE_INGEST_FILTER_KEY) == false) {
378  }
379  try {
380  Map<String, FilesSet> fileIngestFilters = FilesSetsManager.getInstance()
383  fileIngestFilters.put(fSet.getName(), fSet);
384  }
385  this.fileIngestFilter = fileIngestFilters.get(ModuleSettings.getConfigSetting(
386  this.executionContext, IngestJobSettings.LAST_FILE_INGEST_FILTER_KEY));
388  this.fileIngestFilter = FilesSetsManager.getDefaultFilter();
389  LOGGER.log(Level.SEVERE, "Failed to get file ingest filter from .properties file, default filter being used", ex); //NON-NLS
390  }
391  }
392 
401  private static HashSet<String> getModulesNamesFromSetting(String context, String key, String defaultSetting) {
402  if (ModuleSettings.settingExists(context, key) == false) {
403  ModuleSettings.setConfigSetting(context, key, defaultSetting);
404  }
405  HashSet<String> moduleNames = new HashSet<>();
406  String modulesSetting = ModuleSettings.getConfigSetting(context, key);
407  if (!modulesSetting.isEmpty()) {
408  String[] settingNames = modulesSetting.split(", ");
409  for (String name : settingNames) {
410  // Map some old core module names to the current core module names.
411  switch (name) {
412  case "Thunderbird Parser": //NON-NLS
413  case "MBox Parser": //NON-NLS
414  moduleNames.add("Email Parser"); //NON-NLS
415  break;
416  case "File Extension Mismatch Detection": //NON-NLS
417  moduleNames.add("Extension Mismatch Detector"); //NON-NLS
418  break;
419  case "EWF Verify": //NON-NLS
420  case "E01 Verify": //NON-NLS
421  moduleNames.add("E01 Verifier"); //NON-NLS
422  break;
423  case "Archive Extractor": //NON-NLS
424  moduleNames.add("Embedded File Extractor"); //NON-NLS
425  break;
426  default:
427  moduleNames.add(name);
428  }
429  }
430  }
431  return moduleNames;
432  }
433 
442  static List<String> getEnabledModules(String context) {
443  return new ArrayList<>(getModulesNamesFromSetting(context, ENABLED_MODULES_KEY, ""));
444  }
445 
456  private boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) {
457  return moduleSettingsFilePath.contains(pythonModuleSettingsPrefixCS);
458  }
459 
469  IngestModuleIngestJobSettings settings = null;
470  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
471  File settingsFile = new File(moduleSettingsFilePath);
472  if (settingsFile.exists()) {
473  if (!isPythonModuleSettingsFile(moduleSettingsFilePath)) {
474  try (NbObjectInputStream in = new NbObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
475  settings = (IngestModuleIngestJobSettings) in.readObject();
476  } catch (IOException | ClassNotFoundException ex) {
477  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
478  LOGGER.log(Level.WARNING, warning, ex);
479  this.warnings.add(warning);
480  }
481  } else {
482  try (PythonObjectInputStream in = new PythonObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
483  settings = (IngestModuleIngestJobSettings) in.readObject();
484  } catch (IOException | ClassNotFoundException exception) {
485  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
486  LOGGER.log(Level.WARNING, warning, exception);
487  this.warnings.add(warning);
488  }
489  }
490  }
491  if (settings == null) {
492  settings = factory.getDefaultIngestJobSettings();
493  }
494  return settings;
495  }
496 
506  String fileName = factory.getClass().getCanonicalName() + IngestJobSettings.MODULE_SETTINGS_FILE_EXT;
507  Path path = Paths.get(this.moduleSettingsFolderPath, fileName);
508  return path.toAbsolutePath().toString();
509  }
510 
514  private void store() {
518  HashSet<String> enabledModuleNames = new HashSet<>();
519  HashSet<String> disabledModuleNames = new HashSet<>();
520  for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
521  saveModuleSettings(moduleTemplate.getModuleFactory(), moduleTemplate.getModuleSettings());
522  String moduleName = moduleTemplate.getModuleName();
523  if (moduleTemplate.isEnabled()) {
524  enabledModuleNames.add(moduleName);
525  } else {
526  disabledModuleNames.add(moduleName);
527  }
528  }
531 
535  ModuleSettings.setConfigSetting(this.executionContext, LAST_FILE_INGEST_FILTER_KEY, fileIngestFilter.getName());
536  }
537 
546  String moduleSettingsFilePath = Paths.get(this.moduleSettingsFolderPath, FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + MODULE_SETTINGS_FILE_EXT).toString();
547  try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) {
548  out.writeObject(settings);
549  } catch (IOException ex) {
550  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsSave.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
551  LOGGER.log(Level.SEVERE, warning, ex);
552  this.warnings.add(warning);
553  }
554  }
555 
564  private static String makeCommaSeparatedValuesList(HashSet<String> input) {
565  if (input == null || input.isEmpty()) {
566  return "";
567  }
568 
569  ArrayList<String> list = new ArrayList<>();
570  list.addAll(input);
571  StringBuilder csvList = new StringBuilder();
572  for (int i = 0; i < list.size() - 1; ++i) {
573  csvList.append(list.get(i)).append(", ");
574  }
575  csvList.append(list.get(list.size() - 1));
576  return csvList.toString();
577  }
578 
579 }
void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings)
static String makeCommaSeparatedValuesList(HashSet< String > input)
IngestModuleIngestJobSettings getDefaultIngestJobSettings()
final List< IngestModuleTemplate > moduleTemplates
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory factory)
static HashSet< String > getModulesNamesFromSetting(String context, String key, String defaultSetting)
String getModuleSettingsFilePath(IngestModuleFactory factory)
static String getConfigSetting(String moduleName, String settingName)
synchronized static Logger getLogger(String name)
Definition: Logger.java:161
boolean isPythonModuleSettingsFile(String moduleSettingsFilePath)
static boolean settingExists(String moduleName, String settingName)
IngestJobSettings(String context, IngestType ingestType)

Copyright © 2012-2016 Basis Technology. Generated on: Mon Apr 24 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.