Autopsy  4.20.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJobSettings.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
22 import java.io.File;
23 import java.io.FileInputStream;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.text.MessageFormat;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.Collections;
33 import java.util.HashSet;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.Objects;
37 import java.util.Set;
38 import java.util.logging.Level;
39 import java.util.stream.Collectors;
40 import java.util.stream.Stream;
41 import org.openide.util.NbBundle;
42 import org.openide.util.io.NbObjectInputStream;
43 import org.openide.util.io.NbObjectOutputStream;
50 
54 public final class IngestJobSettings {
55 
56  private static final String ENABLED_MODULES_PROPERTY = "Enabled_Ingest_Modules"; //NON-NLS
57  private static final String DISABLED_MODULES_PROPERTY = "Disabled_Ingest_Modules"; //NON-NLS
58  private static final String LAST_FILE_INGEST_FILTER_PROPERTY = "Last_File_Ingest_Filter"; //NON-NLS
59  private static final String MODULE_SETTINGS_FOLDER_NAME = "IngestSettings"; //NON-NLS
60 
61  private static final Set<String> DEFAULT_DISABLED_MODULES = Stream.of(
62  "Plaso",
64  ).collect(Collectors.toSet());
65 
66  private static final String MODULE_SETTINGS_FOLDER = Paths.get(
67  Paths.get(PlatformUtil.getUserConfigDirectory()).relativize(Paths.get(PlatformUtil.getModuleConfigDirectory())).toString(),
69  ).toString();
70 
71  private static final String MODULE_SETTINGS_FOLDER_PATH = Paths.get(
74  ).toAbsolutePath().toString();
75 
76  private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
77  private static final CharSequence PYTHON_CLASS_PROXY_PREFIX = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); //NON-NLS
78  private static final Logger logger = Logger.getLogger(IngestJobSettings.class.getName());
79  private final IngestType ingestType;
80  private final List<IngestModuleTemplate> moduleTemplates = new ArrayList<>();
81  private final List<String> warnings = new ArrayList<>();
82  private String executionContext;
84  private String moduleSettingsFolderPath;
85 
89  static String getBaseSettingsPath() {
91  }
92 
101  static String getModuleSettingsResource(String executionContext) {
102  return Paths.get(MODULE_SETTINGS_FOLDER, executionContext).toString();
103  }
104 
117  static Path getSavedModuleSettingsFolder(String executionContext) {
118  return Paths.get(getBaseSettingsPath(), executionContext);
119  }
120 
132  public IngestJobSettings(final String executionContext) {
134  }
135 
150  public IngestJobSettings(String executionContext, IngestType ingestType) {
151  this.ingestType = ingestType;
152  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
153  this.executionContext = executionContext;
154  } else {
155  this.executionContext = executionContext + "." + this.ingestType.name();
156  }
158  this.load();
159  }
160 
174  public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates) {
175  this.ingestType = ingestType;
176  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
177  this.executionContext = executionContext;
178  } else {
179  this.executionContext = executionContext + "." + this.ingestType.name();
180  }
181  this.moduleTemplates.addAll(moduleTemplates);
182  }
183 
198  public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates, FilesSet fileFilter) {
200  this.setFileFilter(fileFilter);
201  }
202 
210  return getSavedModuleSettingsFolder(executionContext);
211  }
212 
216  public void save() {
218  this.store();
219  }
220 
230  public void saveAs(String executionContext) {
231  this.executionContext = executionContext;
233  this.store();
234  }
235 
242  public List<String> getWarnings() {
243  List<String> warningMessages = new ArrayList<>(this.warnings);
244  this.warnings.clear();
245  return warningMessages;
246  }
247 
257  public String getExecutionContext() {
258  return this.executionContext;
259  }
260 
267  if (fileFilter == null) {
268  fileFilter = FilesSetsManager.getDefaultFilter();
269  }
270  return fileFilter;
271  }
272 
278  public void setFileFilter(FilesSet fileIngestFilter) {
279  this.fileFilter = fileIngestFilter;
280  }
281 
287  public List<IngestModuleTemplate> getIngestModuleTemplates() {
288  return Collections.unmodifiableList(this.moduleTemplates);
289  }
290 
296  public void setIngestModuleTemplates(List<IngestModuleTemplate> moduleTemplates) {
297  this.moduleTemplates.clear();
298  this.moduleTemplates.addAll(moduleTemplates);
299  }
300 
306  public List<IngestModuleTemplate> getEnabledIngestModuleTemplates() {
307  List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
308  for (IngestModuleTemplate moduleTemplate : this.moduleTemplates) {
309  if (moduleTemplate.isEnabled()) {
310  enabledModuleTemplates.add(moduleTemplate);
311  }
312  }
313  return enabledModuleTemplates;
314  }
315 
322  public boolean getProcessUnallocatedSpace() {
323  boolean processUnallocated = true;
324  if (!Objects.isNull(this.fileFilter)) {
325  processUnallocated = (this.fileFilter.ingoresUnallocatedSpace() == false);
326  }
327  return processUnallocated;
328  }
329 
335  try {
336  Path folder = getSavedModuleSettingsFolder();
337  Files.createDirectories(folder);
338  this.moduleSettingsFolderPath = folder.toAbsolutePath().toString();
339  } catch (IOException | SecurityException ex) {
340  logger.log(Level.SEVERE, "Failed to create ingest module settings directory " + this.moduleSettingsFolderPath, ex); //NON-NLS
341  this.warnings.add(NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.createModuleSettingsFolder.warning")); //NON-NLS
342  }
343  }
344 
349  private void load() {
354  List<IngestModuleFactory> moduleFactories = new ArrayList<>();
355  List<IngestModuleFactory> allModuleFactories = IngestModuleFactoryLoader.getIngestModuleFactories();
356  HashSet<String> loadedModuleNames = new HashSet<>();
357 
358  // Add modules that are going to be used for this ingest depending on type.
359  for (IngestModuleFactory moduleFactory : allModuleFactories) {
360  if (moduleFactory.isDataArtifactIngestModuleFactory() || ingestType.equals(IngestType.ALL_MODULES)) {
361  moduleFactories.add(moduleFactory);
362  } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) {
363  moduleFactories.add(moduleFactory);
364  } else if (this.ingestType.equals(IngestType.FILES_ONLY) && moduleFactory.isFileIngestModuleFactory()) {
365  moduleFactories.add(moduleFactory);
366  }
367  }
368 
369  for (IngestModuleFactory moduleFactory : moduleFactories) {
370  loadedModuleNames.add(moduleFactory.getModuleDisplayName());
371  }
372 
373 
374  Set<String> defaultEnabledAndLoaded = new HashSet<>();
375  Set<String> defaultDisabledAndLoaded = new HashSet<>();
376  for (String loadedModule: loadedModuleNames) {
377  if (DEFAULT_DISABLED_MODULES.contains(loadedModule)) {
378  defaultDisabledAndLoaded.add(loadedModule);
379  } else {
380  defaultEnabledAndLoaded.add(loadedModule);
381  }
382  }
383 
388  HashSet<String> enabledModuleNames = getModulesNames(this.executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(defaultEnabledAndLoaded));
389  HashSet<String> disabledModuleNames = getModulesNames(this.executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(defaultDisabledAndLoaded)); //NON-NLS
390 
391  // double check to ensure all loaded modules are present in one of the lists in case more modules (in case settings didn't have a module)
392  for (String loadedModule : loadedModuleNames) {
393  // if neither enabled modules or disabled modules contains the loaded module, add it to the default location
394  if (!enabledModuleNames.contains(loadedModule) && !disabledModuleNames.contains(loadedModule)) {
395  if (DEFAULT_DISABLED_MODULES.contains(loadedModule)) {
396  disabledModuleNames.add(loadedModule);
397  } else {
398  enabledModuleNames.add(loadedModule);
399  }
400  }
401  }
402 
406  List<String> missingModuleNames = new ArrayList<>();
407  for (String moduleName : enabledModuleNames) {
408  if (!loadedModuleNames.contains(moduleName)) {
409  missingModuleNames.add(moduleName);
410  }
411  }
412  for (String moduleName : disabledModuleNames) {
413  if (!loadedModuleNames.contains(moduleName)) {
414  logger.log(Level.WARNING, MessageFormat.format("A module marked as disabled in the ingest job settings, ''{0}'', could not be found.", moduleName));
415  }
416  }
417  for (String moduleName : missingModuleNames) {
418  enabledModuleNames.remove(moduleName);
419  disabledModuleNames.remove(moduleName);
420  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.missingModule.warning", moduleName); //NON-NLS
421  logger.log(Level.WARNING, warning);
422  this.warnings.add(warning);
423  }
424 
429  for (IngestModuleFactory moduleFactory : moduleFactories) {
430  IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, loadModuleSettings(moduleFactory));
431  String moduleName = moduleTemplate.getModuleName();
432  if (enabledModuleNames.contains(moduleName)) {
433  moduleTemplate.setEnabled(true);
434  } else if (disabledModuleNames.contains(moduleName)) {
435  moduleTemplate.setEnabled(false);
436  } else {
437  // The module factory was loaded, but the module name does not
438  // appear in the enabled/disabled module settings. Treat the
439  // module as a new module and enable it by default.
440  moduleTemplate.setEnabled(true);
441  enabledModuleNames.add(moduleName);
442  }
443  this.moduleTemplates.add(moduleTemplate);
444  }
445 
450  String ingestModuleResource = getModuleSettingsResource(this.executionContext);
451  ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
452  ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));
453 
459  }
460  try {
461  Map<String, FilesSet> fileIngestFilters = FilesSetsManager.getInstance()
464  fileIngestFilters.put(fSet.getName(), fSet);
465  }
466  this.fileFilter = fileIngestFilters.get(ModuleSettings.getConfigSetting(ingestModuleResource, IngestJobSettings.LAST_FILE_INGEST_FILTER_PROPERTY));
468  this.fileFilter = FilesSetsManager.getDefaultFilter();
469  logger.log(Level.SEVERE, "Failed to get file filter from .properties file, default filter being used", ex); //NON-NLS
470  }
471  }
472 
484  private static HashSet<String> getModulesNames(String executionContext, String propertyName, String defaultSetting) {
485  String ingestModuleResource = getModuleSettingsResource(executionContext);
486  if (ModuleSettings.settingExists(ingestModuleResource, propertyName) == false) {
487  ModuleSettings.setConfigSetting(ingestModuleResource, propertyName, defaultSetting);
488  }
489  HashSet<String> moduleNames = new HashSet<>();
490  String modulesSetting = ModuleSettings.getConfigSetting(ingestModuleResource, propertyName);
491  if (!modulesSetting.isEmpty()) {
492  String[] settingNames = modulesSetting.split(", ");
493  for (String name : settingNames) {
494  /*
495  * Map some obsolete core ingest module names to the current
496  * core ingest module names.
497  */
498  switch (name) {
499  case "Thunderbird Parser": //NON-NLS
500  case "MBox Parser": //NON-NLS
501  moduleNames.add("Email Parser"); //NON-NLS
502  break;
503  case "File Extension Mismatch Detection": //NON-NLS
504  moduleNames.add("Extension Mismatch Detector"); //NON-NLS
505  break;
506  case "EWF Verify": //NON-NLS
507  case "E01 Verify": //NON-NLS
508  case "E01 Verifier": // NON-NLS
509  moduleNames.add("Data Source Integrity"); //NON-NLS
510  break;
511  case "Archive Extractor": //NON-NLS
512  moduleNames.add("Embedded File Extractor"); //NON-NLS
513  break;
514  case "Correlation Engine": //NON-NLS
515  moduleNames.add("Central Repository"); //NON-NLS
516  break;
517  case "Exif Parser": //NON-NLS
518  moduleNames.add("Picture Analyzer"); //NON-NLS
519  break;
520  case "Drone Analyzer":
521  moduleNames.add("DJI Drone Analyzer");
522  break;
523  default:
524  moduleNames.add(name);
525  }
526  }
527  }
528  return moduleNames;
529  }
530 
539  static List<String> getEnabledModules(String context) {
540  return new ArrayList<>(getModulesNames(context, ENABLED_MODULES_PROPERTY, ""));
541  }
542 
553  private boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) {
554  return moduleSettingsFilePath.contains(PYTHON_CLASS_PROXY_PREFIX);
555  }
556 
565  IngestModuleIngestJobSettings settings = null;
566  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
567  File settingsFile = new File(moduleSettingsFilePath);
568  if (settingsFile.exists()) {
569  try (NbObjectInputStream in = new NbObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
570  settings = (IngestModuleIngestJobSettings) in.readObject();
571  } catch (IOException | ClassNotFoundException ex) {
572  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
573  logger.log(Level.WARNING, warning, ex);
574  this.warnings.add(warning);
575  }
576  }
577  if (settings == null) {
578  settings = factory.getDefaultIngestJobSettings();
579  }
580  return settings;
581  }
582 
592  String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT;
593  Path path = Paths.get(this.moduleSettingsFolderPath, fileName);
594  return path.toAbsolutePath().toString();
595  }
596 
600  private void store() {
604  HashSet<String> enabledModuleNames = new HashSet<>();
605  HashSet<String> disabledModuleNames = new HashSet<>();
606  for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
607  saveModuleSettings(moduleTemplate.getModuleFactory(), moduleTemplate.getModuleSettings());
608  String moduleName = moduleTemplate.getModuleName();
609  if (moduleTemplate.isEnabled()) {
610  enabledModuleNames.add(moduleName);
611  } else {
612  disabledModuleNames.add(moduleName);
613  }
614  }
615 
616  String ingestModuleResource = getModuleSettingsResource(this.executionContext);
617  ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
618  ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));
619 
623  ModuleSettings.setConfigSetting(ingestModuleResource, LAST_FILE_INGEST_FILTER_PROPERTY, fileFilter.getName());
624  }
625 
633  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
634  try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) {
635  out.writeObject(settings);
636  } catch (IOException ex) {
637  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsSave.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
638  logger.log(Level.SEVERE, warning, ex);
639  this.warnings.add(warning);
640  }
641  }
642 
651  private static String makeCsvList(Collection<String> collection) {
652  if (collection == null || collection.isEmpty()) {
653  return "";
654  }
655 
656  ArrayList<String> list = new ArrayList<>();
657  list.addAll(collection);
658  StringBuilder csvList = new StringBuilder();
659  for (int i = 0; i < list.size() - 1; ++i) {
660  csvList.append(list.get(i)).append(", ");
661  }
662  csvList.append(list.get(list.size() - 1));
663  return csvList.toString();
664  }
665 
669  public enum IngestType {
670 
682  FILES_ONLY
683  }
684 
685 }
static synchronized String getConfigSetting(String moduleName, String settingName)
IngestJobSettings(final String executionContext)
void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings)
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
static HashSet< String > getModulesNames(String executionContext, String propertyName, String defaultSetting)
static String makeCsvList(Collection< String > collection)
void setIngestModuleTemplates(List< IngestModuleTemplate > moduleTemplates)
static synchronized boolean settingExists(String moduleName, String settingName)
final List< IngestModuleTemplate > moduleTemplates
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates, FilesSet fileFilter)
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory factory)
String getModuleSettingsFilePath(IngestModuleFactory factory)
default IngestModuleIngestJobSettings getDefaultIngestJobSettings()
List< IngestModuleTemplate > getIngestModuleTemplates()
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates)
boolean isPythonModuleSettingsFile(String moduleSettingsFilePath)
IngestJobSettings(String executionContext, IngestType ingestType)

Copyright © 2012-2022 Basis Technology. Generated on: Tue Aug 1 2023
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.