19 package org.sleuthkit.autopsy.ingest;
23 import java.io.FileInputStream;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.text.MessageFormat;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.Collections;
33 import java.util.HashSet;
34 import java.util.List;
36 import java.util.Objects;
38 import java.util.logging.Level;
39 import java.util.stream.Collectors;
40 import java.util.stream.Stream;
41 import org.openide.util.NbBundle;
42 import org.openide.util.io.NbObjectInputStream;
43 import org.openide.util.io.NbObjectOutputStream;
64 ).collect(Collectors.toSet());
74 ).toAbsolutePath().toString();
77 private static final CharSequence
PYTHON_CLASS_PROXY_PREFIX =
"org.python.proxies.".subSequence(0,
"org.python.proxies.".length() - 1);
81 private final List<String>
warnings =
new ArrayList<>();
89 static String getBaseSettingsPath() {
101 static String getModuleSettingsResource(String executionContext) {
102 return Paths.get(MODULE_SETTINGS_FOLDER, executionContext).toString();
118 return Paths.get(getBaseSettingsPath(), executionContext);
155 this.executionContext = executionContext +
"." + this.ingestType.name();
179 this.executionContext = executionContext +
"." + this.ingestType.name();
181 this.moduleTemplates.addAll(moduleTemplates);
230 public void saveAs(String executionContext) {
243 List<String> warningMessages =
new ArrayList<>(this.
warnings);
244 this.warnings.clear();
245 return warningMessages;
267 if (fileFilter == null) {
279 this.fileFilter = fileIngestFilter;
288 return Collections.unmodifiableList(this.moduleTemplates);
297 this.moduleTemplates.clear();
298 this.moduleTemplates.addAll(moduleTemplates);
307 List<IngestModuleTemplate> enabledModuleTemplates =
new ArrayList<>();
309 if (moduleTemplate.isEnabled()) {
310 enabledModuleTemplates.add(moduleTemplate);
313 return enabledModuleTemplates;
323 boolean processUnallocated =
true;
324 if (!Objects.isNull(
this.fileFilter)) {
327 return processUnallocated;
337 Files.createDirectories(folder);
338 this.moduleSettingsFolderPath = folder.toAbsolutePath().toString();
339 }
catch (IOException | SecurityException ex) {
340 logger.log(Level.SEVERE,
"Failed to create ingest module settings directory " +
this.moduleSettingsFolderPath, ex);
341 this.warnings.add(NbBundle.getMessage(
IngestJobSettings.class,
"IngestJobSettings.createModuleSettingsFolder.warning"));
354 List<IngestModuleFactory> moduleFactories =
new ArrayList<>();
355 List<IngestModuleFactory> allModuleFactories = IngestModuleFactoryLoader.getIngestModuleFactories();
356 HashSet<String> loadedModuleNames =
new HashSet<>();
360 if (moduleFactory.isDataArtifactIngestModuleFactory() || ingestType.equals(
IngestType.
ALL_MODULES)) {
361 moduleFactories.add(moduleFactory);
363 moduleFactories.add(moduleFactory);
364 }
else if (this.ingestType.equals(
IngestType.
FILES_ONLY) && moduleFactory.isFileIngestModuleFactory()) {
365 moduleFactories.add(moduleFactory);
370 loadedModuleNames.add(moduleFactory.getModuleDisplayName());
374 Set<String> defaultEnabledAndLoaded =
new HashSet<>();
375 Set<String> defaultDisabledAndLoaded =
new HashSet<>();
376 for (String loadedModule: loadedModuleNames) {
377 if (DEFAULT_DISABLED_MODULES.contains(loadedModule)) {
378 defaultDisabledAndLoaded.add(loadedModule);
380 defaultEnabledAndLoaded.add(loadedModule);
392 for (String loadedModule : loadedModuleNames) {
394 if (!enabledModuleNames.contains(loadedModule) && !disabledModuleNames.contains(loadedModule)) {
395 if (DEFAULT_DISABLED_MODULES.contains(loadedModule)) {
396 disabledModuleNames.add(loadedModule);
398 enabledModuleNames.add(loadedModule);
406 List<String> missingModuleNames =
new ArrayList<>();
407 for (String moduleName : enabledModuleNames) {
408 if (!loadedModuleNames.contains(moduleName)) {
409 missingModuleNames.add(moduleName);
412 for (String moduleName : disabledModuleNames) {
413 if (!loadedModuleNames.contains(moduleName)) {
414 logger.log(Level.WARNING, MessageFormat.format(
"A module marked as disabled in the ingest job settings, ''{0}'', could not be found.", moduleName));
417 for (String moduleName : missingModuleNames) {
418 enabledModuleNames.remove(moduleName);
419 disabledModuleNames.remove(moduleName);
420 String warning = NbBundle.getMessage(
IngestJobSettings.class,
"IngestJobSettings.missingModule.warning", moduleName);
421 logger.log(Level.WARNING, warning);
422 this.warnings.add(warning);
432 if (enabledModuleNames.contains(moduleName)) {
434 }
else if (disabledModuleNames.contains(moduleName)) {
441 enabledModuleNames.add(moduleName);
443 this.moduleTemplates.add(moduleTemplate);
450 String ingestModuleResource = getModuleSettingsResource(this.executionContext);
464 fileIngestFilters.put(fSet.getName(), fSet);
469 logger.log(Level.SEVERE,
"Failed to get file filter from .properties file, default filter being used", ex);
484 private static HashSet<String>
getModulesNames(String executionContext, String propertyName, String defaultSetting) {
485 String ingestModuleResource = getModuleSettingsResource(executionContext);
489 HashSet<String> moduleNames =
new HashSet<>();
491 if (!modulesSetting.isEmpty()) {
492 String[] settingNames = modulesSetting.split(
", ");
493 for (String name : settingNames) {
499 case "Thunderbird Parser":
501 moduleNames.add(
"Email Parser");
503 case "File Extension Mismatch Detection":
504 moduleNames.add(
"Extension Mismatch Detector");
509 moduleNames.add(
"Data Source Integrity");
511 case "Archive Extractor":
512 moduleNames.add(
"Embedded File Extractor");
514 case "Correlation Engine":
515 moduleNames.add(
"Central Repository");
518 moduleNames.add(
"Picture Analyzer");
520 case "Drone Analyzer":
521 moduleNames.add(
"DJI Drone Analyzer");
524 moduleNames.add(name);
539 static List<String> getEnabledModules(String context) {
540 return new ArrayList<>(
getModulesNames(context, ENABLED_MODULES_PROPERTY,
""));
554 return moduleSettingsFilePath.contains(PYTHON_CLASS_PROXY_PREFIX);
567 File settingsFile =
new File(moduleSettingsFilePath);
568 if (settingsFile.exists()) {
569 try (NbObjectInputStream in =
new NbObjectInputStream(
new FileInputStream(settingsFile.getAbsolutePath()))) {
571 }
catch (IOException | ClassNotFoundException ex) {
573 logger.log(Level.WARNING, warning, ex);
574 this.warnings.add(warning);
577 if (settings == null) {
593 Path path = Paths.get(
this.moduleSettingsFolderPath, fileName);
594 return path.toAbsolutePath().toString();
604 HashSet<String> enabledModuleNames =
new HashSet<>();
605 HashSet<String> disabledModuleNames =
new HashSet<>();
607 saveModuleSettings(moduleTemplate.getModuleFactory(), moduleTemplate.getModuleSettings());
608 String moduleName = moduleTemplate.getModuleName();
609 if (moduleTemplate.isEnabled()) {
610 enabledModuleNames.add(moduleName);
612 disabledModuleNames.add(moduleName);
616 String ingestModuleResource = getModuleSettingsResource(this.executionContext);
634 try (NbObjectOutputStream out =
new NbObjectOutputStream(
new FileOutputStream(moduleSettingsFilePath))) {
635 out.writeObject(settings);
636 }
catch (IOException ex) {
638 logger.log(Level.SEVERE, warning, ex);
639 this.warnings.add(warning);
651 private static String
makeCsvList(Collection<String> collection) {
652 if (collection == null || collection.isEmpty()) {
656 ArrayList<String> list =
new ArrayList<>();
657 list.addAll(collection);
658 StringBuilder csvList =
new StringBuilder();
659 for (
int i = 0; i < list.size() - 1; ++i) {
660 csvList.append(list.get(i)).append(
", ");
662 csvList.append(list.get(list.size() - 1));
663 return csvList.toString();
static synchronized String getConfigSetting(String moduleName, String settingName)
static final String DISABLED_MODULES_PROPERTY
IngestJobSettings(final String executionContext)
static final String LAST_FILE_INGEST_FILTER_PROPERTY
static List< FilesSet > getStandardFileIngestFilters()
static final String ENABLED_MODULES_PROPERTY
void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings)
final IngestType ingestType
void setEnabled(boolean enabled)
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
String moduleSettingsFolderPath
static HashSet< String > getModulesNames(String executionContext, String propertyName, String defaultSetting)
String getExecutionContext()
static synchronized FilesSetsManager getInstance()
static final Set< String > DEFAULT_DISABLED_MODULES
static final String MODULE_SETTINGS_FILE_EXT
Map< String, FilesSet > getCustomFileIngestFilters()
static String makeCsvList(Collection< String > collection)
static final String MODULE_SETTINGS_FOLDER
void setIngestModuleTemplates(List< IngestModuleTemplate > moduleTemplates)
static synchronized boolean settingExists(String moduleName, String settingName)
static final String MODULE_SETTINGS_FOLDER_PATH
final List< IngestModuleTemplate > moduleTemplates
static final String MODULE_SETTINGS_FOLDER_NAME
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates, FilesSet fileFilter)
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
void setFileFilter(FilesSet fileIngestFilter)
IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory factory)
Path getSavedModuleSettingsFolder()
static String getDisplayName()
boolean getProcessUnallocatedSpace()
void createSavedModuleSettingsFolder()
static FilesSet getDefaultFilter()
final List< String > warnings
String getModuleSettingsFilePath(IngestModuleFactory factory)
List< String > getWarnings()
default IngestModuleIngestJobSettings getDefaultIngestJobSettings()
List< IngestModuleTemplate > getIngestModuleTemplates()
static final Logger logger
synchronized static Logger getLogger(String name)
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates)
String getModuleDisplayName()
void saveAs(String executionContext)
boolean isPythonModuleSettingsFile(String moduleSettingsFilePath)
boolean ingoresUnallocatedSpace()
static final CharSequence PYTHON_CLASS_PROXY_PREFIX
IngestJobSettings(String executionContext, IngestType ingestType)
static String normalize(String canonicalClassName)