IngestJobSettings.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2014-2021 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import com.basistech.df.cybertriage.autopsy.malwarescan.MalwareScanIngestModuleFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.openide.util.NbBundle;
import org.openide.util.io.NbObjectInputStream;
import org.openide.util.io.NbObjectOutputStream;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSet;
import org.sleuthkit.autopsy.modules.interestingitems.FilesSetsManager;
import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer;

public final class IngestJobSettings {

    private static final String ENABLED_MODULES_PROPERTY = "Enabled_Ingest_Modules"; //NON-NLS
    private static final String DISABLED_MODULES_PROPERTY = "Disabled_Ingest_Modules"; //NON-NLS
    private static final String LAST_FILE_INGEST_FILTER_PROPERTY = "Last_File_Ingest_Filter"; //NON-NLS
    private static final String MODULE_SETTINGS_FOLDER_NAME = "IngestSettings"; //NON-NLS
    
    private static final Set<String> DEFAULT_DISABLED_MODULES = Stream.of(
            "Plaso",
            MalwareScanIngestModuleFactory.getDisplayName()
    ).collect(Collectors.toSet());
    
    private static final String MODULE_SETTINGS_FOLDER = Paths.get(
            Paths.get(PlatformUtil.getUserConfigDirectory()).relativize(Paths.get(PlatformUtil.getModuleConfigDirectory())).toString(),
            MODULE_SETTINGS_FOLDER_NAME
    ).toString();

    private static final String MODULE_SETTINGS_FOLDER_PATH = Paths.get(
            PlatformUtil.getModuleConfigDirectory(),
            IngestJobSettings.MODULE_SETTINGS_FOLDER_NAME
    ).toAbsolutePath().toString();

    private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
    private static final CharSequence PYTHON_CLASS_PROXY_PREFIX = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); //NON-NLS
    private static final Logger logger = Logger.getLogger(IngestJobSettings.class.getName());
    private final IngestType ingestType;
    private final List<IngestModuleTemplate> moduleTemplates = new ArrayList<>();
    private final List<String> warnings = new ArrayList<>();
    private String executionContext;
    private FilesSet fileFilter;
    private String moduleSettingsFolderPath;

    static String getBaseSettingsPath() {
        return MODULE_SETTINGS_FOLDER_PATH;
    }

    static String getModuleSettingsResource(String executionContext) {
        return Paths.get(MODULE_SETTINGS_FOLDER, executionContext).toString();
    }

    static Path getSavedModuleSettingsFolder(String executionContext) {
        return Paths.get(getBaseSettingsPath(), executionContext);
    }

    public IngestJobSettings(final String executionContext) {
        this(executionContext, IngestType.ALL_MODULES);
    }

    public IngestJobSettings(String executionContext, IngestType ingestType) {
        this.ingestType = ingestType;
        if (this.ingestType.equals(IngestType.ALL_MODULES)) {
            this.executionContext = executionContext;
        } else {
            this.executionContext = executionContext + "." + this.ingestType.name();
        }
        this.createSavedModuleSettingsFolder();
        this.load();
    }

    public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates) {
        this.ingestType = ingestType;
        if (this.ingestType.equals(IngestType.ALL_MODULES)) {
            this.executionContext = executionContext;
        } else {
            this.executionContext = executionContext + "." + this.ingestType.name();
        }
        this.moduleTemplates.addAll(moduleTemplates);
    }

    public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates, FilesSet fileFilter) {
        this(executionContext, ingestType, moduleTemplates);
        this.setFileFilter(fileFilter);
    }

    public Path getSavedModuleSettingsFolder() {
        return getSavedModuleSettingsFolder(executionContext);
    }
    
    public void save() {
        this.createSavedModuleSettingsFolder();
        this.store();
    }

    public void saveAs(String executionContext) {
        this.executionContext = executionContext;
        this.createSavedModuleSettingsFolder();
        this.store();
    }

    public List<String> getWarnings() {
        List<String> warningMessages = new ArrayList<>(this.warnings);
        this.warnings.clear();
        return warningMessages;
    }

    public String getExecutionContext() {
        return this.executionContext;
    }

    public FilesSet getFileFilter() {
        if (fileFilter == null) {
            fileFilter = FilesSetsManager.getDefaultFilter();
        }
        return fileFilter;
    }

    public void setFileFilter(FilesSet fileIngestFilter) {
        this.fileFilter = fileIngestFilter;
    }

    public List<IngestModuleTemplate> getIngestModuleTemplates() {
        return Collections.unmodifiableList(this.moduleTemplates);
    }

    public void setIngestModuleTemplates(List<IngestModuleTemplate> moduleTemplates) {
        this.moduleTemplates.clear();
        this.moduleTemplates.addAll(moduleTemplates);
    }

    public List<IngestModuleTemplate> getEnabledIngestModuleTemplates() {
        List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
        for (IngestModuleTemplate moduleTemplate : this.moduleTemplates) {
            if (moduleTemplate.isEnabled()) {
                enabledModuleTemplates.add(moduleTemplate);
            }
        }
        return enabledModuleTemplates;
    }

    public boolean getProcessUnallocatedSpace() {
        boolean processUnallocated = true;
        if (!Objects.isNull(this.fileFilter)) {
            processUnallocated = (this.fileFilter.ingoresUnallocatedSpace() == false);
        }
        return processUnallocated;
    }

    private void createSavedModuleSettingsFolder() {
        try {
            Path folder = getSavedModuleSettingsFolder();
            Files.createDirectories(folder);
            this.moduleSettingsFolderPath = folder.toAbsolutePath().toString();
        } catch (IOException | SecurityException ex) {
            logger.log(Level.SEVERE, "Failed to create ingest module settings directory " + this.moduleSettingsFolderPath, ex); //NON-NLS
            this.warnings.add(NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.createModuleSettingsFolder.warning")); //NON-NLS
        }
    }

    private void load() {
        List<IngestModuleFactory> moduleFactories = new ArrayList<>();
        List<IngestModuleFactory> allModuleFactories = IngestModuleFactoryLoader.getIngestModuleFactories();
        HashSet<String> loadedModuleNames = new HashSet<>();

        // Add modules that are going to be used for this ingest depending on type.
        for (IngestModuleFactory moduleFactory : allModuleFactories) {
            if (moduleFactory.isDataArtifactIngestModuleFactory() || ingestType.equals(IngestType.ALL_MODULES)) {
                moduleFactories.add(moduleFactory);
            } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) {
                moduleFactories.add(moduleFactory);
            } else if (this.ingestType.equals(IngestType.FILES_ONLY) && moduleFactory.isFileIngestModuleFactory()) {
                moduleFactories.add(moduleFactory);
            }
        }

        for (IngestModuleFactory moduleFactory : moduleFactories) {
            loadedModuleNames.add(moduleFactory.getModuleDisplayName());
        }


        Set<String> defaultEnabledAndLoaded = new HashSet<>();
        Set<String> defaultDisabledAndLoaded = new HashSet<>();
        for (String loadedModule: loadedModuleNames) {
            if (DEFAULT_DISABLED_MODULES.contains(loadedModule)) {
                defaultDisabledAndLoaded.add(loadedModule);
            } else {
                defaultEnabledAndLoaded.add(loadedModule);
            }
        }

        HashSet<String> enabledModuleNames = getModulesNames(this.executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(defaultEnabledAndLoaded));
        HashSet<String> disabledModuleNames = getModulesNames(this.executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(defaultDisabledAndLoaded)); //NON-NLS

        // double check to ensure all loaded modules are present in one of the lists in case more modules (in case settings didn't have a module)
        for (String loadedModule : loadedModuleNames) {
            // if neither enabled modules or disabled modules contains the loaded module, add it to the default location
            if (!enabledModuleNames.contains(loadedModule) && !disabledModuleNames.contains(loadedModule)) {
                if (DEFAULT_DISABLED_MODULES.contains(loadedModule)) {
                    disabledModuleNames.add(loadedModule);
                } else {
                    enabledModuleNames.add(loadedModule);
                }
            }
        }
        
        List<String> missingModuleNames = new ArrayList<>();
        for (String moduleName : enabledModuleNames) {
            if (!loadedModuleNames.contains(moduleName)) {
                missingModuleNames.add(moduleName);
            }
        }
        for (String moduleName : disabledModuleNames) {
            if (!loadedModuleNames.contains(moduleName)) {
                logger.log(Level.WARNING, MessageFormat.format("A module marked as disabled in the ingest job settings, ''{0}'', could not be found.", moduleName));
            }
        }
        for (String moduleName : missingModuleNames) {
            enabledModuleNames.remove(moduleName);
            disabledModuleNames.remove(moduleName);
            String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.missingModule.warning", moduleName); //NON-NLS
            logger.log(Level.WARNING, warning);
            this.warnings.add(warning);
        }

        for (IngestModuleFactory moduleFactory : moduleFactories) {
            IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, loadModuleSettings(moduleFactory));
            String moduleName = moduleTemplate.getModuleName();
            if (enabledModuleNames.contains(moduleName)) {
                moduleTemplate.setEnabled(true);
            } else if (disabledModuleNames.contains(moduleName)) {
                moduleTemplate.setEnabled(false);
            } else {
                // The module factory was loaded, but the module name does not
                // appear in the enabled/disabled module settings. Treat the
                // module as a new module and enable it by default.
                moduleTemplate.setEnabled(true);
                enabledModuleNames.add(moduleName);
            }
            this.moduleTemplates.add(moduleTemplate);
        }

        String ingestModuleResource = getModuleSettingsResource(this.executionContext);
        ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
        ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));

        if (ModuleSettings.settingExists(ingestModuleResource, IngestJobSettings.LAST_FILE_INGEST_FILTER_PROPERTY) == false) {
            ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.LAST_FILE_INGEST_FILTER_PROPERTY, FilesSetsManager.getDefaultFilter().getName());
        }
        try {
            Map<String, FilesSet> fileIngestFilters = FilesSetsManager.getInstance()
                    .getCustomFileIngestFilters();
            for (FilesSet fSet : FilesSetsManager.getStandardFileIngestFilters()) {
                fileIngestFilters.put(fSet.getName(), fSet);
            }
            this.fileFilter = fileIngestFilters.get(ModuleSettings.getConfigSetting(ingestModuleResource, IngestJobSettings.LAST_FILE_INGEST_FILTER_PROPERTY));
        } catch (FilesSetsManager.FilesSetsManagerException ex) {
            this.fileFilter = FilesSetsManager.getDefaultFilter();
            logger.log(Level.SEVERE, "Failed to get file filter from .properties file, default filter being used", ex); //NON-NLS
        }
    }

    private static HashSet<String> getModulesNames(String executionContext, String propertyName, String defaultSetting) {
        String ingestModuleResource = getModuleSettingsResource(executionContext);
        if (ModuleSettings.settingExists(ingestModuleResource, propertyName) == false) {
            ModuleSettings.setConfigSetting(ingestModuleResource, propertyName, defaultSetting);
        }
        HashSet<String> moduleNames = new HashSet<>();
        String modulesSetting = ModuleSettings.getConfigSetting(ingestModuleResource, propertyName);
        if (!modulesSetting.isEmpty()) {
            String[] settingNames = modulesSetting.split(", ");
            for (String name : settingNames) {
                /*
                 * Map some obsolete core ingest module names to the current
                 * core ingest module names.
                 */
                switch (name) {
                    case "Thunderbird Parser": //NON-NLS
                    case "MBox Parser": //NON-NLS
                        moduleNames.add("Email Parser"); //NON-NLS
                        break;
                    case "File Extension Mismatch Detection": //NON-NLS
                        moduleNames.add("Extension Mismatch Detector"); //NON-NLS
                        break;
                    case "EWF Verify": //NON-NLS
                    case "E01 Verify": //NON-NLS
                    case "E01 Verifier": // NON-NLS
                        moduleNames.add("Data Source Integrity"); //NON-NLS
                        break;
                    case "Archive Extractor": //NON-NLS
                        moduleNames.add("Embedded File Extractor"); //NON-NLS
                        break;
                    case "Correlation Engine": //NON-NLS
                        moduleNames.add("Central Repository"); //NON-NLS
                        break;
                    case "Exif Parser": //NON-NLS
                        moduleNames.add("Picture Analyzer"); //NON-NLS
                        break;
                    case "Drone Analyzer":
                        moduleNames.add("DJI Drone Analyzer");
                        break;
                    default:
                        moduleNames.add(name);
                }
            }
        }
        return moduleNames;
    }

    static List<String> getEnabledModules(String context) {
        return new ArrayList<>(getModulesNames(context, ENABLED_MODULES_PROPERTY, ""));
    }

    private boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) {
        return moduleSettingsFilePath.contains(PYTHON_CLASS_PROXY_PREFIX);
    }

    private IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory factory) {
        IngestModuleIngestJobSettings settings = null;
        String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
        File settingsFile = new File(moduleSettingsFilePath);
        if (settingsFile.exists()) {
            try (NbObjectInputStream in = new NbObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
                settings = (IngestModuleIngestJobSettings) in.readObject();
            } catch (IOException | ClassNotFoundException ex) {
                String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS                 
                logger.log(Level.WARNING, warning, ex);
                this.warnings.add(warning);
            }
        }
        if (settings == null) {
            settings = factory.getDefaultIngestJobSettings();
        }
        return settings;
    }

    private String getModuleSettingsFilePath(IngestModuleFactory factory) {
        String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT;
        Path path = Paths.get(this.moduleSettingsFolderPath, fileName);
        return path.toAbsolutePath().toString();
    }

    private void store() {
        HashSet<String> enabledModuleNames = new HashSet<>();
        HashSet<String> disabledModuleNames = new HashSet<>();
        for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
            saveModuleSettings(moduleTemplate.getModuleFactory(), moduleTemplate.getModuleSettings());
            String moduleName = moduleTemplate.getModuleName();
            if (moduleTemplate.isEnabled()) {
                enabledModuleNames.add(moduleName);
            } else {
                disabledModuleNames.add(moduleName);
            }
        }

        String ingestModuleResource = getModuleSettingsResource(this.executionContext);
        ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
        ModuleSettings.setConfigSetting(ingestModuleResource, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));

        ModuleSettings.setConfigSetting(ingestModuleResource, LAST_FILE_INGEST_FILTER_PROPERTY, fileFilter.getName());
    }

    private void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings) {
        String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
        try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) {
            out.writeObject(settings);
        } catch (IOException ex) {
            String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsSave.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
            logger.log(Level.SEVERE, warning, ex);
            this.warnings.add(warning);
        }
    }

    private static String makeCsvList(Collection<String> collection) {
        if (collection == null || collection.isEmpty()) {
            return "";
        }

        ArrayList<String> list = new ArrayList<>();
        list.addAll(collection);
        StringBuilder csvList = new StringBuilder();
        for (int i = 0; i < list.size() - 1; ++i) {
            csvList.append(list.get(i)).append(", ");
        }
        csvList.append(list.get(list.size() - 1));
        return csvList.toString();
    }

    public enum IngestType {

        ALL_MODULES,
        DATA_SOURCE_ONLY,
        FILES_ONLY
    }

}