PlasoIngestModule.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2018-2019 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.modules.plaso;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import static java.util.Objects.nonNull;
import java.util.logging.Level;
import java.util.stream.Collectors;
import org.openide.modules.InstalledFileLocator;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.ExecUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModule;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.Blackboard.BlackboardException;
import org.sleuthkit.datamodel.BlackboardArtifact;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TL_EVENT;
import org.sleuthkit.datamodel.BlackboardAttribute;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TL_EVENT_TYPE;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TimelineEventType;

public class PlasoIngestModule implements DataSourceIngestModule {

    private static final Logger logger = Logger.getLogger(PlasoIngestModule.class.getName());
    private static final String MODULE_NAME = PlasoModuleFactory.getModuleName();

    private static final String PLASO = "plaso"; //NON-NLS
    private static final String PLASO64 = "plaso-20180818-amd64";//NON-NLS
    private static final String PLASO32 = "plaso-20180818-win32";//NON-NLS
    private static final String LOG2TIMELINE_EXECUTABLE = "Log2timeline.exe";//NON-NLS
    private static final String PSORT_EXECUTABLE = "psort.exe";//NON-NLS
    private static final String COOKIE = "cookie";//NON-NLS
    private static final int LOG2TIMELINE_WORKERS = 2;

    private File log2TimeLineExecutable;
    private File psortExecutable;

    private final PlasoModuleSettings settings;
    private IngestJobContext context;
    private Case currentCase;
    private FileManager fileManager;

    private Image image;
    private AbstractFile previousFile = null; // cache used when looking up files in Autopsy DB

    PlasoIngestModule(PlasoModuleSettings settings) {
        this.settings = settings;
    }

    @NbBundle.Messages({
        "PlasoIngestModule.executable.not.found=Plaso Executable Not Found.",
        "PlasoIngestModule.requires.windows=Plaso module requires windows.",
        "PlasoIngestModule.dataSource.not.an.image=Datasource is not an Image."})
    @Override
    public void startUp(IngestJobContext context) throws IngestModuleException {
        this.context = context;

        if (false == PlatformUtil.isWindowsOS()) {
            throw new IngestModuleException(Bundle.PlasoIngestModule_requires_windows());
        }

        try {
            log2TimeLineExecutable = locateExecutable(LOG2TIMELINE_EXECUTABLE);
            psortExecutable = locateExecutable(PSORT_EXECUTABLE);
        } catch (FileNotFoundException exception) {
            logger.log(Level.WARNING, "Plaso executable not found.", exception); //NON-NLS
            throw new IngestModuleException(Bundle.PlasoIngestModule_executable_not_found(), exception);
        }

        Content dataSource = context.getDataSource();
        if (!(dataSource instanceof Image)) {
            throw new IngestModuleException(Bundle.PlasoIngestModule_dataSource_not_an_image());
        }
        image = (Image) dataSource;
    }

    @NbBundle.Messages({
        "PlasoIngestModule.error.running.log2timeline=Error running log2timeline, see log file.",
        "PlasoIngestModule.error.running.psort=Error running Psort, see log file.",
        "PlasoIngestModule.error.creating.output.dir=Error creating Plaso module output directory.",
        "PlasoIngestModule.starting.log2timeline=Starting Log2timeline",
        "PlasoIngestModule.running.psort=Running Psort",
        "PlasoIngestModule.log2timeline.cancelled=Log2timeline run was canceled",
        "PlasoIngestModule.psort.cancelled=psort run was canceled",
        "PlasoIngestModule.bad.imageFile=Cannot find image file name and path",
        "PlasoIngestModule.completed=Plaso Processing Completed",
        "PlasoIngestModule.has.run=Plaso Plugin has been run.",
        "PlasoIngestModule.psort.fail=Plaso returned an error when sorting events.  Results are not complete."})
    @Override
    public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
        assert dataSource.equals(image);

        statusHelper.switchToDeterminate(100);
        currentCase = Case.getCurrentCase();
        fileManager = currentCase.getServices().getFileManager();

        String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS
        Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), PLASO, currentTime);
        try {
            Files.createDirectories(moduleOutputPath);
        } catch (IOException ex) {
            logger.log(Level.SEVERE, "Error creating Plaso module output directory.", ex); //NON-NLS
            return ProcessResult.ERROR;
        }

        // Run log2timeline
        logger.log(Level.INFO, "Starting Plaso Run.");//NON-NLS
        statusHelper.progress(Bundle.PlasoIngestModule_starting_log2timeline(), 0);
        ProcessBuilder log2TimeLineCommand = buildLog2TimeLineCommand(moduleOutputPath, image);
        try {
            Process log2TimeLineProcess = log2TimeLineCommand.start();
            try (BufferedReader log2TimeLineOutpout = new BufferedReader(new InputStreamReader(log2TimeLineProcess.getInputStream()))) {
                L2TStatusProcessor statusReader = new L2TStatusProcessor(log2TimeLineOutpout, statusHelper, moduleOutputPath);
                new Thread(statusReader, "log2timeline status reader").start();  //NON-NLS
                ExecUtil.waitForTermination(LOG2TIMELINE_EXECUTABLE, log2TimeLineProcess, new DataSourceIngestModuleProcessTerminator(context));
                statusReader.cancel();
            }

            if (context.dataSourceIngestIsCancelled()) {
                logger.log(Level.INFO, "Log2timeline run was canceled"); //NON-NLS
                return ProcessResult.OK;
            }
            if (Files.notExists(moduleOutputPath.resolve(PLASO))) {
                logger.log(Level.WARNING, "Error running log2timeline: there was no storage file."); //NON-NLS
                return ProcessResult.ERROR;
            }

            // sort the output
            statusHelper.progress(Bundle.PlasoIngestModule_running_psort(), 33);
            ProcessBuilder psortCommand = buildPsortCommand(moduleOutputPath);
            int result = ExecUtil.execute(psortCommand, new DataSourceIngestModuleProcessTerminator(context));
             if (result != 0) {
                 logger.log(Level.SEVERE, String.format("Error running Psort, error code returned %d", result)); //NON-NLS
                 MessageNotifyUtil.Notify.error(MODULE_NAME, Bundle.PlasoIngestModule_psort_fail());
                 return ProcessResult.ERROR;
             }

            if (context.dataSourceIngestIsCancelled()) {
                logger.log(Level.INFO, "psort run was canceled"); //NON-NLS
                return ProcessResult.OK;
            }
            Path plasoFile = moduleOutputPath.resolve("plasodb.db3");  //NON-NLS
            if (Files.notExists(plasoFile)) {
                logger.log(Level.SEVERE, "Error running Psort: there was no sqlite db file."); //NON-NLS
                return ProcessResult.ERROR;
            }

            // parse the output and make artifacts
            createPlasoArtifacts(plasoFile.toString(), statusHelper);

        } catch (IOException ex) {
            logger.log(Level.SEVERE, "Error running Plaso.", ex);//NON-NLS
            return ProcessResult.ERROR;
        }

        IngestMessage message = IngestMessage.createMessage(IngestMessage.MessageType.DATA,
                Bundle.PlasoIngestModule_has_run(),
                Bundle.PlasoIngestModule_completed());
        IngestServices.getInstance().postMessage(message);
        return ProcessResult.OK;
    }

    private ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image) {
        //make a csv list of disabled parsers.
        String parsersString = settings.getParsers().entrySet().stream()
                .filter(entry -> entry.getValue() == false)
                .map(entry -> "!" + entry.getKey()) // '!' prepended to parsername disables it. //NON-NLS
                .collect(Collectors.joining(","));//NON-NLS

        ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
                "\"" + log2TimeLineExecutable + "\"", //NON-NLS
                "--vss-stores", "all", //NON-NLS
                "-z", image.getTimeZone(), //NON-NLS
                "--partitions", "all", //NON-NLS
                "--hasher_file_size_limit", "1", //NON-NLS
                "--hashers", "none", //NON-NLS
                "--parsers", "\"" + parsersString + "\"",//NON-NLS
                "--no_dependencies_check", //NON-NLS
                "--workers", String.valueOf(LOG2TIMELINE_WORKERS),//NON-NLS
                moduleOutputPath.resolve(PLASO).toString(),
                image.getPaths()[0]
        );
        processBuilder.redirectError(moduleOutputPath.resolve("log2timeline_err.txt").toFile());  //NON-NLS
        return processBuilder;
    }

    static private ProcessBuilder buildProcessWithRunAsInvoker(String... commandLine) {
        ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
        /* Add an environment variable to force log2timeline/psort to run with
         * the same permissions Autopsy uses. */
        processBuilder.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS
        return processBuilder;
    }

    private ProcessBuilder buildPsortCommand(Path moduleOutputPath) {
        ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
                "\"" + psortExecutable + "\"", //NON-NLS
                "-o", "4n6time_sqlite", //NON-NLS
                "-w", moduleOutputPath.resolve("plasodb.db3").toString(), //NON-NLS
                moduleOutputPath.resolve(PLASO).toString()
        );

        processBuilder.redirectOutput(moduleOutputPath.resolve("psort_output.txt").toFile()); //NON-NLS
        processBuilder.redirectError(moduleOutputPath.resolve("psort_err.txt").toFile());  //NON-NLS
        return processBuilder;
    }

    private static File locateExecutable(String executableName) throws FileNotFoundException {
        String architectureFolder = PlatformUtil.is64BitOS() ? PLASO64 : PLASO32;
        String executableToFindName = Paths.get(PLASO, architectureFolder, executableName).toString();

        File exeFile = InstalledFileLocator.getDefault().locate(executableToFindName, PlasoIngestModule.class.getPackage().getName(), false);
        if (null == exeFile || exeFile.canExecute() == false) {
            throw new FileNotFoundException(executableName + " executable not found.");
        }
        return exeFile;
    }

    @NbBundle.Messages({
        "PlasoIngestModule.exception.posting.artifact=Exception Posting artifact.",
        "PlasoIngestModule.event.datetime=Event Date Time",
        "PlasoIngestModule.event.description=Event Description",
        "PlasoIngestModule.create.artifacts.cancelled=Cancelled Plaso Artifact Creation ",
        "# {0} - file that events are from",
        "PlasoIngestModule.artifact.progress=Adding events to case: {0}",
        "PlasoIngestModule.info.empty.database=Plaso database was empty.",
    })
    private void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper) {
        Blackboard blackboard = currentCase.getSleuthkitCase().getBlackboard();

        String sqlStatement = "SELECT substr(filename,1) AS  filename, "
                              + "   strftime('%s', datetime) AS epoch_date, "
                              + "   description, "
                              + "   source, "
                              + "   type, "
                              + "   sourcetype "
                              + " FROM log2timeline "
                              + " WHERE source NOT IN ('FILE', "
                              + "                       'WEBHIST') " // bad dates and duplicates with what we have.
                              + "   AND sourcetype NOT IN ('UNKNOWN', "
                              + "                          'PE Import Time');"; // lots of bad dates //NON-NLS

        try (SQLiteDBConnect tempdbconnect = new SQLiteDBConnect("org.sqlite.JDBC", "jdbc:sqlite:" + plasoDb); //NON-NLS
                ResultSet resultSet = tempdbconnect.executeQry(sqlStatement)) {
            
            boolean dbHasData = false;
            
            while (resultSet.next()) {
                dbHasData = true;
                
                if (context.dataSourceIngestIsCancelled()) {
                    logger.log(Level.INFO, "Cancelled Plaso Artifact Creation."); //NON-NLS
                    return;
                }

                String currentFileName = resultSet.getString("filename"); //NON-NLS
                statusHelper.progress(Bundle.PlasoIngestModule_artifact_progress(currentFileName), 66);
                Content resolvedFile = getAbstractFile(currentFileName);
                if (resolvedFile == null) {
                    logger.log(Level.INFO, "File {0} from Plaso output not found in case.  Associating it with the data source instead.", currentFileName);//NON-NLS
                    resolvedFile = image;
                }
                
                String description = resultSet.getString("description");
                TimelineEventType eventType = findEventSubtype(currentFileName, resultSet);
                
                // If the description is empty use the event type display name
                // as the description.
                if ( description == null || description.isEmpty() ) {
                    if (eventType != TimelineEventType.OTHER) {
                        description = eventType.getDisplayName();
                    } else {
                        continue;
                    }
                }
               
                Collection<BlackboardAttribute> bbattributes = Arrays.asList(
                        new BlackboardAttribute(
                                TSK_DATETIME, MODULE_NAME,
                                resultSet.getLong("epoch_date")), //NON-NLS
                        new BlackboardAttribute(
                                TSK_DESCRIPTION, MODULE_NAME,
                                description),//NON-NLS
                        new BlackboardAttribute(
                                TSK_TL_EVENT_TYPE, MODULE_NAME,
                                eventType.getTypeID()));
                
                try {
                    BlackboardArtifact bbart = resolvedFile.newArtifact(TSK_TL_EVENT);
                    bbart.addAttributes(bbattributes);
                    try {
                        /* Post the artifact which will index the artifact for
                         * keyword search, and fire an event to notify UI of
                         * this new artifact */
                        blackboard.postArtifact(bbart, MODULE_NAME);
                    } catch (BlackboardException ex) {
                        logger.log(Level.SEVERE, "Error Posting Artifact.", ex);//NON-NLS
                    }
                } catch (TskCoreException ex) {
                    logger.log(Level.SEVERE, "Exception Adding Artifact.", ex);//NON-NLS
                }
            }
            
            // Check if there is data the db
            if( !dbHasData ) {
                logger.log(Level.INFO, String.format("PlasoDB was empty: %s", plasoDb));
                MessageNotifyUtil.Notify.info(MODULE_NAME, Bundle.PlasoIngestModule_info_empty_database());
            } 
        } catch (SQLException ex) {
            logger.log(Level.SEVERE, "Error while trying to read into a sqlite db.", ex);//NON-NLS
        }
    }

    private AbstractFile getAbstractFile(String file) {

        Path path = Paths.get(file);
        String fileName = path.getFileName().toString();
        String filePath = path.getParent().toString().replaceAll("\\\\", "/");//NON-NLS
        if (filePath.endsWith("/") == false) {//NON-NLS
            filePath += "/";//NON-NLS
        }

        // check the cached file
        //TODO: would we reduce 'cache misses' if we retrieved the events sorted by file?  Is that overhead worth it?
        if (previousFile != null
            && previousFile.getName().equalsIgnoreCase(fileName)
            && previousFile.getParentPath().equalsIgnoreCase(filePath)) {
            return previousFile;

        }
        try {
            List<AbstractFile> abstractFiles = fileManager.findFiles(fileName, filePath);
            if (abstractFiles.size() == 1) {// TODO: why do we bother with this check.  also we don't cache the file...
                return abstractFiles.get(0);
            }
            for (AbstractFile resolvedFile : abstractFiles) {
                // double check its an exact match
                if (filePath.equalsIgnoreCase(resolvedFile.getParentPath())) {
                    // cache it for next time
                    previousFile = resolvedFile;
                    return resolvedFile;
                }
            }
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Exception finding file.", ex);
        }
        return null;
    }

    private TimelineEventType findEventSubtype(String fileName, ResultSet row) throws SQLException {
        switch (row.getString("source")) {
            case "WEBHIST":  //These shouldn't actually be present, but keeping the logic just in case...
                if (fileName.toLowerCase().contains(COOKIE)
                    || row.getString("type").toLowerCase().contains(COOKIE)) {//NON-NLS

                    return TimelineEventType.WEB_COOKIE;
                } else {
                    return TimelineEventType.WEB_HISTORY;
                }
            case "EVT":
            case "LOG":
                return TimelineEventType.LOG_ENTRY;
            case "REG":
                switch (row.getString("sourcetype").toLowerCase()) {//NON-NLS
                    case "unknown : usb entries":
                    case "unknown : usbstor entries":
                        return TimelineEventType.DEVICES_ATTACHED;
                    default:
                        return TimelineEventType.REGISTRY;
                }
            default:
                return TimelineEventType.OTHER;
        }
    }

    private static class L2TStatusProcessor implements Runnable, Cancellable {

        private final BufferedReader log2TimeLineOutpout;
        private final DataSourceIngestModuleProgress statusHelper;
        volatile private boolean cancelled = false;
        private final Path outputPath;

        private L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath) throws IOException {
            this.log2TimeLineOutpout = log2TimeLineOutpout;
            this.statusHelper = statusHelper;
            this.outputPath = outputPath;
        }

        @Override
        public void run() {
            try (BufferedWriter writer = Files.newBufferedWriter(outputPath.resolve("log2timeline_output.txt"));) {//NON-NLS
                String line = log2TimeLineOutpout.readLine();
                while (cancelled == false && nonNull(line)) {
                    statusHelper.progress(line);
                    writer.write(line);
                    writer.newLine();
                    line = log2TimeLineOutpout.readLine();
                }
                writer.flush();
            } catch (IOException ex) {
                logger.log(Level.WARNING, "Error reading log2timeline output stream.", ex);//NON-NLS
            }
        }

        @Override
        public boolean cancel() {
            cancelled = true;
            return true;
        }
    }
}