Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
PlasoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2018-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.plaso;
20 
21 import java.io.BufferedReader;
22 import java.io.BufferedWriter;
23 import java.io.File;
24 import java.io.FileNotFoundException;
25 import java.io.IOException;
26 import java.io.InputStreamReader;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.nio.file.Paths;
30 import java.sql.ResultSet;
31 import java.sql.SQLException;
32 import java.text.SimpleDateFormat;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.List;
36 import java.util.Locale;
37 import static java.util.Objects.nonNull;
38 import java.util.concurrent.TimeUnit;
39 import java.util.logging.Level;
40 import java.util.stream.Collectors;
41 import org.openide.modules.InstalledFileLocator;
42 import org.openide.util.Cancellable;
43 import org.openide.util.NbBundle;
57 import org.sleuthkit.datamodel.AbstractFile;
58 import org.sleuthkit.datamodel.Blackboard;
59 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
60 import org.sleuthkit.datamodel.BlackboardArtifact;
61 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TL_EVENT;
62 import org.sleuthkit.datamodel.BlackboardAttribute;
63 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME;
64 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION;
65 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TL_EVENT_TYPE;
66 import org.sleuthkit.datamodel.Content;
67 import org.sleuthkit.datamodel.Image;
68 import org.sleuthkit.datamodel.TskCoreException;
69 import org.sleuthkit.datamodel.TimelineEventType;
70 
74 public class PlasoIngestModule implements DataSourceIngestModule {
75 
76  private static final Logger logger = Logger.getLogger(PlasoIngestModule.class.getName());
77  private static final String MODULE_NAME = PlasoModuleFactory.getModuleName();
78 
79  private static final String PLASO = "plaso"; //NON-NLS
80  private static final String PLASO64 = "plaso-20180818-amd64";//NON-NLS
81  private static final String PLASO32 = "plaso-20180818-win32";//NON-NLS
82  private static final String LOG2TIMELINE_EXECUTABLE = "Log2timeline.exe";//NON-NLS
83  private static final String PSORT_EXECUTABLE = "psort.exe";//NON-NLS
84  private static final String COOKIE = "cookie";//NON-NLS
85  private static final int LOG2TIMELINE_WORKERS = 2;
86  private static final long TERMINATION_CHECK_INTERVAL = 5;
87  private static final TimeUnit TERMINATION_CHECK_INTERVAL_UNITS = TimeUnit.SECONDS;
88 
89  private File log2TimeLineExecutable;
90  private File psortExecutable;
91 
94  private Case currentCase;
96 
97  private Image image;
98  private AbstractFile previousFile = null; // cache used when looking up files in Autopsy DB
99 
101  this.settings = settings;
102  }
103 
104  @NbBundle.Messages({
105  "PlasoIngestModule.executable.not.found=Plaso Executable Not Found.",
106  "PlasoIngestModule.requires.windows=Plaso module requires windows."})
107  @Override
108  public void startUp(IngestJobContext context) throws IngestModuleException {
109  this.context = context;
110 
111  if (false == PlatformUtil.isWindowsOS()) {
112  throw new IngestModuleException(Bundle.PlasoIngestModule_requires_windows());
113  }
114 
115  try {
116  log2TimeLineExecutable = locateExecutable(LOG2TIMELINE_EXECUTABLE);
117  psortExecutable = locateExecutable(PSORT_EXECUTABLE);
118  } catch (FileNotFoundException exception) {
119  logger.log(Level.WARNING, "Plaso executable not found.", exception); //NON-NLS
120  throw new IngestModuleException(Bundle.PlasoIngestModule_executable_not_found(), exception);
121  }
122 
123  }
124 
125  @NbBundle.Messages({
126  "PlasoIngestModule.error.running.log2timeline=Error running log2timeline, see log file.",
127  "PlasoIngestModule.error.running.psort=Error running Psort, see log file.",
128  "PlasoIngestModule.error.creating.output.dir=Error creating Plaso module output directory.",
129  "PlasoIngestModule.starting.log2timeline=Starting Log2timeline",
130  "PlasoIngestModule.running.psort=Running Psort",
131  "PlasoIngestModule.log2timeline.cancelled=Log2timeline run was canceled",
132  "PlasoIngestModule.psort.cancelled=psort run was canceled",
133  "PlasoIngestModule.bad.imageFile=Cannot find image file name and path",
134  "PlasoIngestModule.completed=Plaso Processing Completed",
135  "PlasoIngestModule.has.run=Plaso",
136  "PlasoIngestModule.psort.fail=Plaso returned an error when sorting events. Results are not complete.",
137  "PlasoIngestModule.dataSource.not.an.image=Skipping non-disk image datasource"})
138  @Override
139  public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
140 
141  if (!(dataSource instanceof Image)) {
143  Bundle.PlasoIngestModule_has_run(),
144  Bundle.PlasoIngestModule_dataSource_not_an_image());
146  return ProcessResult.OK;
147  } else {
148  image = (Image) dataSource;
149 
150  statusHelper.switchToDeterminate(100);
151  currentCase = Case.getCurrentCase();
152  fileManager = currentCase.getServices().getFileManager();
153 
154  // Use Z here for timezone since the other formats can include a colon on some systems
155  String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss Z", Locale.US).format(System.currentTimeMillis());//NON-NLS
156  Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), PLASO, currentTime);
157  try {
158  Files.createDirectories(moduleOutputPath);
159  } catch (IOException ex) {
160  logger.log(Level.SEVERE, "Error creating Plaso module output directory.", ex); //NON-NLS
161  return ProcessResult.ERROR;
162  }
163 
164  // Run log2timeline
165  logger.log(Level.INFO, "Starting Plaso Run.");//NON-NLS
166  statusHelper.progress(Bundle.PlasoIngestModule_starting_log2timeline(), 0);
167  ProcessBuilder log2TimeLineCommand = buildLog2TimeLineCommand(moduleOutputPath, image);
168  try {
169  Process log2TimeLineProcess = log2TimeLineCommand.start();
170  try (BufferedReader log2TimeLineOutpout = new BufferedReader(new InputStreamReader(log2TimeLineProcess.getInputStream()))) {
171  L2TStatusProcessor statusReader = new L2TStatusProcessor(log2TimeLineOutpout, statusHelper, moduleOutputPath);
172  new Thread(statusReader, "log2timeline status reader").start(); //NON-NLS
173  ExecUtil.waitForTermination(LOG2TIMELINE_EXECUTABLE, log2TimeLineProcess, TERMINATION_CHECK_INTERVAL, TERMINATION_CHECK_INTERVAL_UNITS, new DataSourceIngestModuleProcessTerminator(context));
174  statusReader.cancel();
175  }
176 
177  if (context.dataSourceIngestIsCancelled()) {
178  logger.log(Level.INFO, "Log2timeline run was canceled"); //NON-NLS
179  return ProcessResult.OK;
180  }
181  if (Files.notExists(moduleOutputPath.resolve(PLASO))) {
182  logger.log(Level.WARNING, "Error running log2timeline: there was no storage file."); //NON-NLS
183  return ProcessResult.ERROR;
184  }
185 
186  // sort the output
187  statusHelper.progress(Bundle.PlasoIngestModule_running_psort(), 33);
188  ProcessBuilder psortCommand = buildPsortCommand(moduleOutputPath);
189  int result = ExecUtil.execute(psortCommand, new DataSourceIngestModuleProcessTerminator(context));
190  if (result != 0) {
191  logger.log(Level.SEVERE, String.format("Error running Psort, error code returned %d", result)); //NON-NLS
192  MessageNotifyUtil.Notify.error(MODULE_NAME, Bundle.PlasoIngestModule_psort_fail());
193  return ProcessResult.ERROR;
194  }
195 
196  if (context.dataSourceIngestIsCancelled()) {
197  logger.log(Level.INFO, "psort run was canceled"); //NON-NLS
198  return ProcessResult.OK;
199  }
200  Path plasoFile = moduleOutputPath.resolve("plasodb.db3"); //NON-NLS
201  if (Files.notExists(plasoFile)) {
202  logger.log(Level.SEVERE, "Error running Psort: there was no sqlite db file."); //NON-NLS
203  return ProcessResult.ERROR;
204  }
205 
206  // parse the output and make artifacts
207  createPlasoArtifacts(plasoFile.toString(), statusHelper);
208 
209  } catch (IOException ex) {
210  logger.log(Level.SEVERE, "Error running Plaso.", ex);//NON-NLS
211  return ProcessResult.ERROR;
212  }
213 
215  Bundle.PlasoIngestModule_has_run(),
216  Bundle.PlasoIngestModule_completed());
218  return ProcessResult.OK;
219  }
220  }
221 
222  private ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image) {
223  //make a csv list of disabled parsers.
224  String parsersString = settings.getParsers().entrySet().stream()
225  .filter(entry -> entry.getValue() == false)
226  .map(entry -> "!" + entry.getKey()) // '!' prepended to parsername disables it. //NON-NLS
227  .collect(Collectors.joining(","));//NON-NLS
228 
229  ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
230  "\"" + log2TimeLineExecutable + "\"", //NON-NLS
231  "--vss-stores", "all", //NON-NLS
232  "-z", image.getTimeZone(), //NON-NLS
233  "--partitions", "all", //NON-NLS
234  "--hasher_file_size_limit", "1", //NON-NLS
235  "--hashers", "none", //NON-NLS
236  "--parsers", "\"" + parsersString + "\"",//NON-NLS
237  "--no_dependencies_check", //NON-NLS
238  "--workers", String.valueOf(LOG2TIMELINE_WORKERS),//NON-NLS
239  moduleOutputPath.resolve(PLASO).toString(),
240  image.getPaths()[0]
241  );
242  processBuilder.redirectError(moduleOutputPath.resolve("log2timeline_err.txt").toFile()); //NON-NLS
243  return processBuilder;
244  }
245 
246  static private ProcessBuilder buildProcessWithRunAsInvoker(String... commandLine) {
247  ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
248  /*
249  * Add an environment variable to force log2timeline/psort to run with
250  * the same permissions Autopsy uses.
251  */
252  processBuilder.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS
253  return processBuilder;
254  }
255 
256  private ProcessBuilder buildPsortCommand(Path moduleOutputPath) {
257  ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
258  "\"" + psortExecutable + "\"", //NON-NLS
259  "-o", "4n6time_sqlite", //NON-NLS
260  "-w", moduleOutputPath.resolve("plasodb.db3").toString(), //NON-NLS
261  moduleOutputPath.resolve(PLASO).toString()
262  );
263 
264  processBuilder.redirectOutput(moduleOutputPath.resolve("psort_output.txt").toFile()); //NON-NLS
265  processBuilder.redirectError(moduleOutputPath.resolve("psort_err.txt").toFile()); //NON-NLS
266  return processBuilder;
267  }
268 
269  private static File locateExecutable(String executableName) throws FileNotFoundException {
270  String architectureFolder = PlatformUtil.is64BitOS() ? PLASO64 : PLASO32;
271  String executableToFindName = Paths.get(PLASO, architectureFolder, executableName).toString();
272 
273  File exeFile = InstalledFileLocator.getDefault().locate(executableToFindName, PlasoIngestModule.class.getPackage().getName(), false);
274  if (null == exeFile || exeFile.canExecute() == false) {
275  throw new FileNotFoundException(executableName + " executable not found.");
276  }
277  return exeFile;
278  }
279 
280  @NbBundle.Messages({
281  "PlasoIngestModule.exception.posting.artifact=Exception Posting artifact.",
282  "PlasoIngestModule.event.datetime=Event Date Time",
283  "PlasoIngestModule.event.description=Event Description",
284  "PlasoIngestModule.create.artifacts.cancelled=Cancelled Plaso Artifact Creation ",
285  "# {0} - file that events are from",
286  "PlasoIngestModule.artifact.progress=Adding events to case: {0}",
287  "PlasoIngestModule.info.empty.database=Plaso database was empty.",})
288  private void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper) {
289  Blackboard blackboard = currentCase.getSleuthkitCase().getBlackboard();
290 
291  String sqlStatement = "SELECT substr(filename,1) AS filename, "
292  + " strftime('%s', datetime) AS epoch_date, "
293  + " description, "
294  + " source, "
295  + " type, "
296  + " sourcetype "
297  + " FROM log2timeline "
298  + " WHERE source NOT IN ('FILE', "
299  + " 'WEBHIST') " // bad dates and duplicates with what we have.
300  + " AND sourcetype NOT IN ('UNKNOWN', "
301  + " 'PE Import Time');"; // lots of bad dates //NON-NLS
302 
303  try (SQLiteDBConnect tempdbconnect = new SQLiteDBConnect("org.sqlite.JDBC", "jdbc:sqlite:" + plasoDb); //NON-NLS
304  ResultSet resultSet = tempdbconnect.executeQry(sqlStatement)) {
305 
306  boolean dbHasData = false;
307 
308  while (resultSet.next()) {
309  dbHasData = true;
310 
311  if (context.dataSourceIngestIsCancelled()) {
312  logger.log(Level.INFO, "Cancelled Plaso Artifact Creation."); //NON-NLS
313  return;
314  }
315 
316  String currentFileName = resultSet.getString("filename"); //NON-NLS
317  statusHelper.progress(Bundle.PlasoIngestModule_artifact_progress(currentFileName), 66);
318  Content resolvedFile = getAbstractFile(currentFileName);
319  if (resolvedFile == null) {
320  logger.log(Level.INFO, "File {0} from Plaso output not found in case. Associating it with the data source instead.", currentFileName);//NON-NLS
321  resolvedFile = image;
322  }
323 
324  String description = resultSet.getString("description");
325  TimelineEventType eventType = findEventSubtype(currentFileName, resultSet);
326 
327  // If the description is empty use the event type display name
328  // as the description.
329  if (description == null || description.isEmpty()) {
330  if (eventType != TimelineEventType.STANDARD_ARTIFACT_CATCH_ALL) {
331  description = eventType.getDisplayName();
332  } else {
333  continue;
334  }
335  }
336 
337  Collection<BlackboardAttribute> bbattributes = Arrays.asList(
338  new BlackboardAttribute(
339  TSK_DATETIME, MODULE_NAME,
340  resultSet.getLong("epoch_date")), //NON-NLS
341  new BlackboardAttribute(
342  TSK_DESCRIPTION, MODULE_NAME,
343  description),//NON-NLS
344  new BlackboardAttribute(
345  TSK_TL_EVENT_TYPE, MODULE_NAME,
346  eventType.getTypeID()));
347 
348  try {
349  BlackboardArtifact bbart = resolvedFile.newDataArtifact(new BlackboardArtifact.Type(TSK_TL_EVENT), bbattributes);
350  try {
351  /*
352  * Post the artifact which will index the artifact for
353  * keyword search, and fire an event to notify UI of
354  * this new artifact
355  */
356  blackboard.postArtifact(bbart, MODULE_NAME, context.getJobId());
357  } catch (BlackboardException ex) {
358  logger.log(Level.SEVERE, "Error Posting Artifact.", ex);//NON-NLS
359  }
360  } catch (TskCoreException ex) {
361  logger.log(Level.SEVERE, "Exception Adding Artifact.", ex);//NON-NLS
362  }
363  }
364 
365  // Check if there is data the db
366  if (!dbHasData) {
367  logger.log(Level.INFO, String.format("PlasoDB was empty: %s", plasoDb));
368  MessageNotifyUtil.Notify.info(MODULE_NAME, Bundle.PlasoIngestModule_info_empty_database());
369  }
370  } catch (SQLException ex) {
371  logger.log(Level.SEVERE, "Error while trying to read into a sqlite db.", ex);//NON-NLS
372  }
373  }
374 
375  private AbstractFile getAbstractFile(String file) {
376 
377  Path path = Paths.get(file);
378  String fileName = path.getFileName().toString();
379  String filePath = path.getParent().toString().replaceAll("\\\\", "/");//NON-NLS
380  if (filePath.endsWith("/") == false) {//NON-NLS
381  filePath += "/";//NON-NLS
382  }
383 
384  // check the cached file
385  //TODO: would we reduce 'cache misses' if we retrieved the events sorted by file? Is that overhead worth it?
386  if (previousFile != null
387  && previousFile.getName().equalsIgnoreCase(fileName)
388  && previousFile.getParentPath().equalsIgnoreCase(filePath)) {
389  return previousFile;
390 
391  }
392  try {
393  List<AbstractFile> abstractFiles = fileManager.findFiles(fileName, filePath);
394  if (abstractFiles.size() == 1) {// TODO: why do we bother with this check. also we don't cache the file...
395  return abstractFiles.get(0);
396  }
397  for (AbstractFile resolvedFile : abstractFiles) {
398  // double check its an exact match
399  if (filePath.equalsIgnoreCase(resolvedFile.getParentPath())) {
400  // cache it for next time
401  previousFile = resolvedFile;
402  return resolvedFile;
403  }
404  }
405  } catch (TskCoreException ex) {
406  logger.log(Level.SEVERE, "Exception finding file.", ex);
407  }
408  return null;
409  }
410 
422  private TimelineEventType findEventSubtype(String fileName, ResultSet row) throws SQLException {
423  switch (row.getString("source")) {
424  case "WEBHIST": //These shouldn't actually be present, but keeping the logic just in case...
425  if (fileName.toLowerCase().contains(COOKIE)
426  || row.getString("type").toLowerCase().contains(COOKIE)) {//NON-NLS
427 
428  return TimelineEventType.WEB_COOKIE;
429  } else {
430  return TimelineEventType.WEB_HISTORY;
431  }
432  case "EVT":
433  case "LOG":
434  return TimelineEventType.LOG_ENTRY;
435  case "REG":
436  switch (row.getString("sourcetype").toLowerCase()) {//NON-NLS
437  case "unknown : usb entries":
438  case "unknown : usbstor entries":
439  return TimelineEventType.DEVICES_ATTACHED;
440  default:
441  return TimelineEventType.REGISTRY;
442  }
443  default:
444  return TimelineEventType.STANDARD_ARTIFACT_CATCH_ALL;
445  }
446  }
447 
453  private static class L2TStatusProcessor implements Runnable, Cancellable {
454 
455  private final BufferedReader log2TimeLineOutpout;
457  volatile private boolean cancelled = false;
458  private final Path outputPath;
459 
460  private L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath) throws IOException {
461  this.log2TimeLineOutpout = log2TimeLineOutpout;
462  this.statusHelper = statusHelper;
463  this.outputPath = outputPath;
464  }
465 
466  @Override
467  public void run() {
468  try (BufferedWriter writer = Files.newBufferedWriter(outputPath.resolve("log2timeline_output.txt"));) {//NON-NLS
469  String line = log2TimeLineOutpout.readLine();
470  while (cancelled == false && nonNull(line)) {
471  statusHelper.progress(line);
472  writer.write(line);
473  writer.newLine();
474  line = log2TimeLineOutpout.readLine();
475  }
476  writer.flush();
477  } catch (IOException ex) {
478  logger.log(Level.WARNING, "Error reading log2timeline output stream.", ex);//NON-NLS
479  }
480  }
481 
482  @Override
483  public boolean cancel() {
484  cancelled = true;
485  return true;
486  }
487  }
488 }
static int execute(ProcessBuilder processBuilder)
Definition: ExecUtil.java:172
List< AbstractFile > findFiles(String fileName)
L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath)
static ProcessBuilder buildProcessWithRunAsInvoker(String...commandLine)
static int waitForTermination(String processName, Process process, long terminationCheckInterval, TimeUnit units, ProcessTerminator terminator)
Definition: ExecUtil.java:264
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static void info(String title, String message)
ProcessBuilder buildPsortCommand(Path moduleOutputPath)
void postMessage(final IngestMessage message)
static void error(String title, String message)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper)
ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image)
TimelineEventType findEventSubtype(String fileName, ResultSet row)
void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper)
static synchronized IngestServices getInstance()

Copyright © 2012-2022 Basis Technology. Generated on: Mon Sep 26 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.