23 package org.sleuthkit.autopsy.recentactivity;
25 import java.io.BufferedReader;
27 import org.openide.util.NbBundle;
31 import java.io.FileInputStream;
32 import java.io.FileNotFoundException;
33 import java.io.IOException;
34 import java.io.InputStreamReader;
35 import java.text.ParseException;
36 import java.text.SimpleDateFormat;
37 import java.util.ArrayList;
38 import java.util.List;
40 import java.util.HashSet;
41 import java.util.logging.Level;
43 import java.util.Collection;
44 import java.util.Scanner;
45 import java.util.stream.Collectors;
46 import org.openide.modules.InstalledFileLocator;
53 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
55 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
66 class ExtractIE
extends Extract {
68 private static final Logger logger = Logger.getLogger(ExtractIE.class.getName());
69 private final IngestServices services = IngestServices.getInstance();
70 private final String moduleTempResultsDir;
71 private String PASCO_LIB_PATH;
72 private final String JAVA_PATH;
73 private static final String RESOURCE_URL_PREFIX =
"res://";
74 private static final SimpleDateFormat dateFormatter =
new SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
75 private Content dataSource;
76 private IngestJobContext context;
78 ExtractIE() throws NoCurrentCaseException {
79 moduleName = NbBundle.getMessage(ExtractIE.class,
"ExtractIE.moduleName.text");
80 moduleTempResultsDir = RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(),
"IE") + File.separator +
"results";
81 JAVA_PATH = PlatformUtil.getJavaPath();
85 public void process(Content dataSource, IngestJobContext context) {
86 this.dataSource = dataSource;
87 this.context = context;
97 private void getBookmark() {
99 List<AbstractFile> favoritesFiles;
101 favoritesFiles = fileManager.
findFiles(dataSource,
"%.url",
"Favorites");
102 }
catch (TskCoreException ex) {
103 logger.log(Level.WARNING,
"Error fetching 'url' files for Internet Explorer bookmarks.", ex);
104 this.addErrorMessage(
105 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getBookmark.errMsg.errGettingBookmarks",
110 if (favoritesFiles.isEmpty()) {
111 logger.log(Level.INFO,
"Didn't find any IE bookmark files.");
116 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
117 for (AbstractFile fav : favoritesFiles) {
118 if (fav.getSize() == 0) {
122 if (context.dataSourceIngestIsCancelled()) {
126 String url = getURLFromIEBookmarkFile(fav);
128 String name = fav.getName();
129 Long datetime = fav.getCrtime();
130 String Tempdate = datetime.toString();
131 datetime = Long.valueOf(Tempdate);
132 String domain = extractDomain(url);
134 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
135 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
136 NbBundle.getMessage(
this.getClass(),
137 "ExtractIE.parentModuleName.noSpace"), url));
138 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
139 NbBundle.getMessage(
this.getClass(),
140 "ExtractIE.parentModuleName.noSpace"), name));
141 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
142 NbBundle.getMessage(
this.getClass(),
143 "ExtractIE.parentModuleName.noSpace"), datetime));
144 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
145 NbBundle.getMessage(
this.getClass(),
146 "ExtractIE.parentModuleName.noSpace"),
147 NbBundle.getMessage(
this.getClass(),
"ExtractIE.moduleName.text")));
148 if (domain != null && domain.isEmpty() ==
false) {
149 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
150 NbBundle.getMessage(
this.getClass(),
151 "ExtractIE.parentModuleName.noSpace"), domain));
154 BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, fav, bbattributes);
156 bbartifacts.add(bbart);
159 services.fireModuleDataEvent(
new ModuleDataEvent(
160 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
161 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK, bbartifacts));
164 private String getURLFromIEBookmarkFile(AbstractFile fav) {
165 BufferedReader reader =
new BufferedReader(
new InputStreamReader(
new ReadContentInputStream(fav)));
166 String line, url =
"";
168 line = reader.readLine();
169 while (null != line) {
172 if (line.startsWith(
"URL")) {
173 url = line.substring(line.indexOf(
"=") + 1);
176 line = reader.readLine();
178 }
catch (IOException ex) {
179 logger.log(Level.WARNING,
"Failed to read from content: " + fav.getName(), ex);
180 this.addErrorMessage(
181 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg", this.getName(),
183 }
catch (IndexOutOfBoundsException ex) {
184 logger.log(Level.WARNING,
"Failed while getting URL of IE bookmark. Unexpected format of the bookmark file: " + fav.getName(), ex);
185 this.addErrorMessage(
186 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg2", this.getName(),
191 }
catch (IOException ex) {
192 logger.log(Level.WARNING,
"Failed to close reader.", ex);
202 private void getCookie() {
204 List<AbstractFile> cookiesFiles;
206 cookiesFiles = fileManager.
findFiles(dataSource,
"%.txt",
"Cookies");
207 }
catch (TskCoreException ex) {
208 logger.log(Level.WARNING,
"Error getting cookie files for IE");
209 this.addErrorMessage(
210 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errGettingFile", this.getName()));
214 if (cookiesFiles.isEmpty()) {
215 logger.log(Level.INFO,
"Didn't find any IE cookies files.");
220 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
221 for (AbstractFile cookiesFile : cookiesFiles) {
222 if (context.dataSourceIngestIsCancelled()) {
225 if (cookiesFile.getSize() == 0) {
229 byte[] t =
new byte[(int) cookiesFile.getSize()];
231 final int bytesRead = cookiesFile.read(t, 0, cookiesFile.getSize());
232 }
catch (TskCoreException ex) {
233 logger.log(Level.WARNING,
"Error reading bytes of Internet Explorer cookie.", ex);
234 this.addErrorMessage(
235 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errReadingIECookie",
236 this.getName(), cookiesFile.getName()));
239 String cookieString =
new String(t);
240 String[] values = cookieString.split(
"\n");
241 String url = values.length > 2 ? values[2] :
"";
242 String value = values.length > 1 ? values[1] :
"";
243 String name = values.length > 0 ? values[0] :
"";
244 Long datetime = cookiesFile.getCrtime();
245 String tempDate = datetime.toString();
246 datetime = Long.valueOf(tempDate);
247 String domain = extractDomain(url);
249 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
250 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
251 NbBundle.getMessage(
this.getClass(),
252 "ExtractIE.parentModuleName.noSpace"), url));
253 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
254 NbBundle.getMessage(
this.getClass(),
255 "ExtractIE.parentModuleName.noSpace"), datetime));
256 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
257 NbBundle.getMessage(
this.getClass(),
258 "ExtractIE.parentModuleName.noSpace"), (name != null) ? name :
""));
259 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
260 NbBundle.getMessage(
this.getClass(),
261 "ExtractIE.parentModuleName.noSpace"), value));
262 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
263 NbBundle.getMessage(
this.getClass(),
264 "ExtractIE.parentModuleName.noSpace"),
265 NbBundle.getMessage(
this.getClass(),
"ExtractIE.moduleName.text")));
266 if (domain != null && domain.isEmpty() ==
false) {
267 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
268 NbBundle.getMessage(
this.getClass(),
269 "ExtractIE.parentModuleName.noSpace"), domain));
271 BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
273 bbartifacts.add(bbart);
276 services.fireModuleDataEvent(
new ModuleDataEvent(
277 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
278 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE, bbartifacts));
284 private void getHistory() {
285 logger.log(Level.INFO,
"Pasco results path: {0}", moduleTempResultsDir);
286 boolean foundHistory =
false;
288 final File pascoRoot = InstalledFileLocator.getDefault().locate(
"pasco2", ExtractIE.class.getPackage().getName(),
false);
289 if (pascoRoot == null) {
290 this.addErrorMessage(
291 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.unableToGetHist", this.getName()));
292 logger.log(Level.SEVERE,
"Error finding pasco program ");
296 final String pascoHome = pascoRoot.getAbsolutePath();
297 logger.log(Level.INFO,
"Pasco2 home: {0}", pascoHome);
299 PASCO_LIB_PATH = pascoHome + File.separator +
"pasco2.jar" + File.pathSeparator
300 + pascoHome + File.separator +
"*";
302 File resultsDir =
new File(moduleTempResultsDir);
307 List<AbstractFile> indexFiles;
309 indexFiles = fileManager.
findFiles(dataSource,
"index.dat");
310 }
catch (TskCoreException ex) {
311 this.addErrorMessage(NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errGettingHistFiles",
313 logger.log(Level.WARNING,
"Error fetching 'index.data' files for Internet Explorer history.");
317 if (indexFiles.isEmpty()) {
318 String msg = NbBundle.getMessage(this.getClass(),
"ExtractIE.getHistory.errMsg.noHistFiles");
319 logger.log(Level.INFO, msg);
324 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
326 String indexFileName;
327 for (AbstractFile indexFile : indexFiles) {
333 indexFileName =
"index" + Integer.toString((
int) indexFile.getId()) +
".dat";
335 temps = RAImageIngestModule.getRATempPath(currentCase,
"IE") + File.separator + indexFileName;
336 File datFile =
new File(temps);
337 if (context.dataSourceIngestIsCancelled()) {
341 ContentUtils.writeToFile(indexFile, datFile, context::dataSourceIngestIsCancelled);
342 }
catch (IOException e) {
343 logger.log(Level.WARNING,
"Error while trying to write index.dat file " + datFile.getAbsolutePath(), e);
344 this.addErrorMessage(
345 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errWriteFile", this.getName(),
346 datFile.getAbsolutePath()));
350 String filename =
"pasco2Result." + indexFile.getId() +
".txt";
351 boolean bPascProcSuccess = executePasco(temps, filename);
352 if (context.dataSourceIngestIsCancelled()) {
358 if (bPascProcSuccess) {
360 bbartifacts.addAll(parsePascoOutput(indexFile, filename).stream()
361 .filter(bbart -> bbart.getArtifactTypeID() == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID())
362 .collect(Collectors.toList()));
368 logger.log(Level.WARNING,
"pasco execution failed on: {0}",
this.getName());
369 this.addErrorMessage(
370 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errProcHist", this.getName()));
375 services.fireModuleDataEvent(
new ModuleDataEvent(
376 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
377 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
389 private boolean executePasco(String indexFilePath, String outputFileName) {
390 boolean success =
true;
392 final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName;
393 final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName +
".err";
394 logger.log(Level.INFO,
"Writing pasco results to: {0}", outputFileFullPath);
395 List<String> commandLine =
new ArrayList<>();
396 commandLine.add(JAVA_PATH);
397 commandLine.add(
"-cp");
398 commandLine.add(PASCO_LIB_PATH);
399 commandLine.add(
"isi.pasco2.Main");
400 commandLine.add(
"-T");
401 commandLine.add(
"history");
402 commandLine.add(indexFilePath);
403 ProcessBuilder processBuilder =
new ProcessBuilder(commandLine);
404 processBuilder.redirectOutput(
new File(outputFileFullPath));
405 processBuilder.redirectError(
new File(errFileFullPath));
415 ExecUtil.execute(processBuilder,
new DataSourceIngestModuleProcessTerminator(context));
417 }
catch (IOException ex) {
419 logger.log(Level.SEVERE,
"Unable to execute Pasco to process Internet Explorer web history.", ex);
433 private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName) {
435 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
436 String fnAbs = moduleTempResultsDir + File.separator + pascoOutputFileName;
438 File file =
new File(fnAbs);
439 if (file.exists() ==
false) {
440 this.addErrorMessage(
441 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(),
443 logger.log(Level.WARNING,
"Pasco Output not found: {0}", file.getPath());
449 if (file.length() == 0) {
455 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
456 }
catch (FileNotFoundException ex) {
457 this.addErrorMessage(
458 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsing", this.getName(),
460 logger.log(Level.WARNING,
"Unable to find the Pasco file at " + file.getPath(), ex);
465 Set<String> reportedUserAccounts =
new HashSet<>();
467 while (fileScanner.hasNext()) {
468 String line = fileScanner.nextLine();
469 if (!line.startsWith(
"URL")) {
473 String[] lineBuff = line.split(
"\\t");
475 if (lineBuff.length < 4) {
476 logger.log(Level.INFO,
"Found unrecognized IE history format.");
480 String actime = lineBuff[3];
481 Long ftime = (long) 0;
483 String realurl = null;
490 if (lineBuff[1].contains(
"@")) {
491 String url[] = lineBuff[1].split(
"@", 2);
496 domain = extractDomain(url[0]);
498 if (domain != null && domain.isEmpty() ==
false) {
502 realurl = lineBuff[1].trim();
509 user = user.replace(
"Visited:",
"");
510 user = user.replace(
":Host:",
"");
511 user = user.replaceAll(
"(:)(.*?)(:)",
"");
514 realurl = realurl.replace(
"Visited:",
"");
515 realurl = realurl.replaceAll(
":(.*?):",
"");
516 realurl = realurl.replace(
":Host:",
"");
517 realurl = realurl.trim();
518 domain = extractDomain(realurl);
524 realurl = lineBuff[1].trim();
525 domain = extractDomain(realurl);
528 if (!actime.isEmpty()) {
530 Long epochtime = dateFormatter.parse(actime).getTime();
531 ftime = epochtime / 1000;
532 }
catch (ParseException e) {
533 this.addErrorMessage(
534 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
536 logger.log(Level.WARNING, String.format(
"Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e);
541 BlackboardArtifact bbart = origFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
542 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
543 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
544 NbBundle.getMessage(
this.getClass(),
545 "ExtractIE.parentModuleName.noSpace"), realurl));
548 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
549 NbBundle.getMessage(
this.getClass(),
550 "ExtractIE.parentModuleName.noSpace"), ftime));
551 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
552 NbBundle.getMessage(
this.getClass(),
553 "ExtractIE.parentModuleName.noSpace"),
""));
555 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
556 NbBundle.getMessage(
this.getClass(),
557 "ExtractIE.parentModuleName.noSpace"),
558 NbBundle.getMessage(
this.getClass(),
559 "ExtractIE.moduleName.text")));
560 if (domain != null && domain.isEmpty() ==
false) {
561 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
562 NbBundle.getMessage(
this.getClass(),
563 "ExtractIE.parentModuleName.noSpace"), domain));
565 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
566 NbBundle.getMessage(
this.getClass(),
567 "ExtractIE.parentModuleName.noSpace"), user));
568 bbart.addAttributes(bbattributes);
571 this.indexArtifact(bbart);
572 bbartifacts.add(bbart);
574 if ((!user.isEmpty()) && (!reportedUserAccounts.contains(user))) {
575 BlackboardArtifact osAttr = origFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT);
576 osAttr.addAttribute(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
577 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName.noSpace"), user));
580 this.indexArtifact(osAttr);
581 bbartifacts.add(osAttr);
583 reportedUserAccounts.add(user);
585 }
catch (TskCoreException ex) {
586 logger.log(Level.SEVERE,
"Error writing Internet Explorer web history artifact to the blackboard.", ex);
601 private String extractDomain(String url) {
602 if (url == null || url.isEmpty()) {
606 if (url.toLowerCase().startsWith(RESOURCE_URL_PREFIX)) {
613 return NetworkUtils.extractDomain(url);
synchronized List< AbstractFile > findFiles(String fileName)