23 package org.sleuthkit.autopsy.recentactivity;
25 import java.io.BufferedReader;
26 import org.openide.util.NbBundle;
30 import java.io.FileInputStream;
31 import java.io.FileNotFoundException;
32 import java.io.IOException;
33 import java.io.InputStreamReader;
34 import java.text.ParseException;
35 import java.text.SimpleDateFormat;
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.logging.Level;
40 import java.util.Collection;
41 import java.util.Scanner;
42 import java.util.stream.Collectors;
43 import org.openide.modules.InstalledFileLocator;
44 import org.openide.util.NbBundle.Messages;
51 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
53 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
65 class ExtractIE
extends Extract {
67 private static final Logger logger = Logger.getLogger(ExtractIE.class.getName());
68 private final IngestServices services = IngestServices.getInstance();
69 private final String moduleTempResultsDir;
70 private String PASCO_LIB_PATH;
71 private final String JAVA_PATH;
72 private static final String RESOURCE_URL_PREFIX =
"res://";
73 private static final SimpleDateFormat dateFormatter =
new SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
74 private Content dataSource;
75 private IngestJobContext context;
78 "Progress_Message_IE_History=IE History",
79 "Progress_Message_IE_Bookmarks=IE Bookmarks",
80 "Progress_Message_IE_Cookies=IE Cookies",
81 "Progress_Message_IE_Downloads=IE Downloads",
82 "Progress_Message_IE_FormHistory=IE Form History",
83 "Progress_Message_IE_AutoFill=IE Auto Fill",
84 "Progress_Message_IE_Logins=IE Logins",
87 ExtractIE() throws NoCurrentCaseException {
88 moduleName = NbBundle.getMessage(ExtractIE.class,
"ExtractIE.moduleName.text");
89 moduleTempResultsDir = RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(),
"IE") + File.separator +
"results";
90 JAVA_PATH = PlatformUtil.getJavaPath();
94 public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
95 this.dataSource = dataSource;
96 this.context = context;
99 progressBar.progress(Bundle.Progress_Message_IE_Bookmarks());
102 progressBar.progress(Bundle.Progress_Message_IE_Cookies());
105 progressBar.progress(Bundle.Progress_Message_IE_History());
112 private void getBookmark() {
114 List<AbstractFile> favoritesFiles;
116 favoritesFiles = fileManager.
findFiles(dataSource,
"%.url",
"Favorites");
117 }
catch (TskCoreException ex) {
118 logger.log(Level.WARNING,
"Error fetching 'url' files for Internet Explorer bookmarks.", ex);
119 this.addErrorMessage(
120 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getBookmark.errMsg.errGettingBookmarks",
125 if (favoritesFiles.isEmpty()) {
126 logger.log(Level.INFO,
"Didn't find any IE bookmark files.");
131 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
132 for (AbstractFile fav : favoritesFiles) {
133 if (fav.getSize() == 0) {
137 if (context.dataSourceIngestIsCancelled()) {
141 String url = getURLFromIEBookmarkFile(fav);
143 String name = fav.getName();
144 Long datetime = fav.getCrtime();
145 String Tempdate = datetime.toString();
146 datetime = Long.valueOf(Tempdate);
147 String domain = extractDomain(url);
149 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
150 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
151 RecentActivityExtracterModuleFactory.getModuleName(), url));
152 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
153 RecentActivityExtracterModuleFactory.getModuleName(), name));
154 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
155 RecentActivityExtracterModuleFactory.getModuleName(), datetime));
156 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
157 RecentActivityExtracterModuleFactory.getModuleName(),
158 NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
159 if (domain != null && domain.isEmpty() ==
false) {
160 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
161 RecentActivityExtracterModuleFactory.getModuleName(), domain));
164 BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, fav, bbattributes);
166 bbartifacts.add(bbart);
169 services.fireModuleDataEvent(
new ModuleDataEvent(
170 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
171 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK, bbartifacts));
174 private String getURLFromIEBookmarkFile(AbstractFile fav) {
175 BufferedReader reader =
new BufferedReader(
new InputStreamReader(
new ReadContentInputStream(fav)));
176 String line, url =
"";
178 line = reader.readLine();
179 while (null != line) {
182 if (line.startsWith(
"URL")) {
183 url = line.substring(line.indexOf(
"=") + 1);
186 line = reader.readLine();
188 }
catch (IOException ex) {
189 logger.log(Level.WARNING,
"Failed to read from content: " + fav.getName(), ex);
190 this.addErrorMessage(
191 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg", this.getName(),
193 }
catch (IndexOutOfBoundsException ex) {
194 logger.log(Level.WARNING,
"Failed while getting URL of IE bookmark. Unexpected format of the bookmark file: " + fav.getName(), ex);
195 this.addErrorMessage(
196 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg2", this.getName(),
201 }
catch (IOException ex) {
202 logger.log(Level.WARNING,
"Failed to close reader.", ex);
212 private void getCookie() {
214 List<AbstractFile> cookiesFiles;
216 cookiesFiles = fileManager.
findFiles(dataSource,
"%.txt",
"Cookies");
217 }
catch (TskCoreException ex) {
218 logger.log(Level.WARNING,
"Error getting cookie files for IE");
219 this.addErrorMessage(
220 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errGettingFile", this.getName()));
224 if (cookiesFiles.isEmpty()) {
225 logger.log(Level.INFO,
"Didn't find any IE cookies files.");
230 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
231 for (AbstractFile cookiesFile : cookiesFiles) {
232 if (context.dataSourceIngestIsCancelled()) {
235 if (cookiesFile.getSize() == 0) {
239 byte[] t =
new byte[(int) cookiesFile.getSize()];
241 final int bytesRead = cookiesFile.read(t, 0, cookiesFile.getSize());
242 }
catch (TskCoreException ex) {
243 logger.log(Level.WARNING,
"Error reading bytes of Internet Explorer cookie.", ex);
244 this.addErrorMessage(
245 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errReadingIECookie",
246 this.getName(), cookiesFile.getName()));
249 String cookieString =
new String(t);
250 String[] values = cookieString.split(
"\n");
251 String url = values.length > 2 ? values[2] :
"";
252 String value = values.length > 1 ? values[1] :
"";
253 String name = values.length > 0 ? values[0] :
"";
254 Long datetime = cookiesFile.getCrtime();
255 String tempDate = datetime.toString();
256 datetime = Long.valueOf(tempDate);
257 String domain = extractDomain(url);
259 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
260 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
261 RecentActivityExtracterModuleFactory.getModuleName(), url));
262 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
263 RecentActivityExtracterModuleFactory.getModuleName(), datetime));
264 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
265 RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name :
""));
266 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
267 RecentActivityExtracterModuleFactory.getModuleName(), value));
268 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
269 RecentActivityExtracterModuleFactory.getModuleName(),
270 NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
271 if (domain != null && domain.isEmpty() ==
false) {
272 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
273 RecentActivityExtracterModuleFactory.getModuleName(), domain));
275 BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
277 bbartifacts.add(bbart);
280 services.fireModuleDataEvent(
new ModuleDataEvent(
281 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
282 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE, bbartifacts));
288 private void getHistory() {
289 logger.log(Level.INFO,
"Pasco results path: {0}", moduleTempResultsDir);
290 boolean foundHistory =
false;
292 final File pascoRoot = InstalledFileLocator.getDefault().locate(
"pasco2", ExtractIE.class.getPackage().getName(),
false);
293 if (pascoRoot == null) {
294 this.addErrorMessage(
295 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.unableToGetHist", this.getName()));
296 logger.log(Level.SEVERE,
"Error finding pasco program ");
300 final String pascoHome = pascoRoot.getAbsolutePath();
301 logger.log(Level.INFO,
"Pasco2 home: {0}", pascoHome);
303 PASCO_LIB_PATH = pascoHome + File.separator +
"pasco2.jar" + File.pathSeparator
304 + pascoHome + File.separator +
"*";
306 File resultsDir =
new File(moduleTempResultsDir);
311 List<AbstractFile> indexFiles;
313 indexFiles = fileManager.
findFiles(dataSource,
"index.dat");
314 }
catch (TskCoreException ex) {
315 this.addErrorMessage(NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errGettingHistFiles",
317 logger.log(Level.WARNING,
"Error fetching 'index.data' files for Internet Explorer history.");
321 if (indexFiles.isEmpty()) {
322 String msg = NbBundle.getMessage(this.getClass(),
"ExtractIE.getHistory.errMsg.noHistFiles");
323 logger.log(Level.INFO, msg);
328 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
330 String indexFileName;
331 for (AbstractFile indexFile : indexFiles) {
337 indexFileName =
"index" + Integer.toString((
int) indexFile.getId()) +
".dat";
339 temps = RAImageIngestModule.getRATempPath(currentCase,
"IE") + File.separator + indexFileName;
340 File datFile =
new File(temps);
341 if (context.dataSourceIngestIsCancelled()) {
345 ContentUtils.writeToFile(indexFile, datFile, context::dataSourceIngestIsCancelled);
346 }
catch (IOException e) {
347 logger.log(Level.WARNING,
"Error while trying to write index.dat file " + datFile.getAbsolutePath(), e);
348 this.addErrorMessage(
349 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errWriteFile", this.getName(),
350 datFile.getAbsolutePath()));
354 String filename =
"pasco2Result." + indexFile.getId() +
".txt";
355 boolean bPascProcSuccess = executePasco(temps, filename);
356 if (context.dataSourceIngestIsCancelled()) {
362 if (bPascProcSuccess) {
364 bbartifacts.addAll(parsePascoOutput(indexFile, filename).stream()
365 .filter(bbart -> bbart.getArtifactTypeID() == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID())
366 .collect(Collectors.toList()));
372 logger.log(Level.WARNING,
"pasco execution failed on: {0}",
this.getName());
373 this.addErrorMessage(
374 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errProcHist", this.getName()));
379 services.fireModuleDataEvent(
new ModuleDataEvent(
380 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
381 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
393 private boolean executePasco(String indexFilePath, String outputFileName) {
394 boolean success =
true;
396 final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName;
397 final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName +
".err";
398 logger.log(Level.INFO,
"Writing pasco results to: {0}", outputFileFullPath);
399 List<String> commandLine =
new ArrayList<>();
400 commandLine.add(JAVA_PATH);
401 commandLine.add(
"-cp");
402 commandLine.add(PASCO_LIB_PATH);
403 commandLine.add(
"isi.pasco2.Main");
404 commandLine.add(
"-T");
405 commandLine.add(
"history");
406 commandLine.add(indexFilePath);
407 ProcessBuilder processBuilder =
new ProcessBuilder(commandLine);
408 processBuilder.redirectOutput(
new File(outputFileFullPath));
409 processBuilder.redirectError(
new File(errFileFullPath));
419 ExecUtil.execute(processBuilder,
new DataSourceIngestModuleProcessTerminator(context));
421 }
catch (IOException ex) {
423 logger.log(Level.SEVERE,
"Unable to execute Pasco to process Internet Explorer web history.", ex);
437 private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName) {
439 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
440 String fnAbs = moduleTempResultsDir + File.separator + pascoOutputFileName;
442 File file =
new File(fnAbs);
443 if (file.exists() ==
false) {
444 this.addErrorMessage(
445 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(),
447 logger.log(Level.WARNING,
"Pasco Output not found: {0}", file.getPath());
453 if (file.length() == 0) {
459 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
460 }
catch (FileNotFoundException ex) {
461 this.addErrorMessage(
462 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsing", this.getName(),
464 logger.log(Level.WARNING,
"Unable to find the Pasco file at " + file.getPath(), ex);
467 while (fileScanner.hasNext()) {
468 String line = fileScanner.nextLine();
469 if (!line.startsWith(
"URL")) {
473 String[] lineBuff = line.split(
"\\t");
475 if (lineBuff.length < 4) {
476 logger.log(Level.INFO,
"Found unrecognized IE history format.");
480 String actime = lineBuff[3];
481 Long ftime = (long) 0;
483 String realurl = null;
490 if (lineBuff[1].contains(
"@")) {
491 String url[] = lineBuff[1].split(
"@", 2);
496 domain = extractDomain(url[0]);
498 if (domain != null && domain.isEmpty() ==
false) {
502 realurl = lineBuff[1].trim();
509 user = user.replace(
"Visited:",
"");
510 user = user.replace(
":Host:",
"");
511 user = user.replaceAll(
"(:)(.*?)(:)",
"");
514 realurl = realurl.replace(
"Visited:",
"");
515 realurl = realurl.replaceAll(
":(.*?):",
"");
516 realurl = realurl.replace(
":Host:",
"");
517 realurl = realurl.trim();
518 domain = extractDomain(realurl);
524 realurl = lineBuff[1].trim();
525 domain = extractDomain(realurl);
528 if (!actime.isEmpty()) {
530 Long epochtime = dateFormatter.parse(actime).getTime();
531 ftime = epochtime / 1000;
532 }
catch (ParseException e) {
533 this.addErrorMessage(
534 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
536 logger.log(Level.WARNING, String.format(
"Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e);
541 BlackboardArtifact bbart = origFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
542 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
543 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
544 RecentActivityExtracterModuleFactory.getModuleName(), realurl));
547 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
548 RecentActivityExtracterModuleFactory.getModuleName(), ftime));
549 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
550 RecentActivityExtracterModuleFactory.getModuleName(),
""));
552 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
553 RecentActivityExtracterModuleFactory.getModuleName(),
554 NbBundle.getMessage(this.getClass(),
555 "ExtractIE.moduleName.text")));
556 if (domain != null && domain.isEmpty() ==
false) {
557 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
558 RecentActivityExtracterModuleFactory.getModuleName(), domain));
560 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
561 RecentActivityExtracterModuleFactory.getModuleName(), user));
562 bbart.addAttributes(bbattributes);
565 this.indexArtifact(bbart);
566 bbartifacts.add(bbart);
567 }
catch (TskCoreException ex) {
568 logger.log(Level.SEVERE,
"Error writing Internet Explorer web history artifact to the blackboard.", ex);
583 private String extractDomain(String url) {
584 if (url == null || url.isEmpty()) {
588 if (url.toLowerCase().startsWith(RESOURCE_URL_PREFIX)) {
595 return NetworkUtils.extractDomain(url);
synchronized List< AbstractFile > findFiles(String fileName)