23 package org.sleuthkit.autopsy.recentactivity;
25 import java.io.BufferedReader;
26 import org.openide.util.NbBundle;
30 import java.io.FileInputStream;
31 import java.io.FileNotFoundException;
32 import java.io.IOException;
33 import java.io.InputStreamReader;
34 import java.text.ParseException;
35 import java.text.SimpleDateFormat;
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.logging.Level;
40 import java.util.Collection;
41 import java.util.Scanner;
42 import java.util.stream.Collectors;
43 import org.openide.modules.InstalledFileLocator;
44 import org.openide.util.NbBundle.Messages;
50 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
52 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
66 class ExtractIE
extends Extract {
68 private static final Logger logger = Logger.getLogger(ExtractIE.class.getName());
69 private final String moduleTempResultsDir;
70 private String PASCO_LIB_PATH;
71 private final String JAVA_PATH;
72 private static final String RESOURCE_URL_PREFIX =
"res://";
73 private static final SimpleDateFormat dateFormatter =
new SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
74 private Content dataSource;
75 private IngestJobContext context;
78 "Progress_Message_IE_History=IE History",
79 "Progress_Message_IE_Bookmarks=IE Bookmarks",
80 "Progress_Message_IE_Cookies=IE Cookies",
81 "Progress_Message_IE_Downloads=IE Downloads",
82 "Progress_Message_IE_FormHistory=IE Form History",
83 "Progress_Message_IE_AutoFill=IE Auto Fill",
84 "Progress_Message_IE_Logins=IE Logins",})
86 ExtractIE() throws NoCurrentCaseException {
87 moduleName = NbBundle.getMessage(ExtractIE.class,
"ExtractIE.moduleName.text");
88 moduleTempResultsDir = RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(),
"IE") + File.separator +
"results";
89 JAVA_PATH = PlatformUtil.getJavaPath();
93 public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
94 this.dataSource = dataSource;
95 this.context = context;
98 progressBar.progress(Bundle.Progress_Message_IE_Bookmarks());
101 progressBar.progress(Bundle.Progress_Message_IE_Cookies());
104 progressBar.progress(Bundle.Progress_Message_IE_History());
111 private void getBookmark() {
113 List<AbstractFile> favoritesFiles;
115 favoritesFiles = fileManager.
findFiles(dataSource,
"%.url",
"Favorites");
116 }
catch (TskCoreException ex) {
117 logger.log(Level.WARNING,
"Error fetching 'url' files for Internet Explorer bookmarks.", ex);
118 this.addErrorMessage(
119 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getBookmark.errMsg.errGettingBookmarks",
124 if (favoritesFiles.isEmpty()) {
125 logger.log(Level.INFO,
"Didn't find any IE bookmark files.");
130 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
131 for (AbstractFile fav : favoritesFiles) {
132 if (fav.getSize() == 0) {
136 if (context.dataSourceIngestIsCancelled()) {
140 String url = getURLFromIEBookmarkFile(fav);
142 String name = fav.getName();
143 Long datetime = fav.getCrtime();
144 String Tempdate = datetime.toString();
145 datetime = Long.valueOf(Tempdate);
146 String domain = extractDomain(url);
148 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
149 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
150 RecentActivityExtracterModuleFactory.getModuleName(), url));
151 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
152 RecentActivityExtracterModuleFactory.getModuleName(), name));
153 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
154 RecentActivityExtracterModuleFactory.getModuleName(), datetime));
155 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
156 RecentActivityExtracterModuleFactory.getModuleName(),
157 NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
158 if (domain != null && domain.isEmpty() ==
false) {
159 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
160 RecentActivityExtracterModuleFactory.getModuleName(), domain));
163 BlackboardArtifact bbart = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, fav, bbattributes);
165 bbartifacts.add(bbart);
169 postArtifacts(bbartifacts);
172 private String getURLFromIEBookmarkFile(AbstractFile fav) {
173 BufferedReader reader =
new BufferedReader(
new InputStreamReader(
new ReadContentInputStream(fav)));
174 String line, url =
"";
176 line = reader.readLine();
177 while (null != line) {
180 if (line.startsWith(
"URL")) {
181 url = line.substring(line.indexOf(
"=") + 1);
184 line = reader.readLine();
186 }
catch (IOException ex) {
187 logger.log(Level.WARNING,
"Failed to read from content: " + fav.getName(), ex);
188 this.addErrorMessage(
189 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg", this.getName(),
191 }
catch (IndexOutOfBoundsException ex) {
192 logger.log(Level.WARNING,
"Failed while getting URL of IE bookmark. Unexpected format of the bookmark file: " + fav.getName(), ex);
193 this.addErrorMessage(
194 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg2", this.getName(),
199 }
catch (IOException ex) {
200 logger.log(Level.WARNING,
"Failed to close reader.", ex);
210 private void getCookie() {
212 List<AbstractFile> cookiesFiles;
214 cookiesFiles = fileManager.
findFiles(dataSource,
"%.txt",
"Cookies");
215 }
catch (TskCoreException ex) {
216 logger.log(Level.WARNING,
"Error getting cookie files for IE");
217 this.addErrorMessage(
218 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errGettingFile", this.getName()));
222 if (cookiesFiles.isEmpty()) {
223 logger.log(Level.INFO,
"Didn't find any IE cookies files.");
228 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
229 for (AbstractFile cookiesFile : cookiesFiles) {
230 if (context.dataSourceIngestIsCancelled()) {
233 if (cookiesFile.getSize() == 0) {
237 byte[] t =
new byte[(int) cookiesFile.getSize()];
239 final int bytesRead = cookiesFile.read(t, 0, cookiesFile.getSize());
240 }
catch (TskCoreException ex) {
241 logger.log(Level.WARNING,
"Error reading bytes of Internet Explorer cookie.", ex);
242 this.addErrorMessage(
243 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errReadingIECookie",
244 this.getName(), cookiesFile.getName()));
247 String cookieString =
new String(t);
248 String[] values = cookieString.split(
"\n");
249 String url = values.length > 2 ? values[2] :
"";
250 String value = values.length > 1 ? values[1] :
"";
251 String name = values.length > 0 ? values[0] :
"";
252 Long datetime = cookiesFile.getCrtime();
253 String tempDate = datetime.toString();
254 datetime = Long.valueOf(tempDate);
255 String domain = extractDomain(url);
257 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
258 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
259 RecentActivityExtracterModuleFactory.getModuleName(), url));
260 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
261 RecentActivityExtracterModuleFactory.getModuleName(), datetime));
262 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
263 RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name :
""));
264 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
265 RecentActivityExtracterModuleFactory.getModuleName(), value));
266 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
267 RecentActivityExtracterModuleFactory.getModuleName(),
268 NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
269 if (domain != null && domain.isEmpty() ==
false) {
270 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
271 RecentActivityExtracterModuleFactory.getModuleName(), domain));
273 BlackboardArtifact bbart = createArtifactWithAttributes(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
275 bbartifacts.add(bbart);
279 postArtifacts(bbartifacts);
285 private void getHistory() {
286 logger.log(Level.INFO,
"Pasco results path: {0}", moduleTempResultsDir);
287 boolean foundHistory =
false;
289 final File pascoRoot = InstalledFileLocator.getDefault().locate(
"pasco2", ExtractIE.class.getPackage().getName(),
false);
290 if (pascoRoot == null) {
291 this.addErrorMessage(
292 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.unableToGetHist", this.getName()));
293 logger.log(Level.SEVERE,
"Error finding pasco program ");
297 final String pascoHome = pascoRoot.getAbsolutePath();
298 logger.log(Level.INFO,
"Pasco2 home: {0}", pascoHome);
300 PASCO_LIB_PATH = pascoHome + File.separator +
"pasco2.jar" + File.pathSeparator
301 + pascoHome + File.separator +
"*";
303 File resultsDir =
new File(moduleTempResultsDir);
307 FileManager fileManager = currentCase.getServices().getFileManager();
308 List<AbstractFile> indexFiles;
310 indexFiles = fileManager.findFiles(dataSource,
"index.dat");
311 }
catch (TskCoreException ex) {
312 this.addErrorMessage(NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errGettingHistFiles",
314 logger.log(Level.WARNING,
"Error fetching 'index.data' files for Internet Explorer history.");
318 if (indexFiles.isEmpty()) {
319 String msg = NbBundle.getMessage(this.getClass(),
"ExtractIE.getHistory.errMsg.noHistFiles");
320 logger.log(Level.INFO, msg);
325 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
327 String indexFileName;
328 for (AbstractFile indexFile : indexFiles) {
335 indexFileName =
"index" + Integer.toString((
int) indexFile.getId()) +
".dat";
337 temps = RAImageIngestModule.getRATempPath(currentCase,
"IE") + File.separator + indexFileName;
338 File datFile =
new File(temps);
339 if (context.dataSourceIngestIsCancelled()) {
343 ContentUtils.writeToFile(indexFile, datFile, context::dataSourceIngestIsCancelled);
344 }
catch (IOException e) {
345 logger.log(Level.WARNING,
"Error while trying to write index.dat file " + datFile.getAbsolutePath(), e);
346 this.addErrorMessage(
347 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errWriteFile", this.getName(),
348 datFile.getAbsolutePath()));
352 String filename =
"pasco2Result." + indexFile.getId() +
".txt";
353 boolean bPascProcSuccess = executePasco(temps, filename);
354 if (context.dataSourceIngestIsCancelled()) {
360 if (bPascProcSuccess) {
362 bbartifacts.addAll(parsePascoOutput(indexFile, filename).stream()
363 .filter(bbart -> bbart.getArtifactTypeID() == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID())
364 .collect(Collectors.toList()));
365 if (context.dataSourceIngestIsCancelled()) {
373 logger.log(Level.WARNING,
"pasco execution failed on: {0}", filename);
374 this.addErrorMessage(
375 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errProcHist", this.getName()));
379 postArtifacts(bbartifacts);
391 "# {0} - sub module name",
392 "ExtractIE_executePasco_errMsg_errorRunningPasco={0}: Error analyzing Internet Explorer web history",
394 private boolean executePasco(String indexFilePath, String outputFileName) {
395 boolean success =
true;
397 final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName;
398 final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName +
".err";
399 logger.log(Level.INFO,
"Writing pasco results to: {0}", outputFileFullPath);
400 List<String> commandLine =
new ArrayList<>();
401 commandLine.add(JAVA_PATH);
402 commandLine.add(
"-cp");
403 commandLine.add(PASCO_LIB_PATH);
404 commandLine.add(
"isi.pasco2.Main");
405 commandLine.add(
"-T");
406 commandLine.add(
"history");
407 commandLine.add(indexFilePath);
408 ProcessBuilder processBuilder =
new ProcessBuilder(commandLine);
409 processBuilder.redirectOutput(
new File(outputFileFullPath));
410 processBuilder.redirectError(
new File(errFileFullPath));
420 ExecUtil.execute(processBuilder,
new DataSourceIngestModuleProcessTerminator(context,
true));
422 }
catch (IOException ex) {
423 logger.log(Level.SEVERE,
"Error executing Pasco to process Internet Explorer web history", ex);
424 addErrorMessage(Bundle.ExtractIE_executePasco_errMsg_errorRunningPasco(getName()));
439 private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName) {
441 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
442 String fnAbs = moduleTempResultsDir + File.separator + pascoOutputFileName;
444 File file =
new File(fnAbs);
445 if (file.exists() ==
false) {
446 this.addErrorMessage(
447 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(),
449 logger.log(Level.WARNING,
"Pasco Output not found: {0}", file.getPath());
455 if (file.length() == 0) {
461 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
462 }
catch (FileNotFoundException ex) {
463 this.addErrorMessage(
464 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsing", this.getName(),
466 logger.log(Level.WARNING,
"Unable to find the Pasco file at " + file.getPath(), ex);
469 while (fileScanner.hasNext()) {
471 if (context.dataSourceIngestIsCancelled()) {
475 String line = fileScanner.nextLine();
476 if (!line.startsWith(
"URL")) {
480 String[] lineBuff = line.split(
"\\t");
482 if (lineBuff.length < 4) {
483 logger.log(Level.INFO,
"Found unrecognized IE history format.");
487 String actime = lineBuff[3];
488 Long ftime = (long) 0;
490 String realurl = null;
497 if (lineBuff[1].contains(
"@")) {
498 String url[] = lineBuff[1].split(
"@", 2);
503 domain = extractDomain(url[0]);
505 if (domain != null && domain.isEmpty() ==
false) {
509 realurl = lineBuff[1].trim();
516 user = user.replace(
"Visited:",
"");
517 user = user.replace(
":Host:",
"");
518 user = user.replaceAll(
"(:)(.*?)(:)",
"");
521 realurl = realurl.replace(
"Visited:",
"");
522 realurl = realurl.replaceAll(
":(.*?):",
"");
523 realurl = realurl.replace(
":Host:",
"");
524 realurl = realurl.trim();
525 domain = extractDomain(realurl);
531 realurl = lineBuff[1].trim();
532 domain = extractDomain(realurl);
535 if (!actime.isEmpty()) {
537 Long epochtime = dateFormatter.parse(actime).getTime();
538 ftime = epochtime / 1000;
539 }
catch (ParseException e) {
540 this.addErrorMessage(
541 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
543 logger.log(Level.WARNING, String.format(
"Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e);
548 BlackboardArtifact bbart = origFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
549 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
550 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
551 RecentActivityExtracterModuleFactory.getModuleName(), realurl));
554 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
555 RecentActivityExtracterModuleFactory.getModuleName(), ftime));
556 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
557 RecentActivityExtracterModuleFactory.getModuleName(),
""));
559 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
560 RecentActivityExtracterModuleFactory.getModuleName(),
561 NbBundle.getMessage(this.getClass(),
562 "ExtractIE.moduleName.text")));
563 if (domain != null && domain.isEmpty() ==
false) {
564 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
565 RecentActivityExtracterModuleFactory.getModuleName(), domain));
567 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
568 RecentActivityExtracterModuleFactory.getModuleName(), user));
569 bbart.addAttributes(bbattributes);
571 bbartifacts.add(bbart);
572 }
catch (TskCoreException ex) {
573 logger.log(Level.SEVERE,
"Error writing Internet Explorer web history artifact to the blackboard. Pasco results will be incomplete", ex);
588 private String extractDomain(String url) {
589 if (url == null || url.isEmpty()) {
593 if (url.toLowerCase().startsWith(RESOURCE_URL_PREFIX)) {
600 return NetworkUtils.extractDomain(url);
synchronized List< AbstractFile > findFiles(String fileName)