19 package org.sleuthkit.autopsy.recentactivity;
22 import java.io.FileInputStream;
23 import java.io.FileNotFoundException;
24 import java.io.IOException;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.text.ParseException;
28 import java.text.SimpleDateFormat;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collection;
32 import java.util.HashMap;
33 import java.util.List;
34 import java.util.Scanner;
35 import java.util.logging.Level;
36 import org.openide.modules.InstalledFileLocator;
37 import org.openide.util.NbBundle.Messages;
57 final class ExtractEdge
extends Extract {
59 private static final Logger LOG = Logger.getLogger(ExtractEdge.class.getName());
60 private final Path moduleTempResultPath;
61 private Content dataSource;
62 private IngestJobContext context;
63 private HashMap<String, ArrayList<String>> containersTable;
65 private static final String EDGE =
"Edge";
67 private static final String EDGE_KEYWORD_VISIT =
"Visited:";
68 private static final String IGNORE_COMMA_IN_QUOTES_REGEX =
",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)";
70 private static final String EDGE_TABLE_TYPE_DOWNLOAD =
"iedownload";
71 private static final String EDGE_TABLE_TYPE_HISTORY =
"History";
72 private static final String EDGE_TABLE_TYPE_COOKIE =
"cookie";
74 private static final String EDGE_HEAD_URL =
"url";
75 private static final String EDGE_HEAD_ACCESSTIME =
"accessedtime";
76 private static final String EDGE_HEAD_NAME =
"name";
77 private static final String EDGE_HEAD_CONTAINER_ID =
"containerid";
78 private static final String EDGE_HEAD_RESPONSEHEAD =
"responseheaders";
79 private static final String EDGE_HEAD_TITLE =
"title";
80 private static final String EDGE_HEAD_RDOMAIN =
"rdomain";
81 private static final String EDGE_HEAD_VALUE =
"value";
82 private static final String EDGE_HEAD_LASTMOD =
"lastmodified";
84 private static final String EDGE_WEBCACHE_PREFIX =
"WebCacheV01";
85 private static final String EDGE_CONTAINER_FILE_PREFIX =
"Container_";
86 private static final String EDGE_CONTAINER_FILE_EXT =
".csv";
87 private static final String EDGE_WEBCACHE_EXT =
".dat";
89 private static final String ESE_TOOL_NAME =
"ESEDatabaseView.exe";
90 private static final String EDGE_WEBCACHE_NAME =
"WebCacheV01.dat";
91 private static final String EDGE_SPARTAN_NAME =
"Spartan.edb";
92 private static final String EDGE_CONTAINTERS_FILE_NAME =
"Containers.csv";
93 private static final String EDGE_FAVORITE_FILE_NAME =
"Favorites.csv";
94 private static final String EDGE_OUTPUT_FILE_NAME =
"Output.txt";
95 private static final String EDGE_ERROR_FILE_NAME =
"File.txt";
96 private static final String EDGE_WEBCACHE_FOLDER_NAME =
"WebCache";
97 private static final String EDGE_SPARTAN_FOLDER_NAME =
"MicrosoftEdge";
99 private static final String ESE_TOOL_FOLDER =
"ESEDatabaseView";
100 private static final String EDGE_RESULT_FOLDER_NAME =
"results";
102 private static final SimpleDateFormat DATE_FORMATTER =
new SimpleDateFormat(
"MM/dd/yyyy hh:mm:ss a");
105 "ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer",
106 "ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Edge WebCacheV01 file",
107 "ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file",
108 "ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file",
109 "ExtractEdge_Module_Name=Microsoft Edge",
110 "ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history",
111 "Progress_Message_Edge_History=Microsoft Edge History",
112 "Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks",
113 "Progress_Message_Edge_Cookies=Microsoft Edge Cookies",
119 ExtractEdge() throws NoCurrentCaseException {
120 moduleTempResultPath = Paths.get(RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(), EDGE), EDGE_RESULT_FOLDER_NAME);
124 protected String getName() {
125 return Bundle.ExtractEdge_Module_Name();
129 void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
130 this.dataSource = dataSource;
131 this.context = context;
132 this.setFoundData(
false);
134 List<AbstractFile> webCacheFiles = null;
135 List<AbstractFile> spartanFiles = null;
138 webCacheFiles = fetchWebCacheDBFiles();
139 }
catch (TskCoreException ex) {
140 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_errGettingWebCacheFiles());
141 LOG.log(Level.SEVERE,
"Error fetching 'WebCacheV01.dat' files for Microsoft Edge", ex);
145 spartanFiles = fetchSpartanDBFiles();
146 }
catch (TskCoreException ex) {
147 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
148 LOG.log(Level.SEVERE,
"Error fetching 'spartan.edb' files for Microsoft Edge", ex);
152 if (webCacheFiles == null && spartanFiles == null) {
156 this.setFoundData(
true);
158 if (!PlatformUtil.isWindowsOS()) {
159 LOG.log(Level.WARNING,
"Microsoft Edge files found, unable to parse on Non-Windows system");
163 final String esedumper = getPathForESEDumper();
164 if (esedumper == null) {
165 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_unableFindESEViewer());
166 LOG.log(Level.SEVERE,
"Error finding ESEDatabaseViewer program");
171 this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar);
172 }
catch (IOException | TskCoreException ex) {
173 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail());
174 LOG.log(Level.SEVERE,
"Error returned from processWebCacheDbFile", ex);
177 progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks());
179 this.processSpartanDbFile(esedumper, spartanFiles);
180 }
catch (IOException | TskCoreException ex) {
181 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
182 LOG.log(Level.SEVERE,
"Error returned from processSpartanDbFile", ex);
195 void processWebCacheDbFile(String eseDumperPath, List<AbstractFile> webCacheFiles, DataSourceIngestModuleProgress progressBar)
throws IOException, TskCoreException {
197 for (AbstractFile webCacheFile : webCacheFiles) {
199 if (context.dataSourceIngestIsCancelled()) {
203 clearContainerTable();
206 String tempWebCacheFileName = EDGE_WEBCACHE_PREFIX
207 + Integer.toString((
int) webCacheFile.getId()) + EDGE_WEBCACHE_EXT;
208 File tempWebCacheFile =
new File(RAImageIngestModule.getRATempPath(currentCase, EDGE), tempWebCacheFileName);
211 ContentUtils.writeToFile(webCacheFile, tempWebCacheFile,
212 context::dataSourceIngestIsCancelled);
213 }
catch (IOException ex) {
214 throw new IOException(
"Error writingToFile: " + webCacheFile, ex);
217 File resultsDir =
new File(moduleTempResultPath.toAbsolutePath() + Integer.toString((
int) webCacheFile.getId()));
220 executeDumper(eseDumperPath, tempWebCacheFile.getAbsolutePath(),
221 resultsDir.getAbsolutePath());
223 if (context.dataSourceIngestIsCancelled()) {
227 progressBar.progress(Bundle.Progress_Message_Edge_History());
229 this.getHistory(webCacheFile, resultsDir);
231 if (context.dataSourceIngestIsCancelled()) {
235 progressBar.progress(Bundle.Progress_Message_Edge_Cookies());
237 this.getCookies(webCacheFile, resultsDir);
240 tempWebCacheFile.delete();
241 FileUtil.deleteFileDir(resultsDir);
255 void processSpartanDbFile(String eseDumperPath, List<AbstractFile> spartanFiles)
throws IOException, TskCoreException {
257 for (AbstractFile spartanFile : spartanFiles) {
259 if (context.dataSourceIngestIsCancelled()) {
264 String tempSpartanFileName = EDGE_WEBCACHE_PREFIX
265 + Integer.toString((
int) spartanFile.getId()) + EDGE_WEBCACHE_EXT;
266 File tempSpartanFile =
new File(RAImageIngestModule.getRATempPath(currentCase, EDGE), tempSpartanFileName);
269 ContentUtils.writeToFile(spartanFile, tempSpartanFile,
270 context::dataSourceIngestIsCancelled);
271 }
catch (IOException ex) {
272 throw new IOException(
"Error writingToFile: " + spartanFile, ex);
275 File resultsDir =
new File(moduleTempResultPath.toAbsolutePath() + Integer.toString((
int) spartanFile.getId()));
278 executeDumper(eseDumperPath, tempSpartanFile.getAbsolutePath(),
279 resultsDir.getAbsolutePath());
281 if (context.dataSourceIngestIsCancelled()) {
285 this.getBookmarks(spartanFile, resultsDir);
288 tempSpartanFile.delete();
289 FileUtil.deleteFileDir(resultsDir);
303 private void getHistory(AbstractFile origFile, File resultDir)
throws TskCoreException, FileNotFoundException {
304 ArrayList<File> historyFiles = getHistoryFiles(resultDir);
306 if (historyFiles == null) {
310 for (File file : historyFiles) {
311 if (context.dataSourceIngestIsCancelled()) {
317 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
318 }
catch (FileNotFoundException ex) {
319 LOG.log(Level.WARNING,
"Unable to find the ESEDatabaseView file at " + file.getPath(), ex);
323 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
326 List<String> headers = null;
327 while (fileScanner.hasNext()) {
328 if (context.dataSourceIngestIsCancelled()) {
332 String line = fileScanner.nextLine();
333 if (headers == null) {
334 headers = Arrays.asList(line.toLowerCase().split(
","));
338 if (line.contains(EDGE_KEYWORD_VISIT)) {
339 BlackboardArtifact ba = getHistoryArtifact(origFile, headers, line);
349 if (!bbartifacts.isEmpty()) {
350 postArtifacts(bbartifacts);
363 private void getBookmarks(AbstractFile origFile, File resultDir)
throws TskCoreException {
365 File favoriteFile =
new File(resultDir, EDGE_FAVORITE_FILE_NAME);
368 fileScanner =
new Scanner(
new FileInputStream(favoriteFile));
369 }
catch (FileNotFoundException ex) {
375 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
378 List<String> headers = null;
379 while (fileScanner.hasNext()) {
380 String line = fileScanner.nextLine();
381 if (headers == null) {
382 headers = Arrays.asList(line.toLowerCase().split(
","));
386 BlackboardArtifact ba = getBookmarkArtifact(origFile, headers, line);
395 if (!bbartifacts.isEmpty()) {
396 postArtifacts(bbartifacts);
407 private void getCookies(AbstractFile origFile, File resultDir)
throws TskCoreException {
408 File containerFiles[] = resultDir.listFiles((dir, name) -> name.toLowerCase().contains(EDGE_TABLE_TYPE_COOKIE));
410 if (containerFiles == null) {
414 for (File file : containerFiles) {
415 if (context.dataSourceIngestIsCancelled()) {
421 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
422 }
catch (FileNotFoundException ex) {
423 LOG.log(Level.WARNING,
"Unable to find the ESEDatabaseView file at " + file.getPath(), ex);
427 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
430 List<String> headers = null;
431 while (fileScanner.hasNext()) {
432 if (context.dataSourceIngestIsCancelled()) {
436 String line = fileScanner.nextLine();
437 if (headers == null) {
438 headers = Arrays.asList(line.toLowerCase().split(
","));
442 BlackboardArtifact ba = getCookieArtifact(origFile, headers, line);
451 if (!bbartifacts.isEmpty()) {
452 postArtifacts(bbartifacts);
467 private void getDownloads(AbstractFile origFile, File resultDir)
throws TskCoreException, FileNotFoundException {
468 ArrayList<File> downloadFiles = getDownloadFiles(resultDir);
470 if (downloadFiles == null) {
474 for (File file : downloadFiles) {
475 if (context.dataSourceIngestIsCancelled()) {
481 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
482 }
catch (FileNotFoundException ex) {
483 LOG.log(Level.WARNING,
"Unable to find the ESEDatabaseView file at " + file.getPath(), ex);
486 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
489 List<String> headers = null;
490 while (fileScanner.hasNext()) {
491 if (context.dataSourceIngestIsCancelled()) {
495 String line = fileScanner.nextLine();
496 if (headers == null) {
497 headers = Arrays.asList(line.toLowerCase().split(
","));
501 if (line.contains(EDGE_TABLE_TYPE_DOWNLOAD)) {
503 BlackboardArtifact ba = getDownloadArtifact(origFile, headers, line);
513 postArtifacts(bbartifacts);
522 private String getPathForESEDumper() {
523 Path path = Paths.get(ESE_TOOL_FOLDER, ESE_TOOL_NAME);
524 File eseToolFile = InstalledFileLocator.getDefault().locate(path.toString(),
525 ExtractEdge.class.getPackage().getName(),
false);
526 if (eseToolFile != null) {
527 return eseToolFile.getAbsolutePath();
539 private List<AbstractFile> fetchWebCacheDBFiles() throws TskCoreException {
541 = currentCase.getServices().getFileManager();
542 return fileManager.
findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME);
551 private List<AbstractFile> fetchSpartanDBFiles() throws TskCoreException {
553 = currentCase.getServices().getFileManager();
554 return fileManager.
findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME);
568 private void executeDumper(String dumperPath, String inputFilePath,
569 String outputDir)
throws IOException {
571 final Path outputFilePath = Paths.get(outputDir, EDGE_OUTPUT_FILE_NAME);
572 final Path errFilePath = Paths.get(outputDir, EDGE_ERROR_FILE_NAME);
573 LOG.log(Level.INFO,
"Writing ESEDatabaseViewer results to: {0}", outputDir);
575 List<String> commandLine =
new ArrayList<>();
576 commandLine.add(dumperPath);
577 commandLine.add(
"/table");
578 commandLine.add(inputFilePath);
579 commandLine.add(
"*");
580 commandLine.add(
"/scomma");
581 commandLine.add(outputDir +
"\\" +
"*.csv");
583 ProcessBuilder processBuilder =
new ProcessBuilder(commandLine);
584 processBuilder.redirectOutput(outputFilePath.toFile());
585 processBuilder.redirectError(errFilePath.toFile());
587 ExecUtil.execute(processBuilder,
new DataSourceIngestModuleProcessTerminator(context));
600 private BlackboardArtifact getHistoryArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
601 String[] rowSplit = line.split(
",");
603 int index = headers.indexOf(EDGE_HEAD_URL);
604 String urlUserStr = rowSplit[index];
606 String[] str = urlUserStr.split(
"@");
607 String user = (str[0].replace(EDGE_KEYWORD_VISIT,
"")).trim();
610 index = headers.indexOf(EDGE_HEAD_ACCESSTIME);
611 String accessTime = rowSplit[index].trim();
614 Long epochtime = DATE_FORMATTER.parse(accessTime).getTime();
615 ftime = epochtime / 1000;
616 }
catch (ParseException ex) {
617 LOG.log(Level.WARNING,
"The Accessed Time format in history file seems invalid " + accessTime, ex);
620 BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY);
622 bbart.addAttributes(createHistoryAttribute(url, ftime,
625 NetworkUtils.extractDomain(url), user));
639 private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
640 String[] lineSplit = line.split(
",");
642 String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim();
645 Long epochtime = DATE_FORMATTER.parse(accessTime).getTime();
646 ftime = epochtime / 1000;
647 }
catch (ParseException ex) {
648 LOG.log(Level.WARNING,
"The Accessed Time format in history file seems invalid " + accessTime, ex);
651 String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim();
652 String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim());
653 String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim());
654 String url = flipDomain(domain);
656 BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE);
657 bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url)));
674 private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
675 BlackboardArtifact bbart = null;
677 String[] lineSplit = line.split(
",");
678 String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)];
695 private BlackboardArtifact getBookmarkArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
697 String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1);
699 String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)];
700 String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace(
"\"",
"");
706 BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
707 bbart.addAttributes(createBookmarkAttributes(url, title, null,
708 this.getName(), NetworkUtils.extractDomain(url)));
718 private String hexToChar(String hexString) {
719 String[] hexValues = hexString.split(
" ");
720 StringBuilder output =
new StringBuilder();
722 for (String str : hexValues) {
724 int value = Integer.parseInt(str, 16);
726 output.append((
char) value);
728 }
catch (NumberFormatException ex) {
733 return output.toString();
747 private String flipDomain(String domain) {
748 if (domain == null || domain.isEmpty()) {
752 String[] tokens = domain.split(
"\\.");
754 if (tokens.length < 2 || tokens.length > 3) {
758 StringBuilder buf =
new StringBuilder();
759 if (tokens.length > 2) {
760 buf.append(tokens[2]);
763 buf.append(tokens[1]);
765 buf.append(tokens[0]);
767 return buf.toString();
777 private ArrayList<File> getDownloadFiles(File resultDir)
throws FileNotFoundException {
778 return getContainerFiles(resultDir, EDGE_TABLE_TYPE_DOWNLOAD);
788 private ArrayList<File> getHistoryFiles(File resultDir)
throws FileNotFoundException {
789 return getContainerFiles(resultDir, EDGE_TABLE_TYPE_HISTORY);
800 private ArrayList<File> getContainerFiles(File resultDir, String type)
throws FileNotFoundException {
801 HashMap<String, ArrayList<String>> idTable = getContainerIDTable(resultDir);
803 ArrayList<String> idList = idTable.get(type);
804 if (idList == null) {
808 ArrayList<File> fileList =
new ArrayList<>();
809 for (String str : idList) {
810 String fileName = EDGE_CONTAINER_FILE_PREFIX + str + EDGE_CONTAINER_FILE_EXT;
811 fileList.add(
new File(resultDir, fileName));
827 private HashMap<String, ArrayList<String>> getContainerIDTable(File resultDir)
throws FileNotFoundException {
829 if (containersTable == null) {
830 File containerFile =
new File(resultDir, EDGE_CONTAINTERS_FILE_NAME);
832 try (Scanner fileScanner =
new Scanner(
new FileInputStream(containerFile))) {
833 List<String> headers = null;
834 containersTable =
new HashMap<>();
837 while (fileScanner.hasNext()) {
838 String line = fileScanner.nextLine();
839 if (headers == null) {
840 headers = Arrays.asList(line.toLowerCase().split(
","));
841 nameIdx = headers.indexOf(EDGE_HEAD_NAME);
842 idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID);
844 String[] row = line.split(
",");
845 String name = row[nameIdx];
846 String
id = row[idIdx];
848 ArrayList<String> idList = containersTable.get(name);
849 if (idList == null) {
850 idList =
new ArrayList<>();
851 containersTable.put(name, idList);
860 return containersTable;
866 private void clearContainerTable(){
867 containersTable = null;
synchronized List< AbstractFile > findFiles(String fileName)