Autopsy  4.20.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
ExtractEdge.java
Go to the documentation of this file.
1 /*
2  *
3  * Autopsy Forensic Browser
4  *
5  * Copyright 2019-2021 Basis Technology Corp.
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.recentactivity;
20 
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.FileNotFoundException;
24 import java.io.IOException;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.text.ParseException;
28 import java.text.SimpleDateFormat;
29 import java.text.DateFormat;
30 import java.util.ArrayList;
31 import java.util.Arrays;
32 import java.util.Collection;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Locale;
36 import java.util.Scanner;
37 import java.util.logging.Level;
38 import java.util.regex.Matcher;
39 import java.util.regex.Pattern;
40 import org.openide.modules.InstalledFileLocator;
41 import org.openide.util.NbBundle.Messages;
51 import org.sleuthkit.datamodel.AbstractFile;
52 import org.sleuthkit.datamodel.BlackboardArtifact;
53 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK;
54 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE;
55 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY;
56 import org.sleuthkit.datamodel.Content;
57 import org.sleuthkit.datamodel.TskCoreException;
58 
62 final class ExtractEdge extends Extract {
63 
64  private static final Logger LOG = Logger.getLogger(ExtractEdge.class.getName());
65  private Content dataSource;
66  private final IngestJobContext context;
67  private HashMap<String, ArrayList<String>> containersTable;
68 
69  private static final String EDGE = "Edge"; //NON-NLS
70 
71  private static final String EDGE_KEYWORD_VISIT = "Visited:"; //NON-NLS
72  private static final String IGNORE_COMMA_IN_QUOTES_REGEX = ",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"; //NON-NLS
73 
74  private static final String EDGE_TABLE_TYPE_DOWNLOAD = "iedownload"; //NON-NLS
75  private static final String EDGE_TABLE_TYPE_HISTORY = "History"; //NON-NLS
76  private static final String EDGE_TABLE_TYPE_COOKIE = "cookie"; //NON-NLS
77 
78  private static final String EDGE_HEAD_URL = "url"; //NON-NLS
79  private static final String EDGE_HEAD_ACCESSTIME = "accessedtime"; //NON-NLS
80  private static final String EDGE_HEAD_NAME = "name"; //NON-NLS
81  private static final String EDGE_HEAD_CONTAINER_ID = "containerid"; //NON-NLS
82  private static final String EDGE_HEAD_RESPONSEHEAD = "responseheaders"; //NON-NLS
83  private static final String EDGE_HEAD_TITLE = "title"; //NON-NLS
84  private static final String EDGE_HEAD_RDOMAIN = "rdomain"; //NON-NLS
85  private static final String EDGE_HEAD_VALUE = "value"; //NON-NLS
86  private static final String EDGE_HEAD_LASTMOD = "lastmodified"; //NON-NLS
87 
88  private static final String EDGE_WEBCACHE_PREFIX = "WebCacheV01"; //NON-NLS
89  private static final String EDGE_CONTAINER_FILE_PREFIX = "Container_"; //NON-NLS
90  private static final String EDGE_CONTAINER_FILE_EXT = ".csv"; //NON-NLS
91  private static final String EDGE_WEBCACHE_EXT = ".dat"; //NON-NLS
92 
93  private static final String ESE_TOOL_NAME = "ESEDatabaseView.exe"; //NON-NLS
94  private static final String EDGE_WEBCACHE_NAME = "WebCacheV01.dat"; //NON-NLS
95  private static final String EDGE_SPARTAN_NAME = "Spartan.edb"; //NON-NLS
96  private static final String EDGE_CONTAINTERS_FILE_NAME = "Containers.csv"; //NON-NLS
97  private static final String EDGE_FAVORITE_FILE_NAME = "Favorites.csv"; //NON-NLS
98  private static final String EDGE_OUTPUT_FILE_NAME = "Output.txt"; //NON-NLS
99  private static final String EDGE_ERROR_FILE_NAME = "File.txt"; //NON-NLS
100  private static final String EDGE_WEBCACHE_FOLDER_NAME = "WebCache"; //NON-NLS
101  private static final String EDGE_SPARTAN_FOLDER_NAME = "MicrosoftEdge"; //NON-NLS
102 
103  private static final String ESE_TOOL_FOLDER = "ESEDatabaseView"; //NON-NLS
104  private static final String EDGE_RESULT_FOLDER_NAME = "results"; //NON-NLS
105 
106  // ESEDatabaseView converts long timestamps into a string based on the current locale,
107  // so the default format may not always work.
108  private SimpleDateFormat previouslyValidDateFormat = null;
109 
110  @Messages({
111  "ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer",
112  "ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Edge WebCacheV01 file",
113  "ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file",
114  "ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file",
115  "ExtractEdge_Module_Name=Microsoft Edge Analyzer",
116  "ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history",
117  "Progress_Message_Edge_History=Microsoft Edge History",
118  "Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks",
119  "Progress_Message_Edge_Cookies=Microsoft Edge Cookies",})
120 
124  ExtractEdge(IngestJobContext context) {
125  super(Bundle.ExtractEdge_Module_Name(), context);
126  this.context = context;
127  }
128 
129  @Override
130  protected String getDisplayName() {
131  return Bundle.ExtractEdge_Module_Name();
132  }
133 
134  @Override
135  void process(Content dataSource, DataSourceIngestModuleProgress progressBar) {
136  String moduleTempDir = RAImageIngestModule.getRATempPath(getCurrentCase(), EDGE, context.getJobId());
137  String moduleTempResultDir = Paths.get(moduleTempDir, EDGE_RESULT_FOLDER_NAME).toString();
138 
139  this.dataSource = dataSource;
140  this.setFoundData(false);
141 
142  List<AbstractFile> webCacheFiles = null;
143  List<AbstractFile> spartanFiles = null;
144 
145  try {
146  webCacheFiles = fetchWebCacheDBFiles();
147  } catch (TskCoreException ex) {
148  this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_errGettingWebCacheFiles());
149  LOG.log(Level.SEVERE, "Error fetching 'WebCacheV01.dat' files for Microsoft Edge", ex); //NON-NLS
150  }
151 
152  if (context.dataSourceIngestIsCancelled()) {
153  return;
154  }
155 
156  try {
157  spartanFiles = fetchSpartanDBFiles(); // For later use with bookmarks
158  } catch (TskCoreException ex) {
159  this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
160  LOG.log(Level.SEVERE, "Error fetching 'spartan.edb' files for Microsoft Edge", ex); //NON-NLS
161  }
162 
163  // No edge files found
164  if (webCacheFiles == null && spartanFiles == null) {
165  return;
166  }
167 
168  this.setFoundData(true);
169 
170  if (!PlatformUtil.isWindowsOS()) {
171  LOG.log(Level.WARNING, "Microsoft Edge files found, unable to parse on Non-Windows system"); //NON-NLS
172  return;
173  }
174 
175  if (context.dataSourceIngestIsCancelled()) {
176  return;
177  }
178 
179  final String esedumper = getPathForESEDumper();
180  if (esedumper == null) {
181  LOG.log(Level.SEVERE, "Error finding ESEDatabaseViewer program"); //NON-NLS
182  this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_unableFindESEViewer());
183  return; //If we cannot find the ESEDatabaseView we cannot proceed
184  }
185 
186  try {
187  this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar, moduleTempDir, moduleTempResultDir);
188  } catch (IOException | TskCoreException ex) {
189  LOG.log(Level.SEVERE, "Error processing 'WebCacheV01.dat' files for Microsoft Edge", ex); // NON-NLS
190  this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail());
191  }
192 
193  progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks());
194  try {
195  this.processSpartanDbFile(esedumper, spartanFiles, moduleTempDir, moduleTempResultDir);
196  } catch (IOException | TskCoreException ex) {
197  LOG.log(Level.SEVERE, "Error processing 'spartan.edb' files for Microsoft Edge", ex); // NON-NLS
198  this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
199  }
200  }
201 
214  void processWebCacheDbFile(String eseDumperPath, List<AbstractFile> webCacheFiles, DataSourceIngestModuleProgress progressBar,
215  String moduleTempDir, String moduleTempResultDir) throws IOException, TskCoreException {
216  for (AbstractFile webCacheFile : webCacheFiles) {
217 
218  if (context.dataSourceIngestIsCancelled()) {
219  return;
220  }
221 
222  clearContainerTable();
223 
224  //Run the dumper
225  String tempWebCacheFileName = EDGE_WEBCACHE_PREFIX
226  + Integer.toString((int) webCacheFile.getId()) + EDGE_WEBCACHE_EXT; //NON-NLS
227  File tempWebCacheFile = new File(moduleTempDir, tempWebCacheFileName);
228 
229  try {
230  ContentUtils.writeToFile(webCacheFile, tempWebCacheFile,
231  context::dataSourceIngestIsCancelled);
232  } catch (IOException ex) {
233  throw new IOException("Error writingToFile: " + webCacheFile, ex); //NON-NLS
234  }
235 
236  File resultsDir = new File(moduleTempDir, Integer.toString((int) webCacheFile.getId()));
237  resultsDir.mkdirs();
238  try {
239  executeDumper(eseDumperPath, tempWebCacheFile.getAbsolutePath(),
240  resultsDir.getAbsolutePath());
241 
242  if (context.dataSourceIngestIsCancelled()) {
243  return;
244  }
245 
246  progressBar.progress(Bundle.Progress_Message_Edge_History());
247 
248  this.getHistory(webCacheFile, resultsDir);
249 
250  if (context.dataSourceIngestIsCancelled()) {
251  return;
252  }
253 
254  progressBar.progress(Bundle.Progress_Message_Edge_Cookies());
255 
256  this.getCookies(webCacheFile, resultsDir);
257 
258  } finally {
259  tempWebCacheFile.delete();
260  FileUtil.deleteFileDir(resultsDir);
261  }
262  }
263  }
264 
277  void processSpartanDbFile(String eseDumperPath, List<AbstractFile> spartanFiles, String moduleTempDir, String moduleTempResultDir) throws IOException, TskCoreException {
278  for (AbstractFile spartanFile : spartanFiles) {
279 
280  if (context.dataSourceIngestIsCancelled()) {
281  return;
282  }
283 
284  //Run the dumper
285  String tempSpartanFileName = EDGE_WEBCACHE_PREFIX
286  + Integer.toString((int) spartanFile.getId()) + EDGE_WEBCACHE_EXT;
287  File tempSpartanFile = new File(moduleTempDir, tempSpartanFileName);
288 
289  try {
290  ContentUtils.writeToFile(spartanFile, tempSpartanFile,
291  context::dataSourceIngestIsCancelled);
292  } catch (IOException ex) {
293  throw new IOException("Error writingToFile: " + spartanFile, ex); //NON-NLS
294  }
295 
296  File resultsDir = new File(moduleTempResultDir, Integer.toString((int) spartanFile.getId()));
297  resultsDir.mkdirs();
298  try {
299  executeDumper(eseDumperPath, tempSpartanFile.getAbsolutePath(),
300  resultsDir.getAbsolutePath());
301 
302  if (context.dataSourceIngestIsCancelled()) {
303  return;
304  }
305 
306  this.getBookmarks(spartanFile, resultsDir);
307 
308  } finally {
309  tempSpartanFile.delete();
310  FileUtil.deleteFileDir(resultsDir);
311  }
312  }
313  }
314 
326  private void getHistory(AbstractFile origFile, File resultDir) throws TskCoreException, FileNotFoundException {
327  ArrayList<File> historyFiles = getHistoryFiles(resultDir);
328  if (historyFiles == null) {
329  return;
330  }
331 
332  for (File file : historyFiles) {
333  if (context.dataSourceIngestIsCancelled()) {
334  return;
335  }
336 
337  Scanner fileScanner;
338  try {
339  fileScanner = new Scanner(new FileInputStream(file.toString()));
340  } catch (FileNotFoundException ex) {
341  LOG.log(Level.WARNING, "Unable to find the ESEDatabaseView file at " + file.getPath(), ex); //NON-NLS
342  continue; // If we couldn't open this file, continue to the next file
343  }
344 
345  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
346 
347  try {
348  List<String> headers = null;
349  while (fileScanner.hasNext()) {
350  if (context.dataSourceIngestIsCancelled()) {
351  return;
352  }
353 
354  String line = fileScanner.nextLine();
355  if (headers == null) {
356  headers = Arrays.asList(line.toLowerCase().split(","));
357  continue;
358  }
359 
360  if (line.contains(EDGE_KEYWORD_VISIT)) {
361  BlackboardArtifact ba = getHistoryArtifact(origFile, headers, line);
362  if (ba != null) {
363  bbartifacts.add(ba);
364  }
365  }
366  }
367  } finally {
368  fileScanner.close();
369  }
370 
371  if (!bbartifacts.isEmpty() && !context.dataSourceIngestIsCancelled()) {
372  postArtifacts(bbartifacts);
373  }
374  }
375  }
376 
386  private void getBookmarks(AbstractFile origFile, File resultDir) throws TskCoreException {
387  Scanner fileScanner;
388  File favoriteFile = new File(resultDir, EDGE_FAVORITE_FILE_NAME);
389 
390  try {
391  fileScanner = new Scanner(new FileInputStream(favoriteFile));
392  } catch (FileNotFoundException ex) {
393  // This is a non-fatal error, if the favorites file is not found
394  // there might have not been any favorites\bookmarks
395  return;
396  }
397 
398  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
399 
400  try {
401  List<String> headers = null;
402  while (fileScanner.hasNext()) {
403  String line = fileScanner.nextLine();
404  if (headers == null) {
405  headers = Arrays.asList(line.toLowerCase().split(","));
406  continue;
407  }
408 
409  BlackboardArtifact ba = getBookmarkArtifact(origFile, headers, line);
410  if (ba != null) {
411  bbartifacts.add(ba);
412  }
413  }
414  } finally {
415  fileScanner.close();
416  }
417 
418  if (!bbartifacts.isEmpty() && !context.dataSourceIngestIsCancelled()) {
419  postArtifacts(bbartifacts);
420  }
421  }
422 
431  private void getCookies(AbstractFile origFile, File resultDir) throws TskCoreException {
432  File containerFiles[] = resultDir.listFiles((dir, name) -> name.toLowerCase().contains(EDGE_TABLE_TYPE_COOKIE));
433 
434  if (containerFiles == null) {
435  return;
436  }
437 
438  for (File file : containerFiles) {
439  if (context.dataSourceIngestIsCancelled()) {
440  return;
441  }
442 
443  Scanner fileScanner;
444  try {
445  fileScanner = new Scanner(new FileInputStream(file.toString()));
446  } catch (FileNotFoundException ex) {
447  LOG.log(Level.WARNING, "Unable to find the ESEDatabaseView file at " + file.getPath(), ex); //NON-NLS
448  continue; // If we couldn't open this file, continue to the next file
449  }
450 
451  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
452 
453  try {
454  List<String> headers = null;
455  while (fileScanner.hasNext()) {
456  if (context.dataSourceIngestIsCancelled()) {
457  return;
458  }
459 
460  String line = fileScanner.nextLine();
461  if (headers == null) {
462  headers = Arrays.asList(line.toLowerCase().split(","));
463  continue;
464  }
465 
466  BlackboardArtifact ba = getCookieArtifact(origFile, headers, line);
467  if (ba != null) {
468  bbartifacts.add(ba);
469  }
470  }
471  } finally {
472  fileScanner.close();
473  }
474 
475  if (!bbartifacts.isEmpty() && !context.dataSourceIngestIsCancelled()) {
476  postArtifacts(bbartifacts);
477  }
478  }
479  }
480 
492  private void getDownloads(AbstractFile origFile, File resultDir) throws TskCoreException, FileNotFoundException {
493  ArrayList<File> downloadFiles = getDownloadFiles(resultDir);
494 
495  if (downloadFiles == null) {
496  return;
497  }
498 
499  for (File file : downloadFiles) {
500  if (context.dataSourceIngestIsCancelled()) {
501  return;
502  }
503 
504  Scanner fileScanner;
505  try {
506  fileScanner = new Scanner(new FileInputStream(file.toString()));
507  } catch (FileNotFoundException ex) {
508  LOG.log(Level.WARNING, "Unable to find the ESEDatabaseView file at " + file.getPath(), ex); //NON-NLS
509  continue; // If we couldn't open this file, continue to the next file
510  }
511  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
512 
513  try {
514  List<String> headers = null;
515  while (fileScanner.hasNext()) {
516  if (context.dataSourceIngestIsCancelled()) {
517  return;
518  }
519 
520  String line = fileScanner.nextLine();
521  if (headers == null) {
522  headers = Arrays.asList(line.toLowerCase().split(","));
523  continue;
524  }
525 
526  if (line.contains(EDGE_TABLE_TYPE_DOWNLOAD)) {
527 
528  BlackboardArtifact ba = getDownloadArtifact(origFile, headers, line);
529  if (ba != null) {
530  bbartifacts.add(ba);
531  }
532  }
533  }
534  } finally {
535  fileScanner.close();
536  }
537 
538  if (!context.dataSourceIngestIsCancelled()) {
539  postArtifacts(bbartifacts);
540  }
541  }
542  }
543 
550  private String getPathForESEDumper() {
551  Path path = Paths.get(ESE_TOOL_FOLDER, ESE_TOOL_NAME);
552  File eseToolFile = InstalledFileLocator.getDefault().locate(path.toString(),
553  ExtractEdge.class.getPackage().getName(), false);
554  if (eseToolFile != null) {
555  return eseToolFile.getAbsolutePath();
556  }
557 
558  return null;
559  }
560 
568  private List<AbstractFile> fetchWebCacheDBFiles() throws TskCoreException {
570  = currentCase.getServices().getFileManager();
571  return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME);
572  }
573 
581  private List<AbstractFile> fetchSpartanDBFiles() throws TskCoreException {
583  = currentCase.getServices().getFileManager();
584  return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME);
585  }
586 
599  private void executeDumper(String dumperPath, String inputFilePath,
600  String outputDir) throws IOException {
601 
602  final Path outputFilePath = Paths.get(outputDir, EDGE_OUTPUT_FILE_NAME);
603  final Path errFilePath = Paths.get(outputDir, EDGE_ERROR_FILE_NAME);
604  LOG.log(Level.INFO, "Writing ESEDatabaseViewer results to: {0}", outputDir); //NON-NLS
605 
606  List<String> commandLine = new ArrayList<>();
607  commandLine.add(dumperPath);
608  commandLine.add("/table"); //NON-NLS
609  commandLine.add(inputFilePath);
610  commandLine.add("*"); //NON-NLS
611  commandLine.add("/scomma"); //NON-NLS
612  commandLine.add(outputDir + "\\" + "*.csv"); //NON-NLS
613 
614  ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
615  processBuilder.redirectOutput(outputFilePath.toFile());
616  processBuilder.redirectError(errFilePath.toFile());
617 
618  ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context, true));
619  }
620 
633  private BlackboardArtifact getHistoryArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
634  String[] rowSplit = line.split(",");
635 
636  int index = headers.indexOf(EDGE_HEAD_URL);
637  String urlUserStr = rowSplit[index];
638 
639  String[] str = urlUserStr.split("@");
640  String user = (str[0].replace(EDGE_KEYWORD_VISIT, "")).trim();
641  String url = str[1];
642 
643  index = headers.indexOf(EDGE_HEAD_ACCESSTIME);
644  String accessTime = rowSplit[index].trim();
645  Long ftime = parseTimestamp(accessTime);
646 
647  return createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_HISTORY, origFile, createHistoryAttributes(url, ftime,
648  null, null,
649  this.getDisplayName(),
650  NetworkUtils.extractDomain(url), user));
651  }
652 
664  private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
665  String[] lineSplit = line.split(","); // NON-NLS
666 
667  String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim();
668  Long ftime = parseTimestamp(accessTime);
669 
670  String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim();
671  String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim());
672  String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim());
673  String url = flipDomain(domain);
674 
675  return createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_COOKIE, origFile, createCookieAttributes(url, null, ftime, null, name, value, this.getDisplayName(), NetworkUtils.extractDomain(url)));
676  }
677 
693  private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
694  BlackboardArtifact bbart = null;
695 
696  String[] lineSplit = line.split(","); // NON-NLS
697  String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)];
698 
699  return bbart;
700  }
701 
717  private BlackboardArtifact getBookmarkArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
718  // split on all commas as long as they are not inbetween quotes
719  String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1);
720 
721  String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)];
722  String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); // NON-NLS
723 
724  if (url.isEmpty()) {
725  return null;
726  }
727 
728  return createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, origFile, createBookmarkAttributes(url, title, null,
729  this.getDisplayName(), NetworkUtils.extractDomain(url)));
730  }
731 
746  private Long parseTimestamp(String timeStr) {
747 
748  // If we had a pattern that worked on the last date, use it again.
749  if (previouslyValidDateFormat != null) {
750  try {
751  return previouslyValidDateFormat.parse(timeStr).getTime() / 1000;
752  } catch (ParseException ex) {
753  // Continue on to format detection
754  }
755  }
756 
757  // Try the default US pattern
758  try {
759  SimpleDateFormat usDateFormat = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); //NON-NLS
760  usDateFormat.setLenient(false); // Fail if month or day are out of range
761  Long epochTime = usDateFormat.parse(timeStr).getTime();
762  previouslyValidDateFormat = usDateFormat;
763  return epochTime / 1000;
764  } catch (ParseException ex) {
765  // Continue on to format detection
766  }
767 
768  // This generally doesn't match the data in the file but can give information on whether
769  // the month or day is first.
770  boolean monthFirstFromLocale = true;
771  String localeDatePattern = ((SimpleDateFormat) DateFormat.getDateInstance(
772  DateFormat.SHORT, Locale.getDefault())).toPattern();
773  if (localeDatePattern.startsWith("d")) {
774  monthFirstFromLocale = false;
775  }
776 
777  // Try to determine if the month or day is first by looking at the data.
778  // If both variations appear valid, use the locale result.
779  boolean monthFirst = monthFirstFromLocale;
780  Pattern pattern = Pattern.compile("^([0-9]{1,2})[^0-9]([0-9]{1,2})");
781  Matcher matcher = pattern.matcher(timeStr);
782  if (matcher.find()) {
783  int firstVal = Integer.parseInt(matcher.group(1));
784  int secondVal = Integer.parseInt(matcher.group(2));
785 
786  if (firstVal > 12) {
787  monthFirst = false;
788  } else if (secondVal > 12) {
789  monthFirst = true;
790  }
791  // Otherwise keep the setting from the locale
792  }
793 
794  // See if the time has AM/PM attached
795  boolean hasAmPm = false;
796  if (timeStr.endsWith("M") || timeStr.endsWith("m")) {
797  hasAmPm = true;
798  }
799 
800  // See if the date appears to use forward slashes. If not, assume '.' is being used.
801  boolean hasSlashes = false;
802  if (timeStr.contains("/")) {
803  hasSlashes = true;
804  }
805 
806  // Make our best guess at the pattern
807  String dateFormatPattern;
808  if (monthFirst) {
809  if (hasSlashes) {
810  dateFormatPattern = "MM/dd/yyyy ";
811  } else {
812  dateFormatPattern = "MM.dd.yyyy ";
813  }
814  } else {
815  if (hasSlashes) {
816  dateFormatPattern = "dd/MM/yyyy ";
817  } else {
818  dateFormatPattern = "dd.MM.yyyy ";
819  }
820  }
821 
822  if (hasAmPm) {
823  dateFormatPattern += "hh:mm:ss a";
824  } else {
825  dateFormatPattern += "HH:mm:ss";
826  }
827 
828  try {
829  SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatPattern); //NON-NLS
830  dateFormat.setLenient(false); // Fail if month or day are out of range
831  Long epochTime = dateFormat.parse(timeStr).getTime();
832  previouslyValidDateFormat = dateFormat;
833  return epochTime / 1000;
834  } catch (ParseException ex) {
835  LOG.log(Level.WARNING, "Timestamp could not be parsed ({0})", timeStr); //NON-NLS
836  return null;
837  }
838  }
839 
847  private String hexToChar(String hexString) {
848  String[] hexValues = hexString.split(" "); // NON-NLS
849  StringBuilder output = new StringBuilder();
850 
851  for (String str : hexValues) {
852  try {
853  int value = Integer.parseInt(str, 16);
854  if (value > 31) { // Ignore non-print characters
855  output.append((char) value);
856  }
857  } catch (NumberFormatException ex) {
858  return null;
859  }
860  }
861 
862  return output.toString();
863  }
864 
877  private String flipDomain(String domain) {
878  if (domain == null || domain.isEmpty()) {
879  return null;
880  }
881 
882  String[] tokens = domain.split("\\."); // NON-NLS
883 
884  if (tokens.length < 2 || tokens.length > 3) {
885  return domain; // don't know what to do, just send it back as is
886  }
887 
888  StringBuilder buf = new StringBuilder();
889  if (tokens.length > 2) {
890  buf.append(tokens[2]);
891  buf.append(".");
892  }
893  buf.append(tokens[1]);
894  buf.append(".");
895  buf.append(tokens[0]);
896 
897  return buf.toString();
898  }
899 
908  private ArrayList<File> getDownloadFiles(File resultDir) throws FileNotFoundException {
909  return getContainerFiles(resultDir, EDGE_TABLE_TYPE_DOWNLOAD);
910  }
911 
921  private ArrayList<File> getHistoryFiles(File resultDir) throws FileNotFoundException {
922  return getContainerFiles(resultDir, EDGE_TABLE_TYPE_HISTORY);
923  }
924 
936  private ArrayList<File> getContainerFiles(File resultDir, String type) throws FileNotFoundException {
937  HashMap<String, ArrayList<String>> idTable = getContainerIDTable(resultDir);
938 
939  ArrayList<String> idList = idTable.get(type);
940  if (idList == null) {
941  return null;
942  }
943 
944  ArrayList<File> fileList = new ArrayList<>();
945  for (String str : idList) {
946  String fileName = EDGE_CONTAINER_FILE_PREFIX + str + EDGE_CONTAINER_FILE_EXT;
947  fileList.add(new File(resultDir, fileName));
948  }
949 
950  return fileList;
951  }
952 
965  private HashMap<String, ArrayList<String>> getContainerIDTable(File resultDir) throws FileNotFoundException {
966 
967  if (containersTable == null) {
968  File containerFile = new File(resultDir, EDGE_CONTAINTERS_FILE_NAME);
969 
970  try (Scanner fileScanner = new Scanner(new FileInputStream(containerFile))) {
971  List<String> headers = null;
972  containersTable = new HashMap<>();
973  int nameIdx = 0;
974  int idIdx = 0;
975  while (fileScanner.hasNext()) {
976  String line = fileScanner.nextLine();
977  if (headers == null) {
978  headers = Arrays.asList(line.toLowerCase().split(","));
979  nameIdx = headers.indexOf(EDGE_HEAD_NAME);
980  idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID);
981  } else {
982  String[] row = line.split(","); // NON-NLS
983  String name = row[nameIdx];
984  String id = row[idIdx];
985 
986  ArrayList<String> idList = containersTable.get(name);
987  if (idList == null) {
988  idList = new ArrayList<>();
989  containersTable.put(name, idList);
990  }
991 
992  idList.add(id);
993  }
994  }
995  }
996  }
997 
998  return containersTable;
999  }
1000 
1004  private void clearContainerTable() {
1005  containersTable = null;
1006  }
1007 }
List< AbstractFile > findFiles(String fileName)

Copyright © 2012-2022 Basis Technology. Generated on: Tue Aug 1 2023
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.