19 package org.sleuthkit.autopsy.modules.leappanalyzers;
21 import java.io.BufferedReader;
23 import java.io.FileNotFoundException;
24 import java.io.FileReader;
25 import java.io.IOException;
26 import java.io.UncheckedIOException;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.text.ParseException;
30 import java.text.SimpleDateFormat;
31 import java.util.List;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.Collections;
36 import java.util.Date;
37 import java.util.HashMap;
38 import static java.util.Locale.US;
40 import java.util.logging.Level;
41 import java.util.stream.Collectors;
42 import java.util.stream.IntStream;
43 import java.util.stream.Stream;
44 import javax.xml.parsers.DocumentBuilder;
45 import javax.xml.parsers.DocumentBuilderFactory;
46 import javax.xml.parsers.ParserConfigurationException;
47 import org.apache.commons.collections4.MapUtils;
48 import org.apache.commons.io.FilenameUtils;
49 import org.openide.util.NbBundle;
60 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
64 import org.w3c.dom.Document;
65 import org.w3c.dom.NamedNodeMap;
66 import org.w3c.dom.NodeList;
67 import org.xml.sax.SAXException;
92 TsvColumn(String attributeName, String columnName,
boolean required) {
101 String getAttributeName() {
108 String getColumnName() {
115 boolean isRequired() {
133 this.tsvFiles =
new HashMap<>();
134 this.tsvFileArtifacts =
new HashMap<>();
135 this.tsvFileArtifactComments =
new HashMap<>();
136 this.tsvFileAttributes =
new HashMap<>();
147 "LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.",
148 "LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory.",
149 "LeappFileProcessor.starting.Leapp=Starting Leapp",
150 "LeappFileProcessor.running.Leapp=Running Leapp",
151 "LeappFileProcessor.has.run=Leapp",
152 "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
153 "LeappFileProcessor.completed=Leapp Processing Completed",
154 "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"})
157 List<String> LeappTsvOutputFiles =
findTsvFiles(moduleOutputPath);
159 }
catch (IOException | IngestModuleException ex) {
160 logger.log(Level.SEVERE, String.format(
"Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex);
170 List<String> LeappTsvOutputFiles =
findTsvFiles(moduleOutputPath);
172 }
catch (IngestModuleException ex) {
173 logger.log(Level.SEVERE, String.format(
"Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex);
184 private List<String>
findTsvFiles(Path LeappOutputDir)
throws IngestModuleException {
185 List<String> allTsvFiles =
new ArrayList<>();
186 List<String> foundTsvFiles =
new ArrayList<>();
188 try (Stream<Path> walk = Files.walk(LeappOutputDir)) {
190 allTsvFiles = walk.map(x -> x.toString())
191 .filter(f -> f.toLowerCase().endsWith(
".tsv")).collect(Collectors.toList());
193 for (String tsvFile : allTsvFiles) {
194 if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile.toLowerCase()))) {
195 foundTsvFiles.add(tsvFile);
199 }
catch (IOException | UncheckedIOException e) {
200 throw new IngestModuleException(Bundle.LeappFileProcessor_error_reading_Leapp_directory() + LeappOutputDir.toString(), e);
203 return foundTsvFiles;
216 private void processLeappFiles(List<String> LeappFilesToProcess, AbstractFile LeappImageFile)
throws FileNotFoundException, IOException, IngestModuleException {
217 List<BlackboardArtifact> bbartifacts =
new ArrayList<>();
219 for (String LeappFileName : LeappFilesToProcess) {
220 String fileName = FilenameUtils.getName(LeappFileName);
221 File LeappFile =
new File(LeappFileName);
222 if (tsvFileAttributes.containsKey(fileName)) {
223 List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
227 processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, LeappImageFile);
229 }
catch (TskCoreException ex) {
230 throw new IngestModuleException(String.format(
"Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex);
236 if (!bbartifacts.isEmpty()) {
237 postArtifacts(bbartifacts);
251 private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource)
throws IngestModuleException {
252 List<BlackboardArtifact> bbartifacts =
new ArrayList<>();
254 for (String LeappFileName : LeappFilesToProcess) {
255 String fileName = FilenameUtils.getName(LeappFileName);
256 File LeappFile =
new File(LeappFileName);
257 if (tsvFileAttributes.containsKey(fileName)) {
258 List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
259 BlackboardArtifact.Type artifactType = null;
262 }
catch (TskCoreException ex) {
263 logger.log(Level.SEVERE, String.format(
"Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex);
266 if (artifactType == null) {
271 processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource);
272 }
catch (TskCoreException | IOException ex) {
273 logger.log(Level.SEVERE, String.format(
"Error processing file at %s", LeappFile.toString()), ex);
279 if (!bbartifacts.isEmpty()) {
280 postArtifacts(bbartifacts);
285 private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName, BlackboardArtifact.Type artifactType,
286 List<BlackboardArtifact> bbartifacts, Content dataSource)
throws FileNotFoundException, IOException, IngestModuleException,
289 if (LeappFile == null || !LeappFile.exists() || fileName == null) {
290 logger.log(Level.WARNING, String.format(
"Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() :
"<null>"));
292 }
else if (attrList == null || artifactType == null || dataSource == null) {
293 logger.log(Level.WARNING, String.format(
"attribute list, artifact type or dataSource not provided for %s", LeappFile == null ? LeappFile.toString() :
"<null>"));
297 try (BufferedReader reader =
new BufferedReader(
new FileReader(LeappFile))) {
298 String header = reader.readLine();
300 if (header != null) {
302 String line = reader.readLine();
303 while (line != null) {
304 Collection<BlackboardAttribute> bbattributes =
processReadLine(line, columnNumberToProcess, fileName);
306 if (!bbattributes.isEmpty() && !blkBoard.artifactExists(dataSource, BlackboardArtifact.ARTIFACT_TYPE.fromID(artifactType.getTypeID()), bbattributes)) {
308 if (bbartifact != null) {
309 bbartifacts.add(bbartifact);
313 line = reader.readLine();
329 private Collection<BlackboardAttribute>
processReadLine(String line, Map<Integer, String> columnNumberToProcess, String fileName)
throws IngestModuleException {
330 if (MapUtils.isEmpty(columnNumberToProcess)) {
331 return Collections.emptyList();
332 }
else if (line == null) {
333 logger.log(Level.WARNING,
"Line is null. Returning empty list for attributes.");
334 return Collections.emptyList();
337 String[] columnValues;
343 Integer maxColumnNumber = Collections.max(columnNumberToProcess.keySet());
344 if ((maxColumnNumber > line.split(
"\\t").length) || (columnNumberToProcess.size() > line.split(
"\\t").length)) {
345 columnValues = Arrays.copyOf(line.split(
"\\t"), maxColumnNumber + 1);
347 columnValues = line.split(
"\\t");
350 Collection<BlackboardAttribute> bbattributes =
new ArrayList<BlackboardAttribute>();
352 for (Map.Entry<Integer, String> columnToProcess : columnNumberToProcess.entrySet()) {
353 Integer columnNumber = columnToProcess.getKey();
354 String attributeName = columnToProcess.getValue();
356 if (columnValues[columnNumber] != null) {
359 if (attributeType == null) {
362 String attrType = attributeType.getValueType().getLabel().toUpperCase();
363 checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType, fileName);
364 }
catch (TskCoreException ex) {
365 throw new IngestModuleException(String.format(
"Error getting Attribute type for Attribute Name %s", attributeName), ex);
370 if (tsvFileArtifactComments.containsKey(fileName)) {
371 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, tsvFileArtifactComments.get(fileName)));
378 private void checkAttributeType(Collection<BlackboardAttribute> bbattributes, String attrType, String[] columnValues,
int columnNumber, BlackboardAttribute.Type attributeType,
381 if (columnValues == null || columnNumber < 0 || columnNumber > columnValues.length || columnValues[columnNumber] == null) {
382 logger.log(Level.WARNING, String.format(
"Unable to determine column value at index %d in columnValues: %s",
384 columnValues == null ?
"<null>" :
"[" + String.join(
", ", columnValues) +
"]"));
388 String columnValue = columnValues[columnNumber];
390 if (attrType.matches(
"STRING")) {
391 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, columnValue));
392 }
else if (attrType.matches(
"INTEGER")) {
394 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValue)));
395 }
catch (NumberFormatException ex) {
396 logger.log(Level.WARNING, String.format(
"Unable to format %s as an integer.", columnValue), ex);
398 }
else if (attrType.matches(
"LONG")) {
400 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValue)));
401 }
catch (NumberFormatException ex) {
402 logger.log(Level.WARNING, String.format(
"Unable to format %s as an long.", columnValue), ex);
404 }
else if (attrType.matches(
"DOUBLE")) {
406 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValue)));
407 }
catch (NumberFormatException ex) {
408 logger.log(Level.WARNING, String.format(
"Unable to format %s as an double.", columnValue), ex);
410 }
else if (attrType.matches(
"BYTE")) {
412 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValue)));
413 }
catch (NumberFormatException ex) {
414 logger.log(Level.WARNING, String.format(
"Unable to format %s as an byte.", columnValue), ex);
416 }
else if (attrType.matches(
"DATETIME")) {
418 SimpleDateFormat dateFormat =
new SimpleDateFormat(
"yyyy-MM-d HH:mm:ss", US);
419 Long dateLong = Long.valueOf(0);
421 Date newDate = dateFormat.parse(columnValue);
422 dateLong = newDate.getTime() / 1000;
423 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, dateLong));
424 }
catch (ParseException ex) {
427 logger.log(Level.WARNING, String.format(
"Failed to parse date/time %s for attribute type %s in file %s.", columnValue, attributeType.getDisplayName(), fileName));
429 }
else if (attrType.matches(
"JSON")) {
431 bbattributes.add(
new BlackboardAttribute(attributeType, MODULE_NAME, columnValue));
434 logger.log(Level.WARNING, String.format(
"Attribute Type %s not defined.", attrType));
452 String[] columnNames = line.split(
"\\t");
453 HashMap<Integer, String> columnsToProcess =
new HashMap<>();
455 Integer columnPosition = 0;
456 for (String columnName : columnNames) {
458 String cleanColumnName = columnName.trim().replaceAll(
"[^\\n\\r\\t\\p{Print}]",
"");
460 if (cleanColumnName.equalsIgnoreCase(tsvColumn.getColumnName())) {
461 columnsToProcess.put(columnPosition, tsvColumn.getAttributeName());
468 if (columnsToProcess.size() != attrList.size()) {
469 String missingColumns = IntStream.range(0, attrList.size())
470 .filter((idx) -> !columnsToProcess.containsKey(attrList.get(idx).getAttributeName()))
471 .mapToObj((idx) -> String.format(
"'%s'", attrList.get(idx).getColumnName() == null ?
"<null>" : attrList.get(idx).getColumnName()))
472 .collect(Collectors.joining(
", "));
474 logger.log(Level.WARNING, String.format(
"Columns size expected not found in file %s based on xml from %s. Column Keys Missing = [%s]; Header Line = '%s'.",
475 this.xmlFile == null ?
"<null>" :
this.xmlFile,
481 return columnsToProcess;
485 "LeappFileProcessor.cannot.load.artifact.xml=Cannor load xml artifact file.",
486 "LeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.",
487 "LeappFileProcessor_cannotParseXml=Cannot Parse XML file.",
488 "LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact",
489 "LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts."
499 File f =
new File(path);
500 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
501 DocumentBuilder db = dbf.newDocumentBuilder();
502 xmlinput = db.parse(f);
504 }
catch (IOException e) {
505 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_load_artifact_xml() + e.getLocalizedMessage(), e);
506 }
catch (ParserConfigurationException pce) {
507 throw new IngestModuleException(Bundle.LeappFileProcessor_cannotBuildXmlParser() + pce.getLocalizedMessage(), pce);
508 }
catch (SAXException sxe) {
509 throw new IngestModuleException(Bundle.LeappFileProcessor_cannotParseXml() + sxe.getLocalizedMessage(), sxe);
520 NodeList nlist = xmlinput.getElementsByTagName(
"FileName");
522 for (
int i = 0; i < nlist.getLength(); i++) {
523 NamedNodeMap nnm = nlist.item(i).getAttributes();
524 tsvFiles.put(nnm.getNamedItem(
"filename").getNodeValue().toLowerCase(), nnm.getNamedItem(
"description").getNodeValue());
532 NodeList artifactNlist = xmlinput.getElementsByTagName(
"ArtifactName");
533 for (
int k = 0; k < artifactNlist.getLength(); k++) {
534 NamedNodeMap nnm = artifactNlist.item(k).getAttributes();
535 String artifactName = nnm.getNamedItem(
"artifactname").getNodeValue();
536 String comment = nnm.getNamedItem(
"comment").getNodeValue();
537 String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem(
"filename").getNodeValue();
539 BlackboardArtifact.Type foundArtifactType = null;
542 }
catch (TskCoreException ex) {
543 logger.log(Level.SEVERE, String.format(
"There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex);
546 if (foundArtifactType == null) {
547 logger.log(Level.SEVERE, String.format(
"No known artifact mapping found for [artifact: %s, %s]",
551 tsvFileArtifacts.put(parentName, artifactName);
553 if (!comment.toLowerCase().matches(
"null")) {
554 tsvFileArtifactComments.put(parentName, comment);
561 return String.format(
"file: %s, filename: %s",
562 this.xmlFile == null ?
"<null>" : this.xmlFile,
563 fileName == null ?
"<null>" : fileName);
567 return String.format(
"attribute: %s %s",
568 attributeName == null ?
"<null>" : attributeName,
574 NodeList attributeNlist = xmlinput.getElementsByTagName(
"AttributeName");
575 for (
int k = 0; k < attributeNlist.getLength(); k++) {
576 NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
577 String attributeName = nnm.getNamedItem(
"attributename").getNodeValue();
579 if (!attributeName.toLowerCase().matches(
"null")) {
580 String columnName = nnm.getNamedItem(
"columnName").getNodeValue();
581 String required = nnm.getNamedItem(
"required").getNodeValue();
582 String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem(
"filename").getNodeValue();
584 BlackboardAttribute.Type foundAttrType = null;
587 }
catch (TskCoreException ex) {
588 logger.log(Level.SEVERE, String.format(
"There was an issue that arose while trying to fetch attribute type for %s.", attributeName), ex);
591 if (foundAttrType == null) {
592 logger.log(Level.SEVERE, String.format(
"No known attribute mapping found for [%s]",
getXmlAttrIdentifier(parentName, attributeName)));
595 if (required != null && required.compareToIgnoreCase(
"yes") != 0 && required.compareToIgnoreCase(
"no") != 0) {
596 logger.log(Level.SEVERE, String.format(
"Required value %s did not match 'yes' or 'no' for [%s]",
600 if (columnName == null) {
601 logger.log(Level.SEVERE, String.format(
"No column name provided for [%s]",
getXmlAttrIdentifier(parentName, attributeName)));
602 }
else if (columnName.trim().length() != columnName.length()) {
603 logger.log(Level.SEVERE, String.format(
"Column name '%s' starts or ends with whitespace for [%s]", columnName,
getXmlAttrIdentifier(parentName, attributeName)));
604 }
else if (columnName.matches(
"[^ \\S]")) {
605 logger.log(Level.SEVERE, String.format(
"Column name '%s' contains invalid characters [%s]", columnName,
getXmlAttrIdentifier(parentName, attributeName)));
609 attributeName.toLowerCase(),
610 columnName.toLowerCase(),
611 "yes".compareToIgnoreCase(required) == 0);
613 if (tsvFileAttributes.containsKey(parentName)) {
614 List<TsvColumn> attrList = tsvFileAttributes.get(parentName);
615 attrList.add(thisCol);
616 tsvFileAttributes.replace(parentName, attrList);
618 List<TsvColumn> attrList =
new ArrayList<>();
619 attrList.add(thisCol);
620 tsvFileAttributes.put(parentName, attrList);
641 BlackboardArtifact bbart = abstractFile.newArtifact(type);
642 bbart.addAttributes(bbattributes);
644 }
catch (TskException ex) {
645 logger.log(Level.WARNING, Bundle.LeappFileProcessor_error_creating_new_artifacts(), ex);
664 BlackboardArtifact bbart = dataSource.newArtifact(type);
665 bbart.addAttributes(bbattributes);
667 }
catch (TskException ex) {
668 logger.log(Level.WARNING, Bundle.LeappFileProcessor_error_creating_new_artifacts(), ex);
679 void postArtifacts(Collection<BlackboardArtifact> artifacts) {
680 if (artifacts == null || artifacts.isEmpty()) {
685 Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifacts(artifacts, MODULE_NAME);
686 }
catch (Blackboard.BlackboardException ex) {
687 logger.log(Level.SEVERE, Bundle.LeappFileProcessor_postartifacts_error(), ex);
String getXmlFileIdentifier(String fileName)
BlackboardArtifact createArtifactWithAttributes(int type, Content dataSource, Collection< BlackboardAttribute > bbattributes)
static final String MODULE_NAME
List< String > findTsvFiles(Path LeappOutputDir)
void checkAttributeType(Collection< BlackboardAttribute > bbattributes, String attrType, String[] columnValues, int columnNumber, BlackboardAttribute.Type attributeType, String fileName)
void processLeappFiles(List< String > LeappFilesToProcess, AbstractFile LeappImageFile)
BlackboardArtifact createArtifactWithAttributes(int type, AbstractFile abstractFile, Collection< BlackboardAttribute > bbattributes)
static final Logger logger
Collection< BlackboardAttribute > processReadLine(String line, Map< Integer, String > columnNumberToProcess, String fileName)
void getArtifactNode(Document xmlinput)
final String attributeName
final Map< String, String > tsvFileArtifacts
final Map< String, String > tsvFiles
ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath)
ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile)
void processFile(File LeappFile, List< TsvColumn > attrList, String fileName, BlackboardArtifact.Type artifactType, List< BlackboardArtifact > bbartifacts, Content dataSource)
SleuthkitCase getSleuthkitCase()
LeappFileProcessor(String xmlFile)
final Map< String, String > tsvFileArtifactComments
final Map< String, List< TsvColumn > > tsvFileAttributes
static Case getCurrentCase()
synchronized static Logger getLogger(String name)
static Case getCurrentCaseThrows()
void processLeappFiles(List< String > LeappFilesToProcess, Content dataSource)
void getAttributeNodes(Document xmlinput)
String getXmlAttrIdentifier(String fileName, String attributeName)
Map< Integer, String > findColumnsToProcess(String fileName, String line, List< TsvColumn > attrList)
void getFileNode(Document xmlinput)