19 package org.sleuthkit.autopsy.modules.leappanalyzers;
21 import com.fasterxml.jackson.databind.MappingIterator;
22 import com.fasterxml.jackson.dataformat.csv.CsvMapper;
23 import com.fasterxml.jackson.dataformat.csv.CsvParser;
24 import com.fasterxml.jackson.dataformat.csv.CsvSchema;
25 import com.google.common.collect.ImmutableMap;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.io.UncheckedIOException;
30 import java.nio.file.Files;
31 import java.nio.file.Path;
32 import java.text.DateFormat;
33 import java.text.ParseException;
34 import java.text.SimpleDateFormat;
35 import java.util.List;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import static java.util.Locale.US;
45 import java.util.logging.Level;
46 import java.util.stream.Collectors;
47 import java.util.stream.IntStream;
48 import java.util.stream.Stream;
49 import javax.xml.parsers.DocumentBuilder;
50 import javax.xml.parsers.DocumentBuilderFactory;
51 import javax.xml.parsers.ParserConfigurationException;
52 import org.apache.commons.collections4.CollectionUtils;
53 import org.apache.commons.collections4.MapUtils;
54 import org.apache.commons.io.FilenameUtils;
55 import org.apache.commons.lang3.StringUtils;
56 import org.openide.util.NbBundle;
69 import org.
sleuthkit.datamodel.Blackboard.BlackboardException;
72 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
77 import org.
sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
78 import org.
sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CallMediaType;
79 import org.
sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection;
80 import org.
sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus;
81 import org.
sleuthkit.datamodel.blackboardutils.GeoArtifactsHelper;
82 import org.
sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints;
83 import org.
sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints.TrackPoint;
84 import org.
sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints;
85 import org.
sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints.Waypoint;
86 import org.
sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments;
87 import org.
sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment;
88 import org.w3c.dom.Document;
89 import org.w3c.dom.NamedNodeMap;
90 import org.w3c.dom.NodeList;
91 import org.xml.sax.SAXException;
125 BlackboardAttribute.Type getAttributeType() {
132 String getColumnName() {
139 boolean isRequired() {
154 .put(
"TSK_IP_DHCP",
"DHCP Information")
158 .put(
"zapya.tsv",
"message")
159 .put(
"sms messages.tsv",
"message")
160 .put(
"mms messages.tsv",
"message")
161 .put(
"viber - messages.tsv",
"message")
162 .put(
"viber - contacts.tsv",
"contact")
163 .put(
"viber - call logs.tsv",
"calllog")
164 .put(
"xender file transfer - messages.tsv",
"message")
165 .put(
"xender file transfer - contacts.tsv",
"contact")
166 .put(
"whatsapp - contacts.tsv",
"contact")
167 .put(
"whatsapp - group call logs.tsv",
"calllog")
168 .put(
"whatsapp - single call logs.tsv",
"calllog")
169 .put(
"whatsapp - messages logs.tsv",
"message")
170 .put(
"shareit file transfer.tsv",
"message")
171 .put(
"tangomessages messages.tsv",
"message")
172 .put(
"contacts.tsv",
"contact")
173 .put(
"imo - accountid.tsv",
"contact")
174 .put(
"imo - messages.tsv",
"message")
175 .put(
"textnow - contacts.tsv",
"contact")
176 .put(
"textnow - messages.tsv",
"message")
177 .put(
"line - messages.tsv",
"message")
178 .put(
"line - contacts.tsv",
"contact")
179 .put(
"line - calllogs.tsv",
"calllog")
180 .put(
"skype - messages logs.tsv",
"message")
181 .put(
"skype - contacts.tsv",
"contact")
182 .put(
"skype - call logs.tsv",
"calllog")
183 .put(
"facebook messenger - chats.tsv",
"message")
184 .put(
"facebook messenger - contacts.tsv",
"contact")
185 .put(
"facebook messenger - calls.tsv",
"calllog")
186 .put(
"call logs2.tsv",
"calllog")
187 .put(
"call logs.tsv",
"calllog")
188 .put(
"oruxmaps tracks.tsv",
"trackpoint")
189 .put(
"google map locations.tsv",
"route")
190 .put(
"Contacts.tsv",
"contact")
191 .put(
"sms - imessage.tsv",
"message")
192 .put(
"call history.tsv",
"calllog")
198 this.tsvFiles =
new HashMap<>();
200 this.tsvFileArtifactComments =
new HashMap<>();
201 this.tsvFileAttributes =
new HashMap<>();
214 "LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.",
215 "LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory.",
216 "LeappFileProcessor.starting.Leapp=Starting Leapp",
217 "LeappFileProcessor.running.Leapp=Running Leapp",
218 "LeappFileProcessor.has.run=Leapp",
219 "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
220 "LeappFileProcessor.completed=Leapp Processing Completed",
221 "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"})
224 List<String> LeappTsvOutputFiles =
findTsvFiles(moduleOutputPath);
226 }
catch (IOException | IngestModuleException ex) {
227 logger.log(Level.SEVERE, String.format(
"Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex);
237 List<String> LeappTsvOutputFiles =
findTsvFiles(moduleOutputPath);
239 }
catch (IngestModuleException ex) {
240 logger.log(Level.SEVERE, String.format(
"Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex);
251 private List<String>
findTsvFiles(Path LeappOutputDir)
throws IngestModuleException {
252 List<String> allTsvFiles =
new ArrayList<>();
253 List<String> foundTsvFiles =
new ArrayList<>();
255 try (Stream<Path> walk = Files.walk(LeappOutputDir)) {
257 allTsvFiles = walk.map(x -> x.toString())
258 .filter(f -> f.toLowerCase().endsWith(
".tsv")).collect(Collectors.toList());
260 for (String tsvFile : allTsvFiles) {
261 if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile.toLowerCase()))) {
262 foundTsvFiles.add(tsvFile);
266 }
catch (IOException | UncheckedIOException e) {
267 throw new IngestModuleException(Bundle.LeappFileProcessor_error_reading_Leapp_directory() + LeappOutputDir.toString(), e);
270 return foundTsvFiles;
283 private void processLeappFiles(List<String> LeappFilesToProcess, AbstractFile LeappImageFile)
throws FileNotFoundException, IOException, IngestModuleException {
284 List<BlackboardArtifact> bbartifacts =
new ArrayList<>();
286 for (String LeappFileName : LeappFilesToProcess) {
287 String fileName = FilenameUtils.getName(LeappFileName);
288 File LeappFile =
new File(LeappFileName);
289 if (tsvFileAttributes.containsKey(fileName)) {
290 BlackboardArtifact.Type artifactType = null;
292 List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
294 processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, LeappImageFile);
295 }
catch (TskCoreException ex) {
296 throw new IngestModuleException(String.format(
"Error getting Blackboard Artifact Type for %s", artifactType == null ?
"<null>" : artifactType.toString()), ex);
301 if (!bbartifacts.isEmpty()) {
302 postArtifacts(bbartifacts);
316 private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource)
throws IngestModuleException {
317 List<BlackboardArtifact> bbartifacts =
new ArrayList<>();
319 for (String LeappFileName : LeappFilesToProcess) {
320 String fileName = FilenameUtils.getName(LeappFileName);
321 File LeappFile =
new File(LeappFileName);
322 if (tsvFileAttributes.containsKey(fileName)) {
323 List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
327 processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource);
328 }
catch (TskCoreException | IOException ex) {
329 logger.log(Level.SEVERE, String.format(
"Error processing file at %s", LeappFile.toString()), ex);
335 if (!bbartifacts.isEmpty()) {
336 postArtifacts(bbartifacts);
341 private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName, BlackboardArtifact.Type artifactType,
342 List<BlackboardArtifact> bbartifacts, Content dataSource)
throws FileNotFoundException, IOException, IngestModuleException,
345 String trackpointSegmentName = null;
346 GeoTrackPoints pointList =
new GeoTrackPoints();
347 AbstractFile geoAbstractFile = null;
349 if (LeappFile == null || !LeappFile.exists() || fileName == null) {
350 logger.log(Level.WARNING, String.format(
"Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() :
"<null>"));
352 }
else if (attrList == null || artifactType == null || dataSource == null) {
353 logger.log(Level.WARNING, String.format(
"attribute list, artifact type or dataSource not provided for %s", LeappFile == null ? LeappFile.toString() :
"<null>"));
358 try (MappingIterator<List<String>> iterator =
new CsvMapper()
359 .enable(CsvParser.Feature.WRAP_AS_ARRAY)
360 .readerFor(List.class)
361 .with(CsvSchema.emptySchema().withColumnSeparator(
'\t'))
362 .readValues(LeappFile)) {
364 if (iterator.hasNext()) {
365 List<String> headerItems = iterator.next();
366 Map<String, Integer> columnIndexes = IntStream.range(0, headerItems.size())
367 .mapToObj(idx -> idx)
368 .collect(Collectors.toMap(
369 idx -> headerItems.get(idx) == null ? null : headerItems.get(idx).trim().toLowerCase(),
371 (val1, val2) -> val1));
374 while (iterator.hasNext()) {
375 List<String> columnItems = iterator.next();
376 Collection<BlackboardAttribute> bbattributes =
processReadLine(columnItems, columnIndexes, attrList, fileName, lineNum);
378 if (!bbattributes.isEmpty()) {
379 switch (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(),
"norelationship").toLowerCase()) {
393 geoAbstractFile =
createTrackpoint(bbattributes, dataSource, fileName, trackpointSegmentName, pointList);
397 if (bbartifact != null) {
398 bbartifacts.add(bbartifact);
410 if (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(),
"norelationship").toLowerCase() ==
"trackpoint") {
415 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex);
421 "LeappFileProcessor.cannot.create.waypoint.relationship=Cannot create TSK_WAYPOINT artifact.",
424 private void createRoute (Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName)
throws IngestModuleException {
426 Double startLatitude = Double.valueOf(0);
427 Double startLongitude = Double.valueOf(0);
428 Double endLatitude = Double.valueOf(0);
429 Double endLongitude = Double.valueOf(0);
430 Double zeroValue = Double.valueOf(0);
431 String destinationName =
"";
432 String locationName =
"";
433 Long dateTime = Long.valueOf(0);
434 Collection<BlackboardAttribute> otherAttributes =
new ArrayList<>();
435 String sourceFile = null;
436 AbstractFile absFile = null;
440 for (BlackboardAttribute bba : bbattributes) {
441 switch (bba.getAttributeType().getTypeName()) {
442 case "TSK_GEO_LATITUDE_START":
443 startLatitude = bba.getValueDouble();
445 case "TSK_GEO_LONGITUDE_START":
446 startLongitude = bba.getValueDouble();
448 case "TSK_GEO_LATITUDE_END":
449 startLatitude = bba.getValueDouble();
451 case "TSK_GEO_LONGITUDE_END":
452 startLongitude = bba.getValueDouble();
455 dateTime = bba.getValueLong();
458 destinationName = bba.getValueString();
461 locationName = bba.getValueString();
463 case "TSK_TEXT_FILE":
464 sourceFile = bba.getValueString();
467 comment = bba.getValueString();
470 otherAttributes.add(bba);
475 if (absFile == null) {
476 absFile = (AbstractFile) dataSource;
478 GeoWaypoints waypointList =
new GeoWaypoints();
479 waypointList.addPoint(
new Waypoint(startLatitude, startLongitude, zeroValue,
""));
480 waypointList.addPoint(
new Waypoint(endLatitude, endLongitude, zeroValue, locationName));
484 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_waypoint_relationship() + ex.getLocalizedMessage(), ex);
491 "LeappFileProcessor.cannot.create.trackpoint.relationship=Cannot create TSK_TRACK_POINT artifact.",
494 private AbstractFile
createTrackpoint(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList)
throws IngestModuleException {
496 Double latitude = Double.valueOf(0);
497 Double longitude = Double.valueOf(0);
498 Double altitude = Double.valueOf(0);
499 Double zeroValue = Double.valueOf(0);
500 String segmentName = null;
501 Long dateTime = Long.valueOf(0);
502 Collection<BlackboardAttribute> otherAttributes =
new ArrayList<>();
503 String sourceFile = null;
504 String comment = null;
505 AbstractFile absFile = null;
508 for (BlackboardAttribute bba : bbattributes) {
509 switch (bba.getAttributeType().getTypeName()) {
510 case "TSK_GEO_LATITUDE":
511 latitude = bba.getValueDouble();
513 case "TSK_GEO_LONGITUDE":
514 longitude = bba.getValueDouble();
516 case "TSK_GEO_ALTITUDE":
517 altitude = bba.getValueDouble();
520 dateTime = bba.getValueLong();
523 segmentName = bba.getValueString();
525 case "TSK_TEXT_FILE":
526 sourceFile = bba.getValueString();
529 comment = bba.getValueString();
530 otherAttributes.add(bba);
533 otherAttributes.add(bba);
538 if (absFile == null) {
539 absFile = (AbstractFile) dataSource;
541 if ((trackpointSegmentName == null) || (trackpointSegmentName == segmentName)) {
542 trackpointSegmentName = segmentName;
543 pointList.addPoint(
new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime));
546 trackpointSegmentName = segmentName;
547 pointList =
new GeoTrackPoints();
548 pointList.addPoint(
new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime));
552 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_trackpoint_relationship() + ex.getLocalizedMessage(), ex);
561 "LeappFileProcessor.cannot.create.message.relationship=Cannot create TSK_MESSAGE Relationship.",
564 private void createMessageRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName)
throws IngestModuleException {
566 String messageType = null;
567 String alternateId = null;
568 CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN;
569 String senderId = null;
570 String receipentId = null;
571 String[] receipentIdList = null;
572 Long dateTime = Long.valueOf(0);
573 MessageReadStatus messageStatus = MessageReadStatus.UNKNOWN;
574 String subject = null;
575 String messageText = null;
576 String threadId = null;
577 List<BlackboardAttribute> otherAttributes =
new ArrayList<>();
578 List<FileAttachment> fileAttachments =
new ArrayList<>();
579 String sourceFile = null;
580 MessageAttachments messageAttachments = null;
583 for (BlackboardAttribute bba : bbattributes) {
584 switch (bba.getAttributeType().getTypeName()) {
585 case "TSK_DIRECTION":
586 if (bba.getValueString().toLowerCase().equals(
"outgoing")) {
587 communicationDirection = CommunicationDirection.OUTGOING;
588 }
else if (bba.getValueString().toLowerCase().equals(
"incoming")) {
589 communicationDirection = CommunicationDirection.INCOMING;
592 case "TSK_PHONE_NUMBER_FROM":
593 if (!bba.getValueString().isEmpty()) {
594 senderId = bba.getValueString();
597 case "TSK_PHONE_NUMBER_TO":
598 if (!bba.getValueString().isEmpty()) {
599 receipentIdList = bba.getValueString().split(
",", 0);
603 dateTime = bba.getValueLong();
606 messageType = bba.getValueString();
608 case "TSK_ATTACHMENTS":
609 if (!bba.getValueString().isEmpty()) {
613 case "TSK_TEXT_FILE":
614 sourceFile = bba.getValueString();
616 case "TSK_READ_STATUS":
617 if (bba.getValueInt() == 1 ) {
618 messageStatus = MessageReadStatus.READ;
620 messageStatus = MessageReadStatus.UNREAD;
624 messageText = bba.getValueString();
627 subject = bba.getValueString();
630 alternateId = bba.getValueString();
631 otherAttributes.add(bba);
634 otherAttributes.add(bba);
639 if (absFile == null) {
640 absFile = (AbstractFile) dataSource;
642 CommunicationArtifactsHelper accountArtifact;
644 if (alternateId == null) {
649 moduleName, absFile, accountType, accountType, alternateId);
651 BlackboardArtifact messageArtifact = accountArtifact.addMessage(messageType, communicationDirection, senderId,
652 receipentId, dateTime, messageStatus, subject,
653 messageText, threadId, otherAttributes);
654 if (!fileAttachments.isEmpty()) {
655 messageAttachments =
new MessageAttachments(fileAttachments,
new ArrayList<>());
656 accountArtifact.addAttachments(messageArtifact, messageAttachments);
659 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex);
665 "LeappFileProcessor.cannot.create.contact.relationship=Cannot create TSK_CONTACT Relationship.",
667 private void createContactRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName)
throws IngestModuleException {
669 String alternateId = null;
670 String contactName = null;
671 String phoneNumber = null;
672 String homePhoneNumber = null;
673 String mobilePhoneNumber = null;
674 String emailAddr = null;
675 List<BlackboardAttribute> otherAttributes =
new ArrayList<>();
676 String sourceFile = null;
679 for (BlackboardAttribute bba : bbattributes) {
680 switch (bba.getAttributeType().getTypeName()) {
681 case "TSK_PHONE_NUMBER":
682 if (!bba.getValueString().isEmpty()) {
683 phoneNumber = bba.getValueString();
687 if (!bba.getValueString().isEmpty()) {
688 contactName = bba.getValueString();
691 case "TSK_TEXT_FILE":
692 sourceFile = bba.getValueString();
694 case "TSK_PHONE_NUMBER_HOME":
695 homePhoneNumber = bba.getValueString();
697 case "TSK_PHONE_NUMBER_MOBILE":
698 mobilePhoneNumber = bba.getValueString();
701 emailAddr = bba.getValueString();
704 alternateId = bba.getValueString();
705 otherAttributes.add(bba);
708 otherAttributes.add(bba);
713 if (absFile == null) {
714 absFile = (AbstractFile) dataSource;
717 if (accountType != null) {
719 CommunicationArtifactsHelper accountArtifact;
720 if (alternateId == null) {
725 moduleName, absFile, accountType, accountType, alternateId);
727 BlackboardArtifact messageArtifact = accountArtifact.addContact(contactName, phoneNumber, homePhoneNumber, mobilePhoneNumber, emailAddr, otherAttributes);
730 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_contact_relationship() + ex.getLocalizedMessage(), ex);
735 "LeappFileProcessor.cannot.create.calllog.relationship=Cannot create TSK_CALLLOG Relationship.",
738 private void createCalllogRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName)
throws IngestModuleException {
740 String callerId = null;
741 String alternateId = null;
742 List<String> calleeId = Arrays.asList();
743 CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN;
744 Long startDateTime = Long.valueOf(0);
745 Long endDateTime = Long.valueOf(0);
746 CallMediaType mediaType = CallMediaType.UNKNOWN;
747 List<BlackboardAttribute> otherAttributes =
new ArrayList<>();
748 String sourceFile = null;
751 for (BlackboardAttribute bba : bbattributes) {
752 switch (bba.getAttributeType().getTypeName()) {
753 case "TSK_TEXT_FILE":
754 sourceFile = bba.getValueString();
756 case "TSK_DATETIME_START":
757 startDateTime = bba.getValueLong();
759 case "TSK_DATETIME_END":
760 startDateTime = bba.getValueLong();
762 case "TSK_DIRECTION":
763 if (bba.getValueString().toLowerCase().equals(
"outgoing")) {
764 communicationDirection = CommunicationDirection.OUTGOING;
765 }
else if (bba.getValueString().toLowerCase().equals(
"incoming")) {
766 communicationDirection = CommunicationDirection.INCOMING;
769 case "TSK_PHONE_NUMBER_FROM":
770 if (!bba.getValueString().isEmpty()) {
771 callerId = bba.getValueString();
774 case "TSK_PHONE_NUMBER_TO":
775 if (!bba.getValueString().isEmpty()) {
776 String [] calleeTempList = bba.getValueString().split(
",", 0);
777 calleeId = Arrays.asList(calleeTempList);
781 alternateId = bba.getValueString();
782 otherAttributes.add(bba);
785 otherAttributes.add(bba);
790 if (calleeId.isEmpty() && communicationDirection == CommunicationDirection.OUTGOING) {
791 String [] calleeTempList = callerId.split(
",", 0);
792 calleeId = Arrays.asList(calleeTempList);
796 if (absFile == null) {
797 absFile = (AbstractFile) dataSource;
800 CommunicationArtifactsHelper accountArtifact;
801 if (accountType != null) {
806 moduleName, absFile, accountType, accountType, alternateId);
808 BlackboardArtifact callLogArtifact = accountArtifact.addCalllog(communicationDirection, callerId, calleeId, startDateTime, endDateTime, mediaType, otherAttributes);
810 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_calllog_relationship() + ex.getLocalizedMessage(), ex);
816 switch (AccountTypeName.toLowerCase()) {
818 return Account.Type.ZAPYA;
819 case "sms messages.tsv":
820 return Account.Type.PHONE;
822 return Account.Type.PHONE;
823 case "imo - accountid.tsv":
824 return Account.Type.IMO;
825 case "imo - messages.tsv":
826 return Account.Type.IMO;
827 case "textnow - contacts.tsv":
828 return Account.Type.TEXTNOW;
829 case "textnow - messages.tsv":
830 return Account.Type.TEXTNOW;
831 case "mms messages.tsv":
832 return Account.Type.PHONE;
833 case "viber - call logs.tsv":
834 return Account.Type.VIBER;
835 case "viber - contacts.tsv":
836 return Account.Type.VIBER;
837 case "viber - messages.tsv":
838 return Account.Type.VIBER;
839 case "xender file transfer - messages.tsv":
840 return Account.Type.XENDER;
841 case "xender file transfer - contacts.tsv":
842 return Account.Type.XENDER;
843 case "whatsapp - single call logs.tsv":
844 return Account.Type.WHATSAPP;
845 case "whatsapp - messages logs.tsv":
846 return Account.Type.WHATSAPP;
847 case "whatsapp - group call logs.tsv":
848 return Account.Type.WHATSAPP;
849 case "whatsapp - contacts.tsv":
850 return Account.Type.WHATSAPP;
851 case "tangomessages messages.tsv":
852 return Account.Type.TANGO;
853 case "shareit file transfer.tsv":
854 return Account.Type.SHAREIT;
855 case "line - calllogs.tsv":
856 return Account.Type.LINE;
857 case "line - contacts.tsv":
858 return Account.Type.LINE;
859 case "line - messages.tsv":
860 return Account.Type.LINE;
861 case "skype - call logs.tsv":
862 return Account.Type.SKYPE;
863 case "skype - contacts.tsv":
864 return Account.Type.SKYPE;
865 case "skype - messages logs.tsv":
866 return Account.Type.SKYPE;
867 case "facebook messenger - calls.tsv":
868 return Account.Type.FACEBOOK;
869 case "facebook messenger - contacts.tsv":
870 return Account.Type.FACEBOOK;
871 case "facebook messenger - chats.tsv":
872 return Account.Type.FACEBOOK;
873 case "call logs2.tsv":
874 return Account.Type.PHONE;
875 case "call logs.tsv":
876 return Account.Type.PHONE;
877 case "sms - imessage.tsv":
878 return Account.Type.PHONE;
880 return Account.Type.PHONE;
899 private Collection<BlackboardAttribute>
processReadLine(List<String> lineValues, Map<String, Integer> columnIndexes,
900 List<TsvColumn> attrList, String fileName,
int lineNum)
throws IngestModuleException {
902 if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues)
903 || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) {
904 return Collections.emptyList();
905 }
else if (lineValues.size() != columnIndexes.size()) {
906 logger.log(Level.WARNING, String.format(
907 "Row at line number %d in file %s has %d columns when %d were expected based on the header row.",
908 lineNum, fileName, lineValues.size(), columnIndexes.size()));
909 return Collections.emptyList();
912 List<BlackboardAttribute> attrsToRet =
new ArrayList<>();
914 if (colAttr.getAttributeType() == null) {
919 Integer columnIdx = columnIndexes.get(colAttr.getColumnName());
920 if (columnIdx == null) {
921 logger.log(Level.WARNING, String.format(
"No column mapping found for %s in file %s. Omitting column.", colAttr.getColumnName(), fileName));
925 String value = (columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx);
927 logger.log(Level.WARNING, String.format(
"No value found for column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName));
928 return Collections.emptyList();
933 BlackboardAttribute attr = (value == null) ? null :
getAttribute(colAttr.getAttributeType(), formattedValue, fileName);
935 logger.log(Level.WARNING, String.format(
"Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName));
936 return Collections.emptyList();
938 attrsToRet.add(attr);
941 if (tsvFileArtifactComments.containsKey(fileName)) {
942 attrsToRet.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, tsvFileArtifactComments.get(fileName)));
957 if (colAttr.getAttributeType().getTypeName().equals(
"TSK_DOMAIN")) {
967 private static final DateFormat
TIMESTAMP_FORMAT =
new SimpleDateFormat(
"yyyy-MM-d HH:mm:ss", US);
979 private BlackboardAttribute
getAttribute(BlackboardAttribute.Type attrType, String value, String fileName) {
980 if (attrType == null || value == null) {
981 logger.log(Level.WARNING, String.format(
"Unable to parse attribute type %s for value '%s' in fileName %s",
982 attrType == null ?
"<null>" : attrType.toString(),
983 value == null ?
"<null>" : value,
984 fileName == null ?
"<null>" : fileName));
988 switch (attrType.getValueType()) {
992 (v) ->
new BlackboardAttribute(attrType, moduleName, v));
994 return parseAttrValue(value.trim(), attrType, fileName,
true,
false,
995 (v) ->
new BlackboardAttribute(attrType, moduleName, Double.valueOf(v).intValue()));
997 return parseAttrValue(value.trim(), attrType, fileName,
true,
false,
998 (v) ->
new BlackboardAttribute(attrType, moduleName, Double.valueOf(v).longValue()));
1000 return parseAttrValue(value.trim(), attrType, fileName,
true,
false,
1001 (v) ->
new BlackboardAttribute(attrType, moduleName, (
double) Double.valueOf(v)));
1003 return parseAttrValue(value.trim(), attrType, fileName,
true,
false,
1004 (v) ->
new BlackboardAttribute(attrType, moduleName,
new byte[]{Byte.valueOf(v)}));
1006 return parseAttrValue(value.trim(), attrType, fileName,
true,
true,
1007 (v) ->
new BlackboardAttribute(attrType, moduleName, TIMESTAMP_FORMAT.parse(v).getTime() / 1000));
1010 logger.log(Level.WARNING, String.format(
"Attribute Type %s for file %s not defined.", attrType, fileName));
1029 BlackboardAttribute
apply(String orig)
throws ParseException, NumberFormatException;
1048 value = value.replaceAll(
"\\p{C}",
"");
1050 if (blankIsNull && StringUtils.isBlank(value)) {
1054 if (zeroIsNull && value.matches(
"^\\s*[0\\.]*\\s*$")) {
1059 return valueConverter.
apply(value);
1060 }
catch (NumberFormatException | ParseException ex) {
1061 logger.log(Level.WARNING, String.format(
"Unable to format '%s' as value type %s while converting to attributes from %s.", value, attrType.getValueType().getLabel(), fileName), ex);
1066 @NbBundle.Messages({
1067 "LeappFileProcessor.cannot.load.artifact.xml=Cannot load xml artifact file.",
1068 "LeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.",
1069 "LeappFileProcessor_cannotParseXml=Cannot Parse XML file.",
1070 "LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact",
1071 "LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts."
1081 File f =
new File(path);
1082 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
1083 DocumentBuilder db = dbf.newDocumentBuilder();
1084 xmlinput = db.parse(f);
1086 }
catch (IOException e) {
1087 throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_load_artifact_xml() + e.getLocalizedMessage(), e);
1088 }
catch (ParserConfigurationException pce) {
1089 throw new IngestModuleException(Bundle.LeappFileProcessor_cannotBuildXmlParser() + pce.getLocalizedMessage(), pce);
1090 }
catch (SAXException sxe) {
1091 throw new IngestModuleException(Bundle.LeappFileProcessor_cannotParseXml() + sxe.getLocalizedMessage(), sxe);
1102 NodeList nlist = xmlinput.getElementsByTagName(
"FileName");
1104 for (
int i = 0; i < nlist.getLength(); i++) {
1105 NamedNodeMap nnm = nlist.item(i).getAttributes();
1106 tsvFiles.put(nnm.getNamedItem(
"filename").getNodeValue().toLowerCase(), nnm.getNamedItem(
"description").getNodeValue());
1114 NodeList artifactNlist = xmlinput.getElementsByTagName(
"ArtifactName");
1115 for (
int k = 0; k < artifactNlist.getLength(); k++) {
1116 NamedNodeMap nnm = artifactNlist.item(k).getAttributes();
1117 String artifactName = nnm.getNamedItem(
"artifactname").getNodeValue();
1118 String comment = nnm.getNamedItem(
"comment").getNodeValue();
1119 String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem(
"filename").getNodeValue();
1121 BlackboardArtifact.Type foundArtifactType = null;
1124 }
catch (TskCoreException ex) {
1125 logger.log(Level.SEVERE, String.format(
"There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex);
1128 if (foundArtifactType == null) {
1129 logger.log(Level.SEVERE, String.format(
"No known artifact mapping found for [artifact: %s, %s]",
1135 if (!comment.toLowerCase().matches(
"null")) {
1136 tsvFileArtifactComments.put(parentName, comment);
1143 return String.format(
"file: %s, filename: %s",
1144 this.xmlFile == null ?
"<null>" : this.xmlFile,
1145 fileName == null ?
"<null>" : fileName);
1149 return String.format(
"attribute: %s %s",
1150 attributeName == null ?
"<null>" : attributeName,
1156 NodeList attributeNlist = xmlinput.getElementsByTagName(
"AttributeName");
1157 for (
int k = 0; k < attributeNlist.getLength(); k++) {
1158 NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
1159 String attributeName = nnm.getNamedItem(
"attributename").getNodeValue();
1161 if (!attributeName.toLowerCase().matches(
"null")) {
1162 String columnName = nnm.getNamedItem(
"columnName").getNodeValue();
1163 String required = nnm.getNamedItem(
"required").getNodeValue();
1164 String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem(
"filename").getNodeValue();
1166 BlackboardAttribute.Type foundAttrType = null;
1169 }
catch (TskCoreException ex) {
1170 logger.log(Level.SEVERE, String.format(
"There was an issue that arose while trying to fetch attribute type for %s.", attributeName), ex);
1173 if (foundAttrType == null) {
1174 logger.log(Level.SEVERE, String.format(
"No known attribute mapping found for [%s]",
getXmlAttrIdentifier(parentName, attributeName)));
1177 if (required != null && required.compareToIgnoreCase(
"yes") != 0 && required.compareToIgnoreCase(
"no") != 0) {
1178 logger.log(Level.SEVERE, String.format(
"Required value %s did not match 'yes' or 'no' for [%s]",
1182 if (columnName == null) {
1183 logger.log(Level.SEVERE, String.format(
"No column name provided for [%s]",
getXmlAttrIdentifier(parentName, attributeName)));
1184 }
else if (columnName.trim().length() != columnName.length()) {
1185 logger.log(Level.SEVERE, String.format(
"Column name '%s' starts or ends with whitespace for [%s]", columnName,
getXmlAttrIdentifier(parentName, attributeName)));
1186 }
else if (columnName.matches(
"[^ \\S]")) {
1187 logger.log(Level.SEVERE, String.format(
"Column name '%s' contains invalid characters [%s]", columnName,
getXmlAttrIdentifier(parentName, attributeName)));
1192 columnName.trim().toLowerCase(),
1193 "yes".compareToIgnoreCase(required) == 0);
1195 if (tsvFileAttributes.containsKey(parentName)) {
1196 List<TsvColumn> attrList = tsvFileAttributes.get(parentName);
1197 attrList.add(thisCol);
1198 tsvFileAttributes.replace(parentName, attrList);
1200 List<TsvColumn> attrList =
new ArrayList<>();
1201 attrList.add(thisCol);
1202 tsvFileAttributes.put(parentName, attrList);
1220 private BlackboardArtifact
createArtifactWithAttributes(BlackboardArtifact.Type artType, Content dataSource, Collection<BlackboardAttribute> bbattributes) {
1222 switch (artType.getCategory()) {
1224 return dataSource.newDataArtifact(artType, bbattributes);
1225 case ANALYSIS_RESULT:
1226 return dataSource.newAnalysisResult(artType, Score.SCORE_UNKNOWN, null, null, null, bbattributes).getAnalysisResult();
1228 logger.log(Level.SEVERE,
"Unknown category type: " + artType.getCategory().getDisplayName());
1231 }
catch (TskException ex) {
1232 logger.log(Level.WARNING, Bundle.LeappFileProcessor_error_creating_new_artifacts(), ex);
1243 void postArtifacts(Collection<BlackboardArtifact> artifacts) {
1244 if (artifacts == null || artifacts.isEmpty()) {
1249 Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifacts(artifacts, moduleName);
1250 }
catch (Blackboard.BlackboardException ex) {
1251 logger.log(Level.SEVERE, Bundle.LeappFileProcessor_postartifacts_error(), ex);
1274 static List<AbstractFile> findLeappFilesToProcess(Content dataSource) {
1276 List<AbstractFile> leappFiles =
new ArrayList<>();
1278 FileManager fileManager = getCurrentCase().getServices().getFileManager();
1282 leappFiles = fileManager.
findFiles(dataSource,
"%",
"/");
1283 }
catch (TskCoreException ex) {
1284 logger.log(Level.WARNING,
"No files found to process");
1288 List<AbstractFile> leappFilesToProcess =
new ArrayList<>();
1289 for (AbstractFile leappFile : leappFiles) {
1290 if (((leappFile.getLocalAbsPath() != null)
1291 && !leappFile.isVirtual())
1292 && leappFile.getNameExtension() != null
1293 && ALLOWED_EXTENSIONS.contains(leappFile.getNameExtension().toLowerCase())) {
1294 leappFilesToProcess.add(leappFile);
1298 return leappFilesToProcess;
1307 for (Map.Entry<String, String> customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) {
1308 String artifactName = customArtifact.getKey();
1309 String artifactDescription = customArtifact.getValue();
1312 BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(artifactName, artifactDescription);
1313 }
catch (Blackboard.BlackboardException ex) {
1314 logger.log(Level.WARNING, String.format(
"Failed to create custom artifact type %s.", artifactName), ex);
1321 if (fileNamePath == null) {
1325 List<AbstractFile> files;
1327 String fileName = FilenameUtils.getName(fileNamePath);
1328 String filePath = FilenameUtils.normalize(FilenameUtils.getPath(fileNamePath),
true);
1333 files = fileManager.
findFiles(dataSource, fileName);
1335 }
catch (TskCoreException ex) {
1336 logger.log(Level.WARNING,
"Unable to find prefetch files.", ex);
1340 for (AbstractFile pFile : files) {
1342 if (pFile.getParentPath().toLowerCase().endsWith(filePath.toLowerCase())) {
void createMessageRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.Type artType, Content dataSource, Collection< BlackboardAttribute > bbattributes)
String getXmlFileIdentifier(String fileName)
final BlackboardAttribute.Type attributeType
FileManager getFileManager()
void createContactRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
static String extractDomain(String urlString)
List< String > findTsvFiles(Path LeappOutputDir)
List< AbstractFile > findFiles(String fileName)
void processLeappFiles(List< String > LeappFilesToProcess, AbstractFile LeappImageFile)
void createCalllogRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
static final Logger logger
static final DateFormat TIMESTAMP_FORMAT
final Map< String, BlackboardArtifact.Type > tsvFileArtifacts
static final Set< String > ALLOWED_EXTENSIONS
void getArtifactNode(Document xmlinput)
AbstractFile findAbstractFile(Content dataSource, String fileNamePath)
void createRoute(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
static final Map< String, String > CUSTOM_ARTIFACT_MAP
Collection< BlackboardAttribute > processReadLine(List< String > lineValues, Map< String, Integer > columnIndexes, List< TsvColumn > attrList, String fileName, int lineNum)
BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter)
final Map< String, String > tsvFiles
ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath)
ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile)
void processFile(File LeappFile, List< TsvColumn > attrList, String fileName, BlackboardArtifact.Type artifactType, List< BlackboardArtifact > bbartifacts, Content dataSource)
static final Map< String, String > ACCOUNT_RELATIONSHIPS
SleuthkitCase getSleuthkitCase()
final Map< String, String > tsvFileArtifactComments
BlackboardAttribute getAttribute(BlackboardAttribute.Type attrType, String value, String fileName)
final Map< String, List< TsvColumn > > tsvFileAttributes
void createCustomArtifacts(Blackboard blkBoard)
static Case getCurrentCase()
synchronized static Logger getLogger(String name)
static Case getCurrentCaseThrows()
LeappFileProcessor(String xmlFile, String moduleName)
BlackboardAttribute apply(String orig)
AbstractFile createTrackpoint(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList)
Account.Type getAccountType(String AccountTypeName)
void processLeappFiles(List< String > LeappFilesToProcess, Content dataSource)
void getAttributeNodes(Document xmlinput)
String getXmlAttrIdentifier(String fileName, String attributeName)
String formatValueBasedOnAttrType(TsvColumn colAttr, String value)
void getFileNode(Document xmlinput)