Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
XRYMessagesFileParser.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2019-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.datasourceprocessors.xry;
20 
21 import java.io.IOException;
22 import java.nio.file.Path;
23 import java.time.format.DateTimeParseException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Objects;
29 import java.util.Optional;
30 import java.util.Set;
31 import java.util.logging.Level;
33 import org.sleuthkit.datamodel.Account;
34 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
35 import org.sleuthkit.datamodel.BlackboardAttribute;
36 import org.sleuthkit.datamodel.Content;
37 import org.sleuthkit.datamodel.InvalidAccountIDException;
38 import org.sleuthkit.datamodel.SleuthkitCase;
39 import org.sleuthkit.datamodel.TskCoreException;
40 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
41 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection;
42 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus;
43 
47 final class XRYMessagesFileParser implements XRYFileParser {
48 
49  private static final Logger logger = Logger.getLogger(
50  XRYMessagesFileParser.class.getName());
51 
52  private static final String PARSER_NAME = "XRY DSP";
53 
58  private enum XryKey {
59  DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
60  DIRECTION("direction", null),
61  MESSAGE("message", null),
62  NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON),
63  TEXT("text", null),
64  TIME("time", null),
65  SERVICE_CENTER("service center", null),
66  FROM("from", null),
67  TO("to", null),
68  //The following keys either need special processing or more time and data to find a type.
69  STORAGE("storage", null),
70  NUMBER("number", null),
71  TYPE("type", null),
72  TEL("tel", null),
73  FOLDER("folder", null),
74  NAME("name", null),
75  INDEX("index", null),
76  STATUS("status", null);
77 
78  private final String name;
79  private final BlackboardAttribute.ATTRIBUTE_TYPE type;
80 
81  XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) {
82  this.name = name;
83  this.type = type;
84  }
85 
86  public BlackboardAttribute.ATTRIBUTE_TYPE getType() {
87  return type;
88  }
89 
90  public String getDisplayName() {
91  return name;
92  }
93 
101  public static boolean contains(String name) {
102  try {
103  XryKey.fromDisplayName(name);
104  return true;
105  } catch (IllegalArgumentException ex) {
106  return false;
107  }
108  }
109 
121  public static XryKey fromDisplayName(String name) {
122  String normalizedName = name.trim().toLowerCase();
123  for (XryKey keyChoice : XryKey.values()) {
124  if (normalizedName.equals(keyChoice.name)) {
125  return keyChoice;
126  }
127  }
128 
129  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
130  + " All keys should be tested with contains.", name));
131  }
132  }
133 
137  private enum XryNamespace {
138  FROM("from"),
139  PARTICIPANT("participant"),
140  TO("to"),
141  NONE(null);
142 
143  private final String name;
144 
145  XryNamespace(String name) {
146  this.name = name;
147  }
148 
157  public static boolean contains(String xryNamespace) {
158  try {
159  XryNamespace.fromDisplayName(xryNamespace);
160  return true;
161  } catch (IllegalArgumentException ex) {
162  return false;
163  }
164  }
165 
178  public static XryNamespace fromDisplayName(String xryNamespace) {
179  String normalizedNamespace = xryNamespace.trim().toLowerCase();
180  for (XryNamespace keyChoice : XryNamespace.values()) {
181  if (normalizedNamespace.equals(keyChoice.name)) {
182  return keyChoice;
183  }
184  }
185 
186  throw new IllegalArgumentException(String.format("Namespace [%s] was not found."
187  + " All namespaces should be tested with contains.", xryNamespace));
188  }
189  }
190 
194  private enum XryMetaKey {
195  REFERENCE_NUMBER("reference number"),
196  SEGMENT_COUNT("segments"),
197  SEGMENT_NUMBER("segment number");
198 
199  private final String name;
200 
201  XryMetaKey(String name) {
202  this.name = name;
203  }
204 
205  public String getDisplayName() {
206  return name;
207  }
208 
216  public static boolean contains(String name) {
217  try {
219  return true;
220  } catch (IllegalArgumentException ex) {
221  return false;
222  }
223  }
224 
236  public static XryMetaKey fromDisplayName(String name) {
237  String normalizedName = name.trim().toLowerCase();
238  for (XryMetaKey keyChoice : XryMetaKey.values()) {
239  if (normalizedName.equals(keyChoice.name)) {
240  return keyChoice;
241  }
242  }
243 
244  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
245  + " All keys should be tested with contains.", name));
246  }
247  }
248 
270  @Override
271  public void parse(XRYFileReader reader, Content parent, SleuthkitCase currentCase) throws IOException, TskCoreException, BlackboardException {
272  Path reportPath = reader.getReportPath();
273  logger.log(Level.INFO, String.format("[XRY DSP] Processing report at"
274  + " [ %s ]", reportPath.toString()));
275 
276  //Keep track of the reference numbers that have been parsed.
277  Set<Integer> referenceNumbersSeen = new HashSet<>();
278 
279  while (reader.hasNextEntity()) {
280  String xryEntity = reader.nextEntity();
281 
282  // This call will combine all segmented text into a single key value pair
283  List<XRYKeyValuePair> pairs = getXRYKeyValuePairs(xryEntity, reader, referenceNumbersSeen);
284 
285  // Transform all the data from XRY land into the appropriate CommHelper
286  // data types.
287  final String messageType = PARSER_NAME;
288  CommunicationDirection direction = CommunicationDirection.UNKNOWN;
289  String senderId = null;
290  final List<String> recipientIdsList = new ArrayList<>();
291  long dateTime = 0L;
292  MessageReadStatus readStatus = MessageReadStatus.UNKNOWN;
293  final String subject = null;
294  String text = null;
295  final String threadId = null;
296  final Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
297 
298  for (XRYKeyValuePair pair : pairs) {
299  XryNamespace namespace = XryNamespace.NONE;
300  if (XryNamespace.contains(pair.getNamespace())) {
301  namespace = XryNamespace.fromDisplayName(pair.getNamespace());
302  }
303  XryKey key = XryKey.fromDisplayName(pair.getKey());
304  String normalizedValue = pair.getValue().toLowerCase().trim();
305 
306  switch (key) {
307  case TEL:
308  case NUMBER:
309  if (!XRYUtils.isPhoneValid(pair.getValue())) {
310  continue;
311  }
312 
313  // Apply namespace or direction
314  if (namespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
315  senderId = pair.getValue();
316  } else if (namespace == XryNamespace.TO || direction == CommunicationDirection.OUTGOING) {
317  recipientIdsList.add(pair.getValue());
318  } else {
319  try {
320  currentCase.getCommunicationsManager().createAccountFileInstance(
321  Account.Type.PHONE, pair.getValue(), PARSER_NAME, parent, null, null);
322  } catch (InvalidAccountIDException ex) {
323  logger.log(Level.WARNING, String.format("Invalid account identifier %s", pair.getValue()), ex);
324  }
325 
326  otherAttributes.add(new BlackboardAttribute(
327  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
328  PARSER_NAME, pair.getValue()));
329  }
330  break;
331  // Although confusing, as these are also 'name spaces', it appears
332  // later versions of XRY just made these standardized lines.
333  case FROM:
334  if (!XRYUtils.isPhoneValid(pair.getValue())) {
335  continue;
336  }
337 
338  senderId = pair.getValue();
339  break;
340  case TO:
341  if (!XRYUtils.isPhoneValid(pair.getValue())) {
342  continue;
343  }
344 
345  recipientIdsList.add(pair.getValue());
346  break;
347  case TIME:
348  try {
349  //Tranform value to seconds since epoch
350  long dateTimeSinceInEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
351  dateTime = dateTimeSinceInEpoch;
352  } catch (DateTimeParseException ex) {
353  logger.log(Level.WARNING, String.format("[%s] Assumption"
354  + " about the date time formatting of messages is "
355  + "not right. Here is the pair [ %s ]", PARSER_NAME, pair), ex);
356  }
357  break;
358  case TYPE:
359  switch (normalizedValue) {
360  case "incoming":
361  direction = CommunicationDirection.INCOMING;
362  break;
363  case "outgoing":
364  direction = CommunicationDirection.OUTGOING;
365  break;
366  case "deliver":
367  case "submit":
368  case "status report":
369  //Ignore for now.
370  break;
371  default:
372  logger.log(Level.WARNING, String.format("[%s] Unrecognized "
373  + " value for key pair [ %s ].", PARSER_NAME, pair));
374  }
375  break;
376  case STATUS:
377  switch (normalizedValue) {
378  case "read":
379  readStatus = MessageReadStatus.READ;
380  break;
381  case "unread":
382  readStatus = MessageReadStatus.UNREAD;
383  break;
384  case "deleted":
385  otherAttributes.add(new BlackboardAttribute(
386  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED,
387  PARSER_NAME, pair.getValue()));
388  break;
389  case "sending failed":
390  case "unsent":
391  case "sent":
392  //Ignoring for now.
393  break;
394  default:
395  logger.log(Level.WARNING, String.format("[%s] Unrecognized "
396  + " value for key pair [ %s ].", PARSER_NAME, pair));
397  }
398  break;
399  case TEXT:
400  case MESSAGE:
401  text = pair.getValue();
402  break;
403  case DIRECTION:
404  switch (normalizedValue) {
405  case "incoming":
406  direction = CommunicationDirection.INCOMING;
407  break;
408  case "outgoing":
409  direction = CommunicationDirection.OUTGOING;
410  break;
411  default:
412  direction = CommunicationDirection.UNKNOWN;
413  break;
414  }
415  break;
416  case SERVICE_CENTER:
417  if (!XRYUtils.isPhoneValid(pair.getValue())) {
418  continue;
419  }
420 
421  otherAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
422  PARSER_NAME, pair.getValue()));
423  break;
424  default:
425  //Otherwise, the XryKey enum contains the correct BlackboardAttribute
426  //type.
427  if (key.getType() != null) {
428  otherAttributes.add(new BlackboardAttribute(key.getType(),
429  PARSER_NAME, pair.getValue()));
430  } else {
431  logger.log(Level.INFO, String.format("[%s] Key value pair "
432  + "(in brackets) [ %s ] was recognized but "
433  + "more data or time is needed to finish implementation. Discarding... ",
434  PARSER_NAME, pair));
435  }
436  }
437  }
438 
439  CommunicationArtifactsHelper helper = new CommunicationArtifactsHelper(
440  currentCase, PARSER_NAME, parent, Account.Type.PHONE, null);
441 
442  helper.addMessage(messageType, direction, senderId, recipientIdsList,
443  dateTime, readStatus, subject, text, threadId, otherAttributes);
444  }
445  }
446 
451  private List<XRYKeyValuePair> getXRYKeyValuePairs(String xryEntity,
452  XRYFileReader reader, Set<Integer> referenceValues) throws IOException {
453  String[] xryLines = xryEntity.split("\n");
454  //First line of the entity is the title, each XRY entity is non-empty.
455  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
456 
457  List<XRYKeyValuePair> pairs = new ArrayList<>();
458 
459  //Count the key value pairs in the XRY entity.
460  int keyCount = getCountOfKeyValuePairs(xryLines);
461  for (int i = 1; i <= keyCount; i++) {
462  //Get the ith key value pair in the entity. Always expect to have
463  //a valid value.
464  XRYKeyValuePair pair = getKeyValuePairByIndex(xryLines, i).get();
465  if (XryMetaKey.contains(pair.getKey())) {
466  //Skip meta keys, they are being handled seperately.
467  continue;
468  }
469 
470  if (!XryKey.contains(pair.getKey())) {
471  logger.log(Level.WARNING, String.format("[XRY DSP] The following key, "
472  + "value pair (in brackets) [ %s ], "
473  + "was not recognized. Discarding...", pair));
474  continue;
475  }
476 
477  if (pair.getValue().isEmpty()) {
478  logger.log(Level.WARNING, String.format("[XRY DSP] The following key "
479  + "(in brackets) [ %s ] was recognized, but the value "
480  + "was empty. Discarding...", pair.getKey()));
481  continue;
482  }
483 
484  //Assume text and message are the only fields that can be segmented
485  //among multiple XRY entities.
486  if (pair.hasKey(XryKey.TEXT.getDisplayName())
487  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
488  String segmentedText = getSegmentedText(xryLines, reader, referenceValues);
489  pair = new XRYKeyValuePair(pair.getKey(),
490  //Assume text is segmented by word.
491  pair.getValue() + " " + segmentedText,
492  pair.getNamespace());
493  }
494 
495  pairs.add(pair);
496  }
497 
498  return pairs;
499  }
500 
505  private Integer getCountOfKeyValuePairs(String[] xryEntity) {
506  int count = 0;
507  for (int i = 1; i < xryEntity.length; i++) {
508  if (XRYKeyValuePair.isPair(xryEntity[i])) {
509  count++;
510  }
511  }
512  return count;
513  }
514 
528  private String getSegmentedText(String[] xryEntity, XRYFileReader reader,
529  Set<Integer> referenceNumbersSeen) throws IOException {
530  Optional<Integer> referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER);
531  //Check if there is any segmented text.
532  if (!referenceNumber.isPresent()) {
533  return "";
534  }
535 
536  logger.log(Level.INFO, String.format("[XRY DSP] Message entity "
537  + "appears to be segmented with reference number [ %d ]", referenceNumber.get()));
538 
539  if (referenceNumbersSeen.contains(referenceNumber.get())) {
540  logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already "
541  + "been seen. This means that the segments are not "
542  + "contiguous. Any segments contiguous with this "
543  + "one will be aggregated and another "
544  + "(otherwise duplicate) artifact will be created.", referenceNumber.get()));
545  }
546 
547  referenceNumbersSeen.add(referenceNumber.get());
548 
549  Optional<Integer> segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER);
550  if (!segmentNumber.isPresent()) {
551  logger.log(Level.SEVERE, String.format("No segment "
552  + "number was found on the message entity"
553  + "with reference number [%d]", referenceNumber.get()));
554  return "";
555  }
556 
557  StringBuilder segmentedText = new StringBuilder();
558 
559  int currentSegmentNumber = segmentNumber.get();
560  while (reader.hasNextEntity()) {
561  //Peek at the next to see if it has the same reference number.
562  String nextEntity = reader.peek();
563  String[] nextEntityLines = nextEntity.split("\n");
564  Optional<Integer> nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER);
565 
566  if (!nextReferenceNumber.isPresent()
567  || !Objects.equals(nextReferenceNumber, referenceNumber)) {
568  //Don't consume the next entity. It is not related
569  //to the current message thread.
570  break;
571  }
572 
573  //Consume the entity, it is a part of the message thread.
574  reader.nextEntity();
575 
576  Optional<Integer> nextSegmentNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.SEGMENT_NUMBER);
577 
578  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] "
579  + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get()));
580 
581  if (!nextSegmentNumber.isPresent()) {
582  logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference"
583  + " number [ %d ] did not have a segment number associated with it."
584  + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get()));
585  } else if (nextSegmentNumber.get() != currentSegmentNumber + 1) {
586  logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous "
587  + "segments are not ascending incrementally. Encountered "
588  + "segment [ %d ] after segment [ %d ]. This means the reconstructed "
589  + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber));
590  }
591 
592  int keyCount = getCountOfKeyValuePairs(nextEntityLines);
593  for (int i = 1; i <= keyCount; i++) {
594  XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get();
595  if (pair.hasKey(XryKey.TEXT.getDisplayName())
596  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
597  segmentedText.append(pair.getValue()).append(' ');
598  }
599  }
600 
601  if (nextSegmentNumber.isPresent()) {
602  currentSegmentNumber = nextSegmentNumber.get();
603  }
604  }
605 
606  //Remove the trailing space.
607  if (segmentedText.length() > 0) {
608  segmentedText.setLength(segmentedText.length() - 1);
609  }
610 
611  return segmentedText.toString();
612  }
613 
622  private Optional<Integer> getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) {
623  for (String xryLine : xryLines) {
624  if (!XRYKeyValuePair.isPair(xryLine)) {
625  continue;
626  }
627 
628  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
629  if (pair.hasKey(metaKey.getDisplayName())) {
630  try {
631  return Optional.of(Integer.parseInt(pair.getValue()));
632  } catch (NumberFormatException ex) {
633  logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for "
634  + "meta key [ %s ] was not an integer.", pair.getValue(), metaKey), ex);
635  }
636  }
637  }
638  return Optional.empty();
639  }
640 
652  private Optional<XRYKeyValuePair> getKeyValuePairByIndex(String[] xryLines, int index) {
653  int pairsParsed = 0;
654  String namespace = "";
655  for (int i = 1; i < xryLines.length; i++) {
656  String xryLine = xryLines[i];
657  if (XryNamespace.contains(xryLine)) {
658  namespace = xryLine.trim();
659  continue;
660  }
661 
662  if (!XRYKeyValuePair.isPair(xryLine)) {
663  logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value "
664  + "pair on this line (in brackets) [ %s ], but one was not detected."
665  + " Discarding...", xryLine));
666  continue;
667  }
668 
669  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
670  String value = pair.getValue();
671  //Build up multiple lines.
672  for (; (i + 1) < xryLines.length
673  && !XRYKeyValuePair.isPair(xryLines[i + 1])
674  && !XryNamespace.contains(xryLines[i + 1]); i++) {
675  String continuedValue = xryLines[i + 1].trim();
676  //Assume multi lined values are split by word.
677  value = value + " " + continuedValue;
678  }
679 
680  pair = new XRYKeyValuePair(pair.getKey(), value, namespace);
681  pairsParsed++;
682  if (pairsParsed == index) {
683  return Optional.of(pair);
684  }
685  }
686 
687  return Optional.empty();
688  }
689 }

Copyright © 2012-2022 Basis Technology. Generated on: Thu Oct 6 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.