Autopsy  4.20.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
LeappFileProcessor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2020-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.leappanalyzers;
20 
21 import com.fasterxml.jackson.databind.MappingIterator;
22 import com.fasterxml.jackson.dataformat.csv.CsvMapper;
23 import com.fasterxml.jackson.dataformat.csv.CsvParser;
24 import com.fasterxml.jackson.dataformat.csv.CsvSchema;
25 import com.google.common.collect.ImmutableMap;
26 import java.io.File;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.io.UncheckedIOException;
30 import java.nio.file.Files;
31 import java.nio.file.Path;
32 import java.text.DateFormat;
33 import java.text.ParseException;
34 import java.text.SimpleDateFormat;
35 import java.util.List;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import static java.util.Locale.US;
43 import java.util.Map;
44 import java.util.Set;
45 import java.util.logging.Level;
46 import java.util.stream.Collectors;
47 import java.util.stream.IntStream;
48 import java.util.stream.Stream;
49 import javax.xml.parsers.DocumentBuilder;
50 import javax.xml.parsers.DocumentBuilderFactory;
51 import javax.xml.parsers.ParserConfigurationException;
52 import org.apache.commons.collections4.CollectionUtils;
53 import org.apache.commons.collections4.MapUtils;
54 import org.apache.commons.io.FilenameUtils;
55 import org.apache.commons.lang3.StringUtils;
56 import org.openide.util.NbBundle;
57 import org.openide.util.NbBundle.Messages;
58 import org.sleuthkit.autopsy.casemodule.Case;
59 import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase;
60 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
61 import org.sleuthkit.autopsy.casemodule.services.FileManager;
62 import org.sleuthkit.autopsy.coreutils.NetworkUtils;
63 import org.sleuthkit.autopsy.coreutils.Logger;
64 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
65 import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
66 import org.sleuthkit.autopsy.ingest.IngestJobContext;
67 import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
68 import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
69 import org.sleuthkit.datamodel.AbstractFile;
70 import org.sleuthkit.datamodel.Account;
71 import org.sleuthkit.datamodel.Blackboard;
72 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
73 import org.sleuthkit.datamodel.BlackboardArtifact;
74 import org.sleuthkit.datamodel.BlackboardAttribute;
75 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
76 import org.sleuthkit.datamodel.Content;
77 import org.sleuthkit.datamodel.Score;
78 import org.sleuthkit.datamodel.TskCoreException;
79 import org.sleuthkit.datamodel.TskException;
80 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
81 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CallMediaType;
82 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection;
83 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus;
84 import org.sleuthkit.datamodel.blackboardutils.GeoArtifactsHelper;
85 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints;
86 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints.TrackPoint;
87 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints;
88 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints.Waypoint;
89 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments;
90 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment;
91 import org.w3c.dom.Document;
92 import org.w3c.dom.NamedNodeMap;
93 import org.w3c.dom.NodeList;
94 import org.xml.sax.SAXException;
95 
99 public final class LeappFileProcessor {
100 
104  private static class TsvColumn {
105 
106  private final BlackboardAttribute.Type attributeType;
107  private final String columnName;
108  private final boolean required;
109 
119  TsvColumn(BlackboardAttribute.Type attributeType, String columnName, boolean required) {
120  this.attributeType = attributeType;
121  this.columnName = columnName;
122  this.required = required;
123  }
124 
128  BlackboardAttribute.Type getAttributeType() {
129  return attributeType;
130  }
131 
135  String getColumnName() {
136  return columnName;
137  }
138 
142  boolean isRequired() {
143  return required;
144  }
145  }
146 
147  private static final Logger logger = Logger.getLogger(LeappFileProcessor.class.getName());
148  private final String CUSTOM_ARTIFACTS_ATTRIBUTES_FILE = "custom-artifact-attribute-list.csv";
149  private final String ARTIFACT_ATTRIBUTE_REFERENCE_USER = "artifact-attribute-reference-user.xml";
150 
151  private final String xmlFile; //NON-NLS
152  private final String leapModule;
153  private final String moduleName;
154  private final IngestJobContext context;
155 
156  private final Map<String, String> tsvFiles;
157  private final Map<String, BlackboardArtifact.Type> tsvFileArtifacts;
158  private final Map<String, String> tsvFileArtifactComments;
159  private final Map<String, List<TsvColumn>> tsvFileAttributes;
160 
161  private static final Map<String, String> CUSTOM_ARTIFACT_MAP = ImmutableMap.<String, String>builder()
162  .put("TSK_IP_DHCP", "DHCP Information")
163  .build();
164 
165  private static final Map<String, String> ACCOUNT_RELATIONSHIPS = ImmutableMap.<String, String>builder()
166  .put("zapya.tsv", "message")
167  .put("sms messages.tsv", "message")
168  .put("mms messages.tsv", "message")
169  .put("viber - messages.tsv", "message")
170  .put("viber - contacts.tsv", "contact")
171  .put("viber - call logs.tsv", "calllog")
172  .put("xender file transfer - messages.tsv", "message")
173  .put("xender file transfer - contacts.tsv", "contact")
174  .put("whatsapp - contacts.tsv", "contact")
175  .put("whatsapp - group call logs.tsv", "calllog")
176  .put("whatsapp - single call logs.tsv", "calllog")
177  .put("whatsapp - messages logs.tsv", "message")
178  .put("shareit file transfer.tsv", "message")
179  .put("tangomessages messages.tsv", "message")
180  .put("contacts.tsv", "contact")
181  .put("imo - accountid.tsv", "contact")
182  .put("imo - messages.tsv", "message")
183  .put("textnow - contacts.tsv", "contact")
184  .put("textnow - messages.tsv", "message")
185  .put("line - messages.tsv", "message")
186  .put("line - contacts.tsv", "contact")
187  .put("line - calllogs.tsv", "calllog")
188  .put("skype - messages logs.tsv", "message")
189  .put("skype - contacts.tsv", "contact")
190  .put("skype - call logs.tsv", "calllog")
191  .put("facebook messenger - chats.tsv", "message")
192  .put("facebook messenger - contacts.tsv", "contact")
193  .put("facebook messenger - calls.tsv", "calllog")
194  .put("call logs2.tsv", "calllog")
195  .put("call logs.tsv", "calllog")
196  .put("oruxmaps tracks.tsv", "trackpoint")
197  .put("google map locations.tsv", "route")
198  .put("Contacts.tsv", "contact")
199  .put("sms - imessage.tsv", "message")
200  .put("call history.tsv", "calllog")
201  .build();
202 
203  private final Blackboard blkBoard;
204 
205  public LeappFileProcessor(String xmlFile, String moduleName, String leapModule, IngestJobContext context) throws IOException, IngestModuleException, NoCurrentCaseException {
206  this.tsvFiles = new HashMap<>();
207  this.tsvFileArtifacts = new HashMap<>();
208  this.tsvFileArtifactComments = new HashMap<>();
209  this.tsvFileAttributes = new HashMap<>();
210  this.xmlFile = xmlFile;
211  this.moduleName = moduleName;
212  this.context = context;
213  this.leapModule = leapModule;
214 
215  blkBoard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
216 
217  loadCustomArtifactsAttributes(blkBoard, leapModule);
218  createCustomArtifacts(blkBoard);
219  configExtractor();
220  loadConfigFile();
221 
222  }
223 
231  private static String normalizeKey(String origKey) {
232  return StringUtils.defaultString(origKey).trim().toLowerCase();
233  }
234 
235  @NbBundle.Messages({
236  "LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.",
237  "LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory.",
238  "LeappFileProcessor.starting.Leapp=Starting Leapp",
239  "LeappFileProcessor.running.Leapp=Running Leapp",
240  "LeappFileProcessor.has.run=Leapp",
241  "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
242  "LeappFileProcessor.completed=Leapp Processing Completed",
243  "LeappFileProcessor.findTsv=Finding all Leapp ouput",
244  "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"
245  })
246  public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress) {
247  try {
248  if (checkCancelled()) {
249  return ProcessResult.OK;
250  }
251  progress.switchToIndeterminate();
252  progress.progress(Bundle.LeappFileProcessor_findTsv());
253  List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
254  processLeappFiles(LeappTsvOutputFiles, LeappFile, progress);
255  } catch (IngestModuleException ex) {
256  logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
257  return ProcessResult.ERROR;
258  }
259 
260  return ProcessResult.OK;
261  }
262 
263  public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress) {
264  try {
265  if (checkCancelled()) {
266  return ProcessResult.OK;
267  }
268  progress.switchToIndeterminate();
269  progress.progress(Bundle.LeappFileProcessor_findTsv());
270  List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
271  processLeappFiles(LeappTsvOutputFiles, dataSource, progress);
272  } catch (IngestModuleException ex) {
273  logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
274  return ProcessResult.ERROR;
275  }
276 
277  return ProcessResult.OK;
278  }
279 
284  private List<String> findTsvFiles(Path LeappOutputDir) throws IngestModuleException {
285  List<String> allTsvFiles;
286  List<String> foundTsvFiles = new ArrayList<>();
287 
288  try (Stream<Path> walk = Files.walk(LeappOutputDir)) {
289 
290  allTsvFiles = walk.map(x -> x.toString())
291  .filter(f -> f.toLowerCase().endsWith(".tsv")).collect(Collectors.toList());
292 
293  for (String tsvFile : allTsvFiles) {
294  if (tsvFiles.containsKey(normalizeKey(FilenameUtils.getName(tsvFile)))) {
295  foundTsvFiles.add(tsvFile);
296  }
297  }
298 
299  } catch (IOException | UncheckedIOException e) {
300  throw new IngestModuleException(Bundle.LeappFileProcessor_error_reading_Leapp_directory() + LeappOutputDir.toString(), e);
301  }
302 
303  return foundTsvFiles;
304 
305  }
306 
307  private boolean checkCancelled() {
308  if (this.context.dataSourceIngestIsCancelled()) {
309  logger.log(Level.INFO, "Leapp File processing module run was cancelled"); //NON-NLS
310  return true;
311  } else {
312  return false;
313  }
314  }
315 
326  @Messages({
327  "# {0} - fileName",
328  "LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}"
329  })
330  private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException {
331  progress.switchToDeterminate(LeappFilesToProcess.size());
332 
333  for (int i = 0; i < LeappFilesToProcess.size(); i++) {
334  if (checkCancelled()) {
335  return;
336  }
337 
338  String LeappFileName = LeappFilesToProcess.get(i);
339  String fileName = FilenameUtils.getName(LeappFileName);
340  progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i);
341 
342  File LeappFile = new File(LeappFileName);
343  String fileKey = fileName.toLowerCase().trim();
344  if (tsvFileAttributes.containsKey(normalizeKey(fileKey))) {
345  List<TsvColumn> attrList = tsvFileAttributes.get(normalizeKey(fileKey));
346  BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(normalizeKey(fileKey));
347 
348  try {
349  processFile(LeappFile, attrList, fileName, artifactType, dataSource);
350  } catch (TskCoreException | IOException ex) {
351  logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex);
352  }
353  }
354 
355  }
356  }
357 
358  private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName,
359  BlackboardArtifact.Type artifactType, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
360  TskCoreException {
361 
362  String trackpointSegmentName = null;
363  GeoTrackPoints pointList = new GeoTrackPoints();
364  AbstractFile geoAbstractFile = null;
365 
366  if (LeappFile == null || !LeappFile.exists() || fileName == null) {
367  logger.log(Level.WARNING, String.format("Leap file: %s is null or does not exist", LeappFile != null ? LeappFile.toString() : "<null>"));
368  return;
369  } else if (attrList == null || artifactType == null || dataSource == null) {
370  logger.log(Level.WARNING, String.format("attribute list, artifact type or dataSource not provided for %s", LeappFile.toString()));
371  return;
372  }
373 
374  List<BlackboardArtifact> bbartifacts = new ArrayList<>();
375 
376  // based on https://stackoverflow.com/questions/56921465/jackson-csv-schema-for-array
377  try (MappingIterator<List<String>> iterator = new CsvMapper()
378  .enable(CsvParser.Feature.WRAP_AS_ARRAY)
379  .readerFor(List.class)
380  .with(CsvSchema.emptySchema().withColumnSeparator('\t'))
381  .readValues(LeappFile)) {
382 
383  if (iterator.hasNext()) {
384  List<String> headerItems = iterator.next();
385  Map<String, Integer> columnIndexes = IntStream.range(0, headerItems.size())
386  .mapToObj(idx -> idx)
387  .collect(Collectors.toMap(
388  idx -> headerItems.get(idx) == null ? null : headerItems.get(idx).trim().toLowerCase(),
389  idx -> idx,
390  (val1, val2) -> val1));
391 
392  int lineNum = 2;
393  while (iterator.hasNext()) {
394  List<String> columnItems = iterator.next();
395  Collection<BlackboardAttribute> bbattributes = processReadLine(columnItems, columnIndexes, attrList, fileName, lineNum);
396 
397  if (!bbattributes.isEmpty()) {
398  switch (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(), "norelationship").toLowerCase()) {
399  case "message":
400  createMessageRelationship(bbattributes, dataSource, fileName);
401  break;
402  case "contact":
403  createContactRelationship(bbattributes, dataSource, fileName);
404  break;
405  case "calllog":
406  createCalllogRelationship(bbattributes, dataSource, fileName);
407  break;
408  case "route":
409  createRoute(bbattributes, dataSource, fileName);
410  break;
411  case "trackpoint":
412  geoAbstractFile = createTrackpoint(bbattributes, dataSource, fileName, trackpointSegmentName, pointList);
413  break;
414  default: // There is no relationship defined so just process the artifact normally
415  BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType, dataSource, bbattributes);
416  if (bbartifact != null) {
417  bbartifacts.add(bbartifact);
418  }
419  break;
420  }
421  }
422 
423  lineNum++;
424  }
425  }
426  }
427 
428  try {
429  if (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(), "norelationship").toLowerCase().equals("trackpoint")) {
430  (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, "", geoAbstractFile, context.getJobId())).addTrack(trackpointSegmentName, pointList, new ArrayList<>());
431  }
432  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
433  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
434  }
435 
436  if (!bbartifacts.isEmpty()) {
437  postArtifacts(bbartifacts);
438  }
439  }
440 
441  @NbBundle.Messages({
442  "LeappFileProcessor.cannot.create.waypoint.relationship=Cannot create TSK_WAYPOINT artifact."
443  })
444  private void createRoute(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
445 
446  Double startLatitude = Double.valueOf(0);
447  Double startLongitude = Double.valueOf(0);
448  Double endLatitude = Double.valueOf(0);
449  Double endLongitude = Double.valueOf(0);
450  Double zeroValue = Double.valueOf(0);
451  String destinationName = "";
452  String locationName = "";
453  Long dateTime = Long.valueOf(0);
454  Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
455  String sourceFile = null;
456  AbstractFile absFile;
457  String comment = "";
458 
459  try {
460  for (BlackboardAttribute bba : bbattributes) {
461  switch (bba.getAttributeType().getTypeName()) {
462  case "TSK_GEO_LATITUDE_START":
463  startLatitude = bba.getValueDouble();
464  break;
465  case "TSK_GEO_LONGITUDE_START":
466  startLongitude = bba.getValueDouble();
467  break;
468  case "TSK_GEO_LATITUDE_END":
469  startLatitude = bba.getValueDouble();
470  break;
471  case "TSK_GEO_LONGITUDE_END":
472  startLongitude = bba.getValueDouble();
473  break;
474  case "TSK_DATETIME":
475  dateTime = bba.getValueLong();
476  break;
477  case "TSK_NAME":
478  destinationName = bba.getValueString();
479  break;
480  case "TSK_LOCATION":
481  locationName = bba.getValueString();
482  break;
483  case "TSK_TEXT_FILE":
484  sourceFile = bba.getValueString();
485  break;
486  case "TSK_COMMENT":
487  comment = bba.getValueString();
488  break;
489  default:
490  otherAttributes.add(bba);
491  break;
492  }
493  }
494  absFile = findAbstractFile(dataSource, sourceFile);
495  if (absFile == null) {
496  absFile = (AbstractFile) dataSource;
497  }
498  GeoWaypoints waypointList = new GeoWaypoints();
499  waypointList.addPoint(new Waypoint(startLatitude, startLongitude, zeroValue, ""));
500  waypointList.addPoint(new Waypoint(endLatitude, endLongitude, zeroValue, locationName));
501  (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, comment, absFile, context.getJobId())).addRoute(destinationName, dateTime, waypointList, new ArrayList<>());
502 
503  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
504  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_waypoint_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
505  }
506 
507  }
508 
509  @NbBundle.Messages({
510  "LeappFileProcessor.cannot.create.trackpoint.relationship=Cannot create TSK_TRACK_POINT artifact."
511  })
512  private AbstractFile createTrackpoint(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList) throws IngestModuleException {
513 
514  Double latitude = Double.valueOf(0);
515  Double longitude = Double.valueOf(0);
516  Double altitude = Double.valueOf(0);
517  Double zeroValue = Double.valueOf(0);
518  String segmentName = null;
519  Long dateTime = Long.valueOf(0);
520  Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
521  String sourceFile = null;
522  String comment = null;
523  AbstractFile absFile = null;
524 
525  try {
526  for (BlackboardAttribute bba : bbattributes) {
527  switch (bba.getAttributeType().getTypeName()) {
528  case "TSK_GEO_LATITUDE":
529  latitude = bba.getValueDouble();
530  break;
531  case "TSK_GEO_LONGITUDE":
532  longitude = bba.getValueDouble();
533  break;
534  case "TSK_GEO_ALTITUDE":
535  altitude = bba.getValueDouble();
536  break;
537  case "TSK_DATETIME":
538  dateTime = bba.getValueLong();
539  break;
540  case "TSK_NAME":
541  segmentName = bba.getValueString();
542  break;
543  case "TSK_TEXT_FILE":
544  sourceFile = bba.getValueString();
545  break;
546  case "TSK_COMMENT":
547  comment = bba.getValueString();
548  otherAttributes.add(bba);
549  break;
550  default:
551  otherAttributes.add(bba);
552  break;
553  }
554  }
555  absFile = findAbstractFile(dataSource, sourceFile);
556  if (absFile == null) {
557  absFile = (AbstractFile) dataSource;
558  }
559  if ((trackpointSegmentName == null) || (trackpointSegmentName.equals(segmentName))) {
560  pointList.addPoint(new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime));
561  } else {
562  (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, comment, absFile, context.getJobId())).addTrack(segmentName, pointList, new ArrayList<>());
563  pointList.addPoint(new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime));
564 
565  }
566  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
567  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_trackpoint_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
568  }
569 
570  return absFile;
571 
572  }
573 
574  @NbBundle.Messages({
575  "LeappFileProcessor.cannot.create.message.relationship=Cannot create TSK_MESSAGE Relationship."
576  })
577  private void createMessageRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
578 
579  String messageType = null;
580  String alternateId = null;
581  CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN;
582  String senderId = null;
583  String receipentId = null;
584  String[] receipentIdList = null;
585  Long dateTime = Long.valueOf(0);
586  MessageReadStatus messageStatus = MessageReadStatus.UNKNOWN;
587  String subject = null;
588  String messageText = null;
589  String threadId = null;
590  List<BlackboardAttribute> otherAttributes = new ArrayList<>();
591  List<FileAttachment> fileAttachments = new ArrayList<>();
592  String sourceFile = null;
593  MessageAttachments messageAttachments;
594 
595  try {
596  for (BlackboardAttribute bba : bbattributes) {
597  switch (bba.getAttributeType().getTypeName()) {
598  case "TSK_DIRECTION":
599  if (bba.getValueString().toLowerCase().equals("outgoing")) {
600  communicationDirection = CommunicationDirection.OUTGOING;
601  } else if (bba.getValueString().toLowerCase().equals("incoming")) {
602  communicationDirection = CommunicationDirection.INCOMING;
603  }
604  break;
605  case "TSK_PHONE_NUMBER_FROM":
606  if (!bba.getValueString().isEmpty()) {
607  senderId = bba.getValueString();
608  }
609  break;
610  case "TSK_PHONE_NUMBER_TO":
611  if (!bba.getValueString().isEmpty()) {
612  receipentIdList = bba.getValueString().split(",", 0);
613  }
614  break;
615  case "TSK_DATETIME":
616  dateTime = bba.getValueLong();
617  break;
618  case "TSK_COMMENT":
619  messageType = bba.getValueString();
620  break;
621  case "TSK_ATTACHMENTS":
622  if (!bba.getValueString().isEmpty()) {
623  fileAttachments.add(new FileAttachment(Case.getCurrentCaseThrows().getSleuthkitCase(), dataSource, bba.getValueString()));
624  }
625  break;
626  case "TSK_TEXT_FILE":
627  sourceFile = bba.getValueString();
628  break;
629  case "TSK_READ_STATUS":
630  if (bba.getValueInt() == 1) {
631  messageStatus = MessageReadStatus.READ;
632  } else {
633  messageStatus = MessageReadStatus.UNREAD;
634  }
635  break;
636  case "TSK_TEXT":
637  messageText = bba.getValueString();
638  break;
639  case "TSK_SUBJECT":
640  subject = bba.getValueString();
641  break;
642  case "TSK_ID":
643  alternateId = bba.getValueString();
644  otherAttributes.add(bba);
645  break;
646  default:
647  otherAttributes.add(bba);
648  break;
649  }
650  }
651  AbstractFile absFile = findAbstractFile(dataSource, sourceFile);
652  if (absFile == null) {
653  absFile = (AbstractFile) dataSource;
654  }
655  CommunicationArtifactsHelper accountHelper;
656  Account.Type accountType = getAccountType(fileName);
657  if (alternateId == null) {
658  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
659  moduleName, absFile, accountType, context.getJobId());
660  } else {
661  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
662  moduleName, absFile, accountType, accountType, alternateId, context.getJobId());
663  }
664  BlackboardArtifact messageArtifact = accountHelper.addMessage(messageType, communicationDirection, senderId,
665  receipentId, dateTime, messageStatus, subject,
666  messageText, threadId, otherAttributes);
667  if (!fileAttachments.isEmpty()) {
668  messageAttachments = new MessageAttachments(fileAttachments, new ArrayList<>());
669  accountHelper.addAttachments(messageArtifact, messageAttachments);
670  }
671  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
672  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
673  }
674 
675  }
676 
677  @NbBundle.Messages({
678  "LeappFileProcessor.cannot.create.contact.relationship=Cannot create TSK_CONTACT Relationship."
679  })
680  private void createContactRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
681 
682  String alternateId = null;
683  String contactName = null;
684  String phoneNumber = null;
685  String homePhoneNumber = null;
686  String mobilePhoneNumber = null;
687  String emailAddr = null;
688  List<BlackboardAttribute> otherAttributes = new ArrayList<>();
689  String sourceFile = null;
690 
691  try {
692  for (BlackboardAttribute bba : bbattributes) {
693  switch (bba.getAttributeType().getTypeName()) {
694  case "TSK_PHONE_NUMBER":
695  if (!bba.getValueString().isEmpty()) {
696  phoneNumber = bba.getValueString();
697  }
698  break;
699  case "TSK_NAME":
700  if (!bba.getValueString().isEmpty()) {
701  contactName = bba.getValueString();
702  }
703  break;
704  case "TSK_TEXT_FILE":
705  sourceFile = bba.getValueString();
706  break;
707  case "TSK_PHONE_NUMBER_HOME":
708  homePhoneNumber = bba.getValueString();
709  break;
710  case "TSK_PHONE_NUMBER_MOBILE":
711  mobilePhoneNumber = bba.getValueString();
712  break;
713  case "TSK_EMAIL":
714  emailAddr = bba.getValueString();
715  break;
716  case "TSK_ID":
717  alternateId = bba.getValueString();
718  otherAttributes.add(bba);
719  break;
720  default:
721  otherAttributes.add(bba);
722  break;
723  }
724  }
725  AbstractFile absFile = findAbstractFile(dataSource, sourceFile);
726  if (absFile == null) {
727  absFile = (AbstractFile) dataSource;
728  }
729  Account.Type accountType = getAccountType(fileName);
730  if (accountType != null) {
731 
732  CommunicationArtifactsHelper accountHelper;
733  if (alternateId == null) {
734  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
735  moduleName, absFile, accountType, context.getJobId());
736  } else {
737  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
738  moduleName, absFile, accountType, accountType, alternateId, context.getJobId());
739  }
740  BlackboardArtifact messageArtifact = accountHelper.addContact(contactName, phoneNumber, homePhoneNumber, mobilePhoneNumber, emailAddr, otherAttributes);
741  }
742  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
743  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_contact_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
744  }
745  }
746 
747  @NbBundle.Messages({
748  "LeappFileProcessor.cannot.create.calllog.relationship=Cannot create TSK_CALLLOG Relationship."
749  })
750  private void createCalllogRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
751 
752  String callerId = null;
753  String alternateId = null;
754  List<String> calleeId = Arrays.asList();
755  CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN;
756  Long startDateTime = Long.valueOf(0);
757  Long endDateTime = Long.valueOf(0);
758  CallMediaType mediaType = CallMediaType.UNKNOWN;
759  List<BlackboardAttribute> otherAttributes = new ArrayList<>();
760  String sourceFile = null;
761 
762  try {
763  for (BlackboardAttribute bba : bbattributes) {
764  switch (bba.getAttributeType().getTypeName()) {
765  case "TSK_TEXT_FILE":
766  sourceFile = bba.getValueString();
767  break;
768  case "TSK_DATETIME_START":
769  startDateTime = bba.getValueLong();
770  break;
771  case "TSK_DATETIME_END":
772  startDateTime = bba.getValueLong();
773  break;
774  case "TSK_DIRECTION":
775  if (bba.getValueString().toLowerCase().equals("outgoing")) {
776  communicationDirection = CommunicationDirection.OUTGOING;
777  } else if (bba.getValueString().toLowerCase().equals("incoming")) {
778  communicationDirection = CommunicationDirection.INCOMING;
779  }
780  break;
781  case "TSK_PHONE_NUMBER_FROM":
782  if (!bba.getValueString().isEmpty()) {
783  callerId = bba.getValueString();
784  }
785  break;
786  case "TSK_PHONE_NUMBER_TO":
787  if (!bba.getValueString().isEmpty()) {
788  String[] calleeTempList = bba.getValueString().split(",", 0);
789  calleeId = Arrays.asList(calleeTempList);
790  }
791  break;
792  case "TSK_ID":
793  alternateId = bba.getValueString();
794  otherAttributes.add(bba);
795  break;
796  default:
797  otherAttributes.add(bba);
798  break;
799  }
800  }
801 
802  if (calleeId.isEmpty() && communicationDirection == CommunicationDirection.OUTGOING && callerId != null) {
803  String[] calleeTempList = callerId.split(",", 0);
804  calleeId = Arrays.asList(calleeTempList);
805  callerId = null;
806  }
807  AbstractFile absFile = findAbstractFile(dataSource, sourceFile);
808  if (absFile == null) {
809  absFile = (AbstractFile) dataSource;
810  }
811  Account.Type accountType = getAccountType(fileName);
812  CommunicationArtifactsHelper accountHelper;
813  if (accountType != null) {
814  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
815  moduleName, absFile, accountType, context.getJobId());
816  } else {
817  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
818  moduleName, absFile, accountType, accountType, alternateId, context.getJobId());
819  }
820  accountHelper.addCalllog(communicationDirection, callerId, calleeId, startDateTime, endDateTime, mediaType, otherAttributes);
821  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
822  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_calllog_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
823  }
824 
825  }
826 
827  private Account.Type getAccountType(String AccountTypeName) {
828  switch (AccountTypeName.toLowerCase()) {
829  case "zapya.tsv":
830  return Account.Type.ZAPYA;
831  case "sms messages.tsv":
832  return Account.Type.PHONE;
833  case "contacts.tsv":
834  return Account.Type.PHONE;
835  case "imo - accountid.tsv":
836  return Account.Type.IMO;
837  case "imo - messages.tsv":
838  return Account.Type.IMO;
839  case "textnow - contacts.tsv":
840  return Account.Type.TEXTNOW;
841  case "textnow - messages.tsv":
842  return Account.Type.TEXTNOW;
843  case "mms messages.tsv":
844  return Account.Type.PHONE;
845  case "viber - call logs.tsv":
846  return Account.Type.VIBER;
847  case "viber - contacts.tsv":
848  return Account.Type.VIBER;
849  case "viber - messages.tsv":
850  return Account.Type.VIBER;
851  case "xender file transfer - messages.tsv":
852  return Account.Type.XENDER;
853  case "xender file transfer - contacts.tsv":
854  return Account.Type.XENDER;
855  case "whatsapp - single call logs.tsv":
856  return Account.Type.WHATSAPP;
857  case "whatsapp - messages logs.tsv":
858  return Account.Type.WHATSAPP;
859  case "whatsapp - group call logs.tsv":
860  return Account.Type.WHATSAPP;
861  case "whatsapp - contacts.tsv":
862  return Account.Type.WHATSAPP;
863  case "tangomessages messages.tsv":
864  return Account.Type.TANGO;
865  case "shareit file transfer.tsv":
866  return Account.Type.SHAREIT;
867  case "line - calllogs.tsv":
868  return Account.Type.LINE;
869  case "line - contacts.tsv":
870  return Account.Type.LINE;
871  case "line - messages.tsv":
872  return Account.Type.LINE;
873  case "skype - call logs.tsv":
874  return Account.Type.SKYPE;
875  case "skype - contacts.tsv":
876  return Account.Type.SKYPE;
877  case "skype - messages logs.tsv":
878  return Account.Type.SKYPE;
879  case "facebook messenger - calls.tsv":
880  return Account.Type.FACEBOOK;
881  case "facebook messenger - contacts.tsv":
882  return Account.Type.FACEBOOK;
883  case "facebook messenger - chats.tsv":
884  return Account.Type.FACEBOOK;
885  case "call logs2.tsv":
886  return Account.Type.PHONE;
887  case "call logs.tsv":
888  return Account.Type.PHONE;
889  case "sms - imessage.tsv":
890  return Account.Type.PHONE;
891  default:
892  return Account.Type.PHONE;
893  }
894  }
895 
913  private Collection<BlackboardAttribute> processReadLine(List<String> lineValues, Map<String, Integer> columnIndexes,
914  List<TsvColumn> attrList, String fileName, int lineNum) throws IngestModuleException {
915 
916  // if no attributes, return an empty row
917  if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues)
918  || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) {
919  return Collections.emptyList();
920  }
921 
922  List<BlackboardAttribute> attrsToRet = new ArrayList<>();
923  for (TsvColumn colAttr : attrList) {
924  // if no matching attribute type, keep going
925  if (colAttr.getAttributeType() == null) {
926  // this handles columns that are currently ignored.
927  continue;
928  }
929 
930  Integer columnIdx = columnIndexes.get(colAttr.getColumnName());
931  if (columnIdx == null) {
932  logger.log(Level.WARNING, String.format("No column mapping found for %s in file %s. Omitting column.", colAttr.getColumnName(), fileName));
933  continue;
934  }
935 
936  String value = (columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx);
937  if (value == null) {
938  // if column is required, return empty for this row if no value
939  if (colAttr.isRequired()) {
940  logger.log(Level.WARNING, String.format("No value found for required column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName));
941  return Collections.emptyList();
942  } else {
943  // otherwise, continue to next column
944  logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s. Omitting column.", colAttr.getColumnName(), lineNum, fileName));
945  continue;
946  }
947  }
948 
949  String formattedValue = formatValueBasedOnAttrType(colAttr, value);
950 
951  BlackboardAttribute attr = getAttribute(colAttr.getAttributeType(), formattedValue, fileName);
952  if (attr != null) {
953  attrsToRet.add(attr);
954  } else if (colAttr.isRequired()) {
955  logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName));
956  return Collections.emptyList();
957  }
958  }
959 
960  if (tsvFileArtifactComments.containsKey(normalizeKey(fileName))) {
961  attrsToRet.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, tsvFileArtifactComments.get(normalizeKey(fileName))));
962  }
963 
964  return attrsToRet;
965  }
966 
976  private String formatValueBasedOnAttrType(TsvColumn colAttr, String value) {
977  if (colAttr.getAttributeType().getTypeName().equals("TSK_DOMAIN")) {
978  return NetworkUtils.extractDomain(value);
979  }
980 
981  return value;
982  }
983 
987  private static final DateFormat TIMESTAMP_FORMAT = new SimpleDateFormat("yyyy-MM-d HH:mm:ss", US);
988 
1000  private BlackboardAttribute getAttribute(BlackboardAttribute.Type attrType, String value, String fileName) {
1001  if (attrType == null || value == null) {
1002  logger.log(Level.WARNING, String.format("Unable to parse attribute type %s for value '%s' in fileName %s",
1003  attrType == null ? "<null>" : attrType.toString(),
1004  value == null ? "<null>" : value,
1005  fileName == null ? "<null>" : fileName));
1006  return null;
1007  }
1008 
1009  switch (attrType.getValueType()) {
1010  case JSON:
1011  case STRING:
1012  return parseAttrValue(value, attrType, fileName, false, false,
1013  (v) -> new BlackboardAttribute(attrType, moduleName, v));
1014  case INTEGER:
1015  return parseAttrValue(value.trim(), attrType, fileName, true, false,
1016  (v) -> new BlackboardAttribute(attrType, moduleName, Double.valueOf(v).intValue()));
1017  case LONG:
1018  return parseAttrValue(value.trim(), attrType, fileName, true, false,
1019  (v) -> new BlackboardAttribute(attrType, moduleName, Double.valueOf(v).longValue()));
1020  case DOUBLE:
1021  return parseAttrValue(value.trim(), attrType, fileName, true, false,
1022  (v) -> new BlackboardAttribute(attrType, moduleName, Double.valueOf(v)));
1023  case BYTE:
1024  return parseAttrValue(value.trim(), attrType, fileName, true, false,
1025  (v) -> new BlackboardAttribute(attrType, moduleName, new byte[]{Byte.valueOf(v)}));
1026  case DATETIME:
1027  return parseAttrValue(value.trim(), attrType, fileName, true, true,
1028  (v) -> new BlackboardAttribute(attrType, moduleName, TIMESTAMP_FORMAT.parse(v).getTime() / 1000));
1029  default:
1030  // Log this and continue on with processing
1031  logger.log(Level.WARNING, String.format("Attribute Type %s for file %s not defined.", attrType, fileName)); //NON-NLS
1032  return null;
1033 
1034  }
1035  }
1036 
1040  private interface ParseExceptionFunction {
1041 
1052  BlackboardAttribute apply(String orig) throws ParseException, NumberFormatException;
1053  }
1054 
1070  private BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter) {
1071  // remove non-printable characters from tsv input
1072  // https://stackoverflow.com/a/6199346
1073  String sanitizedValue = value.replaceAll("\\p{C}", "");
1074 
1075  if (blankIsNull && StringUtils.isBlank(sanitizedValue)) {
1076  return null;
1077  }
1078 
1079  if (zeroIsNull && sanitizedValue.matches("^\\s*[0\\.]*\\s*$")) {
1080  return null;
1081  }
1082 
1083  try {
1084  return valueConverter.apply(sanitizedValue);
1085  } catch (NumberFormatException | ParseException ex) {
1086  logger.log(Level.WARNING, String.format("Unable to format '%s' as value type %s while converting to attributes from %s.", sanitizedValue, attrType.getValueType().getLabel(), fileName), ex);
1087  return null;
1088  }
1089  }
1090 
1094  private void loadConfigFile() throws IngestModuleException {
1095  String path = PlatformUtil.getUserConfigDirectory() + File.separator + xmlFile;
1096  loadIndividualConfigFile(path);
1097  String userPath = PlatformUtil.getUserConfigDirectory() + File.separator + leapModule + "-" + ARTIFACT_ATTRIBUTE_REFERENCE_USER;
1098  if (new File(userPath).exists()) {
1099  loadIndividualConfigFile(userPath);
1100  }
1101  }
1102 
1106  @NbBundle.Messages({
1107  "LeappFileProcessor.cannot.load.artifact.xml=Cannot load xml artifact file.",
1108  "LeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.",
1109  "LeappFileProcessor_cannotParseXml=Cannot Parse XML file.",
1110  "LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact",
1111  "LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts."
1112  })
1113  private void loadIndividualConfigFile(String path) throws IngestModuleException {
1114  Document xmlinput;
1115  try {
1116  File f = new File(path);
1117  DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
1118  DocumentBuilder db = dbf.newDocumentBuilder();
1119  xmlinput = db.parse(f);
1120 
1121  } catch (IOException e) {
1122  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_load_artifact_xml() + e.getLocalizedMessage(), e); //NON-NLS
1123  } catch (ParserConfigurationException pce) {
1124  throw new IngestModuleException(Bundle.LeappFileProcessor_cannotBuildXmlParser() + pce.getLocalizedMessage(), pce); //NON-NLS
1125  } catch (SAXException sxe) {
1126  throw new IngestModuleException(Bundle.LeappFileProcessor_cannotParseXml() + sxe.getLocalizedMessage(), sxe); //NON-NLS
1127  }
1128 
1129  getFileNode(xmlinput);
1130  getArtifactNode(xmlinput);
1131  getAttributeNodes(xmlinput);
1132 
1133  }
1134 
1135  private void getFileNode(Document xmlinput) {
1136 
1137  NodeList nlist = xmlinput.getElementsByTagName("FileName"); //NON-NLS
1138 
1139  for (int i = 0; i < nlist.getLength(); i++) {
1140  NamedNodeMap nnm = nlist.item(i).getAttributes();
1141  tsvFiles.put(normalizeKey(nnm.getNamedItem("filename").getNodeValue()), nnm.getNamedItem("description").getNodeValue());
1142 
1143  }
1144 
1145  }
1146 
1147  private void getArtifactNode(Document xmlinput) {
1148 
1149  NodeList artifactNlist = xmlinput.getElementsByTagName("ArtifactName"); //NON-NLS
1150  for (int k = 0; k < artifactNlist.getLength(); k++) {
1151  NamedNodeMap nnm = artifactNlist.item(k).getAttributes();
1152  String artifactName = nnm.getNamedItem("artifactname").getNodeValue();
1153  String comment = nnm.getNamedItem("comment").getNodeValue();
1154  String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
1155 
1156  BlackboardArtifact.Type foundArtifactType = null;
1157  try {
1158  foundArtifactType = Case.getCurrentCase().getSleuthkitCase().getBlackboard().getArtifactType(artifactName);
1159  } catch (TskCoreException ex) {
1160  logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex);
1161  }
1162 
1163  if (foundArtifactType == null) {
1164  logger.log(Level.SEVERE, String.format("No known artifact mapping found for [artifact: %s, %s]",
1165  artifactName, getXmlFileIdentifier(parentName)));
1166  } else {
1167  tsvFileArtifacts.put(normalizeKey(parentName), foundArtifactType);
1168  }
1169 
1170  if (!comment.toLowerCase().matches("null")) {
1171  tsvFileArtifactComments.put(normalizeKey(parentName), comment);
1172  }
1173  }
1174 
1175  }
1176 
1177  private String getXmlFileIdentifier(String fileName) {
1178  return String.format("file: %s, filename: %s",
1179  this.xmlFile == null ? "<null>" : this.xmlFile,
1180  fileName == null ? "<null>" : fileName);
1181  }
1182 
1183  private String getXmlAttrIdentifier(String fileName, String attributeName) {
1184  return String.format("attribute: %s %s",
1185  attributeName == null ? "<null>" : attributeName,
1186  getXmlFileIdentifier(fileName));
1187  }
1188 
1189  private void getAttributeNodes(Document xmlinput) {
1190 
1191  NodeList attributeNlist = xmlinput.getElementsByTagName("AttributeName"); //NON-NLS
1192  for (int k = 0; k < attributeNlist.getLength(); k++) {
1193  NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
1194  String attributeName = nnm.getNamedItem("attributename").getNodeValue();
1195 
1196  if (!attributeName.toLowerCase().matches("null")) {
1197  String columnName = nnm.getNamedItem("columnName").getNodeValue();
1198  String required = nnm.getNamedItem("required").getNodeValue();
1199  String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
1200 
1201  BlackboardAttribute.Type foundAttrType = null;
1202  try {
1203  foundAttrType = Case.getCurrentCase().getSleuthkitCase().getBlackboard().getAttributeType(attributeName.toUpperCase());
1204  } catch (TskCoreException ex) {
1205  logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch attribute type for %s.", attributeName), ex);
1206  }
1207 
1208  if (foundAttrType == null) {
1209  logger.log(Level.SEVERE, String.format("No known attribute mapping found for [%s]", getXmlAttrIdentifier(parentName, attributeName)));
1210  }
1211 
1212  if (required != null && required.compareToIgnoreCase("yes") != 0 && required.compareToIgnoreCase("no") != 0) {
1213  logger.log(Level.SEVERE, String.format("Required value %s did not match 'yes' or 'no' for [%s]",
1214  required, getXmlAttrIdentifier(parentName, attributeName)));
1215  }
1216 
1217  if (columnName == null) {
1218  logger.log(Level.SEVERE, String.format("No column name provided for [%s]", getXmlAttrIdentifier(parentName, attributeName)));
1219  continue;
1220  } else if (columnName.trim().length() != columnName.length()) {
1221  logger.log(Level.SEVERE, String.format("Column name '%s' starts or ends with whitespace for [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName)));
1222  continue;
1223  } else if (columnName.matches("[^ \\S]")) {
1224  logger.log(Level.SEVERE, String.format("Column name '%s' contains invalid characters [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName)));
1225  continue;
1226  }
1227 
1228  TsvColumn thisCol = new TsvColumn(
1229  foundAttrType,
1230  columnName.trim().toLowerCase(),
1231  "yes".compareToIgnoreCase(required) == 0);
1232 
1233  if (tsvFileAttributes.containsKey(normalizeKey(parentName))) {
1234  List<TsvColumn> attrList = tsvFileAttributes.get(normalizeKey(parentName));
1235  attrList.add(thisCol);
1236  tsvFileAttributes.replace(parentName, attrList);
1237  } else {
1238  List<TsvColumn> attrList = new ArrayList<>();
1239  attrList.add(thisCol);
1240  tsvFileAttributes.put(normalizeKey(parentName), attrList);
1241  }
1242  }
1243 
1244  }
1245  }
1246 
1259  private BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.Type artType, Content dataSource, Collection<BlackboardAttribute> bbattributes) {
1260  try {
1261  switch (artType.getCategory()) {
1262  case DATA_ARTIFACT:
1263  return dataSource.newDataArtifact(artType, bbattributes);
1264  case ANALYSIS_RESULT:
1265  return dataSource.newAnalysisResult(artType, Score.SCORE_UNKNOWN, null, null, null, bbattributes).getAnalysisResult();
1266  default:
1267  logger.log(Level.SEVERE, String.format("Unknown category type: %s", artType.getCategory().getDisplayName()));
1268  return null;
1269  }
1270  } catch (TskException ex) {
1271  logger.log(Level.WARNING, Bundle.LeappFileProcessor_error_creating_new_artifacts(), ex); //NON-NLS
1272  }
1273  return null;
1274  }
1275 
1282  void postArtifacts(Collection<BlackboardArtifact> artifacts) {
1283  if (artifacts == null || artifacts.isEmpty()) {
1284  return;
1285  }
1286 
1287  try {
1288  Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifacts(artifacts, moduleName, context.getJobId());
1289  } catch (Blackboard.BlackboardException ex) {
1290  logger.log(Level.SEVERE, Bundle.LeappFileProcessor_postartifacts_error(), ex); //NON-NLS
1291  }
1292  }
1293 
1299  private void configExtractor() throws IOException {
1300  PlatformUtil.extractResourceToUserConfigDir(LeappFileProcessor.class,
1301  xmlFile, true);
1302  }
1303 
1304  private static final Set<String> ALLOWED_EXTENSIONS = new HashSet<>(Arrays.asList("zip", "tar", "tgz"));
1305 
1313  static List<AbstractFile> findLeappFilesToProcess(Content dataSource) {
1314 
1315  List<AbstractFile> leappFiles = new ArrayList<>();
1316 
1317  FileManager fileManager = getCurrentCase().getServices().getFileManager();
1318 
1319  // findFiles use the SQL wildcard % in the file name
1320  try {
1321  leappFiles = fileManager.findFiles(dataSource, "%", "/"); //NON-NLS
1322  } catch (TskCoreException ex) {
1323  logger.log(Level.WARNING, "No files found to process"); //NON-NLS
1324  return leappFiles;
1325  }
1326 
1327  List<AbstractFile> leappFilesToProcess = new ArrayList<>();
1328  for (AbstractFile leappFile : leappFiles) {
1329  if (((leappFile.getLocalAbsPath() != null)
1330  && !leappFile.isVirtual())
1331  && leappFile.getNameExtension() != null
1332  && ALLOWED_EXTENSIONS.contains(leappFile.getNameExtension().toLowerCase())) {
1333  leappFilesToProcess.add(leappFile);
1334  }
1335  }
1336 
1337  return leappFilesToProcess;
1338  }
1339 
1344  private void loadCustomArtifactsAttributes(Blackboard blkBoard, String leapModule) {
1345 
1346  for (Map.Entry<String, String> customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) {
1347  String artifactName = customArtifact.getKey();
1348  String artifactDescription = customArtifact.getValue();
1349  createCustomAttributesArtifacts(blkBoard, "artifact", artifactName, artifactDescription, null);
1350  }
1351 
1352  File customFilePath = new File(PlatformUtil.getUserConfigDirectory() + File.separator + leapModule + '-' + CUSTOM_ARTIFACTS_ATTRIBUTES_FILE);
1353  if (customFilePath.exists()) {
1354  try (MappingIterator<List<String>> iterator = new CsvMapper()
1355  .enable(CsvParser.Feature.WRAP_AS_ARRAY)
1356  .readerFor(List.class)
1357  .with(CsvSchema.emptySchema().withColumnSeparator(','))
1358  .readValues(customFilePath)) {
1359 
1360  if (iterator.hasNext()) {
1361  // Header line we can skip
1362  List<String> headerItems = iterator.next();
1363  int lineNum = 2;
1364  while (iterator.hasNext()) {
1365  List<String> columnItems = iterator.next();
1366  if (columnItems.size() > 3) {
1367  createCustomAttributesArtifacts(blkBoard, columnItems.get(0), columnItems.get(1), columnItems.get(2), columnItems.get(3));
1368  } else {
1369  createCustomAttributesArtifacts(blkBoard, columnItems.get(0), columnItems.get(1), columnItems.get(2), null);
1370  }
1371  }
1372  }
1373  } catch (IOException ex) {
1374  logger.log(Level.WARNING, String.format("Failed to read/open file %s.", customFilePath), ex);
1375  }
1376  }
1377  }
1378 
1383  private void createCustomAttributesArtifacts(Blackboard blkBoard, String atType, String atName, String atDescription, String attrType) {
1384 
1385  if (atType.toLowerCase().equals("artifact")) {
1386  try {
1387  BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(atName.toUpperCase(), atDescription);
1388  } catch (Blackboard.BlackboardException ex) {
1389  logger.log(Level.WARNING, String.format("Failed to create custom artifact type %s.", atName), ex);
1390  }
1391  return;
1392  }
1393 
1394  switch (attrType.toLowerCase()) {
1395  case "json":
1396  case "string":
1397  try {
1398  BlackboardAttribute.Type customAttrbiuteType = blkBoard.getOrAddAttributeType(atName.toUpperCase(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, atDescription);
1399  } catch (Blackboard.BlackboardException ex) {
1400  logger.log(Level.WARNING, String.format("Failed to create custom attribute type %s.", atName), ex);
1401  }
1402  return;
1403  case "integer":
1404  try {
1405  BlackboardAttribute.Type customAttrbiuteType = blkBoard.getOrAddAttributeType(atName.toUpperCase(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, atDescription);
1406  } catch (Blackboard.BlackboardException ex) {
1407  logger.log(Level.WARNING, String.format("Failed to create custom attribute type %s.", atName), ex);
1408  }
1409  return;
1410  case "long":
1411  try {
1412  BlackboardAttribute.Type customAttrbiuteType = blkBoard.getOrAddAttributeType(atName.toUpperCase(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, atDescription);
1413  } catch (Blackboard.BlackboardException ex) {
1414  logger.log(Level.WARNING, String.format("Failed to create custom attribute type %s.", atName), ex);
1415  }
1416  return;
1417  case "double":
1418  try {
1419  BlackboardAttribute.Type customAttrbiuteType = blkBoard.getOrAddAttributeType(atName.toUpperCase(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, atDescription);
1420  } catch (Blackboard.BlackboardException ex) {
1421  logger.log(Level.WARNING, String.format("Failed to create custom attribute type %s.", atName), ex);
1422  }
1423  return;
1424  case "byte":
1425  try {
1426  BlackboardAttribute.Type customAttrbiuteType = blkBoard.getOrAddAttributeType(atName.toUpperCase(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.BYTE, atDescription);
1427  } catch (Blackboard.BlackboardException ex) {
1428  logger.log(Level.WARNING, String.format("Failed to create custom attribute type %s.", atName), ex);
1429  }
1430  return;
1431  case "datetime":
1432  try {
1433  BlackboardAttribute.Type customAttrbiuteType = blkBoard.getOrAddAttributeType(atName.toUpperCase(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DATETIME, atDescription);
1434  } catch (Blackboard.BlackboardException ex) {
1435  logger.log(Level.WARNING, String.format("Failed to create custom attribute type %s.", atName), ex);
1436  }
1437  return;
1438  default:
1439  logger.log(Level.WARNING, String.format("Attribute Type %s for file %s not defined.", attrType, atName)); //NON-NLS
1440  return;
1441 
1442  }
1443  }
1444 
1449  private void createCustomArtifacts(Blackboard blkBoard) {
1450 
1451  for (Map.Entry<String, String> customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) {
1452  String artifactName = customArtifact.getKey();
1453  String artifactDescription = customArtifact.getValue();
1454 
1455  try {
1456  BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(artifactName, artifactDescription);
1457  } catch (Blackboard.BlackboardException ex) {
1458  logger.log(Level.WARNING, String.format("Failed to create custom artifact type %s.", artifactName), ex);
1459  }
1460 
1461  }
1462  }
1463 
1464  private AbstractFile findAbstractFile(Content dataSource, String fileNamePath) {
1465  if (fileNamePath == null) {
1466  return null;
1467  }
1468 
1469  List<AbstractFile> files;
1470 
1471  String fileName = FilenameUtils.getName(fileNamePath);
1472  String filePath = FilenameUtils.normalize(FilenameUtils.getPath(fileNamePath), true);
1473 
1474  FileManager fileManager = Case.getCurrentCase().getServices().getFileManager();
1475 
1476  try {
1477  files = fileManager.findFiles(dataSource, fileName); //NON-NLS
1478 
1479  } catch (TskCoreException ex) {
1480  logger.log(Level.WARNING, "Unable to find prefetch files.", ex); //NON-NLS
1481  return null; // No need to continue
1482  }
1483 
1484  for (AbstractFile pFile : files) {
1485 
1486  if (pFile.getParentPath().toLowerCase().endsWith(filePath.toLowerCase())) {
1487  return pFile;
1488  }
1489  }
1490 
1491  return null;
1492 
1493  }
1494 }
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createMessageRelationship
void createMessageRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
Definition: LeappFileProcessor.java:577
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createArtifactWithAttributes
BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.Type artType, Content dataSource, Collection< BlackboardAttribute > bbattributes)
Definition: LeappFileProcessor.java:1259
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createContactRelationship
void createContactRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
Definition: LeappFileProcessor.java:680
org.sleuthkit.autopsy.coreutils.NetworkUtils.extractDomain
static String extractDomain(String urlString)
Definition: NetworkUtils.java:106
org.sleuthkit.autopsy.casemodule.services.FileManager.findFiles
List< AbstractFile > findFiles(String fileName)
Definition: FileManager.java:175
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createCalllogRelationship
void createCalllogRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
Definition: LeappFileProcessor.java:750
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.processFile
void processFile(File LeappFile, List< TsvColumn > attrList, String fileName, BlackboardArtifact.Type artifactType, Content dataSource)
Definition: LeappFileProcessor.java:358
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.loadCustomArtifactsAttributes
void loadCustomArtifactsAttributes(Blackboard blkBoard, String leapModule)
Definition: LeappFileProcessor.java:1344
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.tsvFileArtifacts
final Map< String, BlackboardArtifact.Type > tsvFileArtifacts
Definition: LeappFileProcessor.java:157
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.findAbstractFile
AbstractFile findAbstractFile(Content dataSource, String fileNamePath)
Definition: LeappFileProcessor.java:1464
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createRoute
void createRoute(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
Definition: LeappFileProcessor.java:444
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.processReadLine
Collection< BlackboardAttribute > processReadLine(List< String > lineValues, Map< String, Integer > columnIndexes, List< TsvColumn > attrList, String fileName, int lineNum)
Definition: LeappFileProcessor.java:913
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.parseAttrValue
BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter)
Definition: LeappFileProcessor.java:1070
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.processLeappFiles
void processLeappFiles(List< String > LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress)
Definition: LeappFileProcessor.java:330
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createCustomAttributesArtifacts
void createCustomAttributesArtifacts(Blackboard blkBoard, String atType, String atName, String atDescription, String attrType)
Definition: LeappFileProcessor.java:1383
org.sleuthkit.autopsy.coreutils.PlatformUtil.extractResourceToUserConfigDir
static< T > boolean extractResourceToUserConfigDir(final Class< T > resourceClass, final String resourceFileName, boolean overWrite)
Definition: PlatformUtil.java:283
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.getAttribute
BlackboardAttribute getAttribute(BlackboardAttribute.Type attrType, String value, String fileName)
Definition: LeappFileProcessor.java:1000
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.processFileSystem
ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress)
Definition: LeappFileProcessor.java:263
org.sleuthkit.autopsy.coreutils.Logger.getLogger
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.createTrackpoint
AbstractFile createTrackpoint(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList)
Definition: LeappFileProcessor.java:512
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.LeappFileProcessor
LeappFileProcessor(String xmlFile, String moduleName, String leapModule, IngestJobContext context)
Definition: LeappFileProcessor.java:205
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.getXmlAttrIdentifier
String getXmlAttrIdentifier(String fileName, String attributeName)
Definition: LeappFileProcessor.java:1183
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.processFiles
ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress)
Definition: LeappFileProcessor.java:246
org.sleuthkit.autopsy.modules.leappanalyzers.LeappFileProcessor.formatValueBasedOnAttrType
String formatValueBasedOnAttrType(TsvColumn colAttr, String value)
Definition: LeappFileProcessor.java:976

Copyright © 2012-2022 Basis Technology. Generated on: Tue Aug 1 2023
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.