Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
LeappFileProcessor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2020-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.leappanalyzers;
20 
21 import com.fasterxml.jackson.databind.MappingIterator;
22 import com.fasterxml.jackson.dataformat.csv.CsvMapper;
23 import com.fasterxml.jackson.dataformat.csv.CsvParser;
24 import com.fasterxml.jackson.dataformat.csv.CsvSchema;
25 import com.google.common.collect.ImmutableMap;
26 import java.io.File;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.io.UncheckedIOException;
30 import java.nio.file.Files;
31 import java.nio.file.Path;
32 import java.text.DateFormat;
33 import java.text.ParseException;
34 import java.text.SimpleDateFormat;
35 import java.util.List;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import static java.util.Locale.US;
43 import java.util.Map;
44 import java.util.Set;
45 import java.util.logging.Level;
46 import java.util.stream.Collectors;
47 import java.util.stream.IntStream;
48 import java.util.stream.Stream;
49 import javax.xml.parsers.DocumentBuilder;
50 import javax.xml.parsers.DocumentBuilderFactory;
51 import javax.xml.parsers.ParserConfigurationException;
52 import org.apache.commons.collections4.CollectionUtils;
53 import org.apache.commons.collections4.MapUtils;
54 import org.apache.commons.io.FilenameUtils;
55 import org.apache.commons.lang3.StringUtils;
56 import org.openide.util.NbBundle;
57 import org.openide.util.NbBundle.Messages;
69 import org.sleuthkit.datamodel.AbstractFile;
70 import org.sleuthkit.datamodel.Account;
71 import org.sleuthkit.datamodel.Blackboard;
72 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
73 import org.sleuthkit.datamodel.BlackboardArtifact;
74 import org.sleuthkit.datamodel.BlackboardAttribute;
75 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
76 import org.sleuthkit.datamodel.Content;
77 import org.sleuthkit.datamodel.Score;
78 import org.sleuthkit.datamodel.TskCoreException;
79 import org.sleuthkit.datamodel.TskException;
80 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
81 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CallMediaType;
82 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection;
83 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus;
84 import org.sleuthkit.datamodel.blackboardutils.GeoArtifactsHelper;
85 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints;
86 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints.TrackPoint;
87 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints;
88 import org.sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints.Waypoint;
89 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments;
90 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment;
91 import org.w3c.dom.Document;
92 import org.w3c.dom.NamedNodeMap;
93 import org.w3c.dom.NodeList;
94 import org.xml.sax.SAXException;
95 
99 public final class LeappFileProcessor {
100 
104  private static class TsvColumn {
105 
106  private final BlackboardAttribute.Type attributeType;
107  private final String columnName;
108  private final boolean required;
109 
119  TsvColumn(BlackboardAttribute.Type attributeType, String columnName, boolean required) {
121  this.columnName = columnName;
122  this.required = required;
123  }
124 
128  BlackboardAttribute.Type getAttributeType() {
129  return attributeType;
130  }
131 
135  String getColumnName() {
136  return columnName;
137  }
138 
142  boolean isRequired() {
143  return required;
144  }
145  }
146 
147  private static final Logger logger = Logger.getLogger(LeappFileProcessor.class.getName());
148  private final String xmlFile; //NON-NLS
149  private final String moduleName;
150  private final IngestJobContext context;
151 
152  private final Map<String, String> tsvFiles;
153  private final Map<String, BlackboardArtifact.Type> tsvFileArtifacts;
154  private final Map<String, String> tsvFileArtifactComments;
155  private final Map<String, List<TsvColumn>> tsvFileAttributes;
156 
157  private static final Map<String, String> CUSTOM_ARTIFACT_MAP = ImmutableMap.<String, String>builder()
158  .put("TSK_IP_DHCP", "DHCP Information")
159  .build();
160 
161  private static final Map<String, String> ACCOUNT_RELATIONSHIPS = ImmutableMap.<String, String>builder()
162  .put("zapya.tsv", "message")
163  .put("sms messages.tsv", "message")
164  .put("mms messages.tsv", "message")
165  .put("viber - messages.tsv", "message")
166  .put("viber - contacts.tsv", "contact")
167  .put("viber - call logs.tsv", "calllog")
168  .put("xender file transfer - messages.tsv", "message")
169  .put("xender file transfer - contacts.tsv", "contact")
170  .put("whatsapp - contacts.tsv", "contact")
171  .put("whatsapp - group call logs.tsv", "calllog")
172  .put("whatsapp - single call logs.tsv", "calllog")
173  .put("whatsapp - messages logs.tsv", "message")
174  .put("shareit file transfer.tsv", "message")
175  .put("tangomessages messages.tsv", "message")
176  .put("contacts.tsv", "contact")
177  .put("imo - accountid.tsv", "contact")
178  .put("imo - messages.tsv", "message")
179  .put("textnow - contacts.tsv", "contact")
180  .put("textnow - messages.tsv", "message")
181  .put("line - messages.tsv", "message")
182  .put("line - contacts.tsv", "contact")
183  .put("line - calllogs.tsv", "calllog")
184  .put("skype - messages logs.tsv", "message")
185  .put("skype - contacts.tsv", "contact")
186  .put("skype - call logs.tsv", "calllog")
187  .put("facebook messenger - chats.tsv", "message")
188  .put("facebook messenger - contacts.tsv", "contact")
189  .put("facebook messenger - calls.tsv", "calllog")
190  .put("call logs2.tsv", "calllog")
191  .put("call logs.tsv", "calllog")
192  .put("oruxmaps tracks.tsv", "trackpoint")
193  .put("google map locations.tsv", "route")
194  .put("Contacts.tsv", "contact")
195  .put("sms - imessage.tsv", "message")
196  .put("call history.tsv", "calllog")
197  .build();
198 
199  private final Blackboard blkBoard;
200 
201  public LeappFileProcessor(String xmlFile, String moduleName, IngestJobContext context) throws IOException, IngestModuleException, NoCurrentCaseException {
202  this.tsvFiles = new HashMap<>();
203  this.tsvFileArtifacts = new HashMap<>();
204  this.tsvFileArtifactComments = new HashMap<>();
205  this.tsvFileAttributes = new HashMap<>();
206  this.xmlFile = xmlFile;
207  this.moduleName = moduleName;
208  this.context = context;
209 
210  blkBoard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
211 
212  createCustomArtifacts(blkBoard);
213  configExtractor();
214  loadConfigFile();
215 
216  }
217 
218  @NbBundle.Messages({
219  "LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.",
220  "LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory.",
221  "LeappFileProcessor.starting.Leapp=Starting Leapp",
222  "LeappFileProcessor.running.Leapp=Running Leapp",
223  "LeappFileProcessor.has.run=Leapp",
224  "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
225  "LeappFileProcessor.completed=Leapp Processing Completed",
226  "LeappFileProcessor.findTsv=Finding all Leapp ouput",
227  "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"
228  })
229  public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress) {
230  try {
231  if (checkCancelled()) {
232  return ProcessResult.OK;
233  }
234  progress.switchToIndeterminate();
235  progress.progress(Bundle.LeappFileProcessor_findTsv());
236  List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
237  processLeappFiles(LeappTsvOutputFiles, LeappFile, progress);
238  } catch (IngestModuleException ex) {
239  logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
240  return ProcessResult.ERROR;
241  }
242 
243  return ProcessResult.OK;
244  }
245 
246  public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress) {
247  try {
248  if (checkCancelled()) {
249  return ProcessResult.OK;
250  }
251  progress.switchToIndeterminate();
252  progress.progress(Bundle.LeappFileProcessor_findTsv());
253  List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
254  processLeappFiles(LeappTsvOutputFiles, dataSource, progress);
255  } catch (IngestModuleException ex) {
256  logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
257  return ProcessResult.ERROR;
258  }
259 
260  return ProcessResult.OK;
261  }
262 
267  private List<String> findTsvFiles(Path LeappOutputDir) throws IngestModuleException {
268  List<String> allTsvFiles;
269  List<String> foundTsvFiles = new ArrayList<>();
270 
271  try (Stream<Path> walk = Files.walk(LeappOutputDir)) {
272 
273  allTsvFiles = walk.map(x -> x.toString())
274  .filter(f -> f.toLowerCase().endsWith(".tsv")).collect(Collectors.toList());
275 
276  for (String tsvFile : allTsvFiles) {
277  if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile.toLowerCase()))) {
278  foundTsvFiles.add(tsvFile);
279  }
280  }
281 
282  } catch (IOException | UncheckedIOException e) {
283  throw new IngestModuleException(Bundle.LeappFileProcessor_error_reading_Leapp_directory() + LeappOutputDir.toString(), e);
284  }
285 
286  return foundTsvFiles;
287 
288  }
289 
290  private boolean checkCancelled() {
291  if (this.context.dataSourceIngestIsCancelled()) {
292  logger.log(Level.INFO, "Leapp File processing module run was cancelled"); //NON-NLS
293  return true;
294  } else {
295  return false;
296  }
297  }
298 
309  @Messages({
310  "# {0} - fileName",
311  "LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}"
312  })
313  private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException {
314  progress.switchToDeterminate(LeappFilesToProcess.size());
315 
316  for (int i = 0; i < LeappFilesToProcess.size(); i++) {
317  if (checkCancelled()) {
318  return;
319  }
320 
321  String LeappFileName = LeappFilesToProcess.get(i);
322  String fileName = FilenameUtils.getName(LeappFileName);
323  progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i);
324 
325  File LeappFile = new File(LeappFileName);
326  if (tsvFileAttributes.containsKey(fileName)) {
327  List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
328  BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(fileName);
329 
330  try {
331  processFile(LeappFile, attrList, fileName, artifactType, dataSource);
332  } catch (TskCoreException | IOException ex) {
333  logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex);
334  }
335  }
336 
337  }
338  }
339 
340  private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName,
341  BlackboardArtifact.Type artifactType, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
342  TskCoreException {
343 
344  String trackpointSegmentName = null;
345  GeoTrackPoints pointList = new GeoTrackPoints();
346  AbstractFile geoAbstractFile = null;
347 
348  if (LeappFile == null || !LeappFile.exists() || fileName == null) {
349  logger.log(Level.WARNING, String.format("Leap file: %s is null or does not exist", LeappFile != null ? LeappFile.toString() : "<null>"));
350  return;
351  } else if (attrList == null || artifactType == null || dataSource == null) {
352  logger.log(Level.WARNING, String.format("attribute list, artifact type or dataSource not provided for %s", LeappFile.toString()));
353  return;
354  }
355 
356  List<BlackboardArtifact> bbartifacts = new ArrayList<>();
357 
358  // based on https://stackoverflow.com/questions/56921465/jackson-csv-schema-for-array
359  try (MappingIterator<List<String>> iterator = new CsvMapper()
360  .enable(CsvParser.Feature.WRAP_AS_ARRAY)
361  .readerFor(List.class)
362  .with(CsvSchema.emptySchema().withColumnSeparator('\t'))
363  .readValues(LeappFile)) {
364 
365  if (iterator.hasNext()) {
366  List<String> headerItems = iterator.next();
367  Map<String, Integer> columnIndexes = IntStream.range(0, headerItems.size())
368  .mapToObj(idx -> idx)
369  .collect(Collectors.toMap(
370  idx -> headerItems.get(idx) == null ? null : headerItems.get(idx).trim().toLowerCase(),
371  idx -> idx,
372  (val1, val2) -> val1));
373 
374  int lineNum = 2;
375  while (iterator.hasNext()) {
376  List<String> columnItems = iterator.next();
377  Collection<BlackboardAttribute> bbattributes = processReadLine(columnItems, columnIndexes, attrList, fileName, lineNum);
378 
379  if (!bbattributes.isEmpty()) {
380  switch (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(), "norelationship").toLowerCase()) {
381  case "message":
382  createMessageRelationship(bbattributes, dataSource, fileName);
383  break;
384  case "contact":
385  createContactRelationship(bbattributes, dataSource, fileName);
386  break;
387  case "calllog":
388  createCalllogRelationship(bbattributes, dataSource, fileName);
389  break;
390  case "route":
391  createRoute(bbattributes, dataSource, fileName);
392  break;
393  case "trackpoint":
394  geoAbstractFile = createTrackpoint(bbattributes, dataSource, fileName, trackpointSegmentName, pointList);
395  break;
396  default: // There is no relationship defined so just process the artifact normally
397  BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType, dataSource, bbattributes);
398  if (bbartifact != null) {
399  bbartifacts.add(bbartifact);
400  }
401  break;
402  }
403  }
404 
405  lineNum++;
406  }
407  }
408  }
409 
410  try {
411  if (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(), "norelationship").toLowerCase().equals("trackpoint")) {
412  (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, "", geoAbstractFile, context.getJobId())).addTrack(trackpointSegmentName, pointList, new ArrayList<>());
413  }
414  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
415  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
416  }
417 
418  if (!bbartifacts.isEmpty()) {
419  postArtifacts(bbartifacts);
420  }
421  }
422 
423  @NbBundle.Messages({
424  "LeappFileProcessor.cannot.create.waypoint.relationship=Cannot create TSK_WAYPOINT artifact."
425  })
426  private void createRoute(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
427 
428  Double startLatitude = Double.valueOf(0);
429  Double startLongitude = Double.valueOf(0);
430  Double endLatitude = Double.valueOf(0);
431  Double endLongitude = Double.valueOf(0);
432  Double zeroValue = Double.valueOf(0);
433  String destinationName = "";
434  String locationName = "";
435  Long dateTime = Long.valueOf(0);
436  Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
437  String sourceFile = null;
438  AbstractFile absFile;
439  String comment = "";
440 
441  try {
442  for (BlackboardAttribute bba : bbattributes) {
443  switch (bba.getAttributeType().getTypeName()) {
444  case "TSK_GEO_LATITUDE_START":
445  startLatitude = bba.getValueDouble();
446  break;
447  case "TSK_GEO_LONGITUDE_START":
448  startLongitude = bba.getValueDouble();
449  break;
450  case "TSK_GEO_LATITUDE_END":
451  startLatitude = bba.getValueDouble();
452  break;
453  case "TSK_GEO_LONGITUDE_END":
454  startLongitude = bba.getValueDouble();
455  break;
456  case "TSK_DATETIME":
457  dateTime = bba.getValueLong();
458  break;
459  case "TSK_NAME":
460  destinationName = bba.getValueString();
461  break;
462  case "TSK_LOCATION":
463  locationName = bba.getValueString();
464  break;
465  case "TSK_TEXT_FILE":
466  sourceFile = bba.getValueString();
467  break;
468  case "TSK_COMMENT":
469  comment = bba.getValueString();
470  break;
471  default:
472  otherAttributes.add(bba);
473  break;
474  }
475  }
476  absFile = findAbstractFile(dataSource, sourceFile);
477  if (absFile == null) {
478  absFile = (AbstractFile) dataSource;
479  }
480  GeoWaypoints waypointList = new GeoWaypoints();
481  waypointList.addPoint(new Waypoint(startLatitude, startLongitude, zeroValue, ""));
482  waypointList.addPoint(new Waypoint(endLatitude, endLongitude, zeroValue, locationName));
483  (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, comment, absFile, context.getJobId())).addRoute(destinationName, dateTime, waypointList, new ArrayList<>());
484 
485  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
486  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_waypoint_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
487  }
488 
489  }
490 
491  @NbBundle.Messages({
492  "LeappFileProcessor.cannot.create.trackpoint.relationship=Cannot create TSK_TRACK_POINT artifact."
493  })
494  private AbstractFile createTrackpoint(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList) throws IngestModuleException {
495 
496  Double latitude = Double.valueOf(0);
497  Double longitude = Double.valueOf(0);
498  Double altitude = Double.valueOf(0);
499  Double zeroValue = Double.valueOf(0);
500  String segmentName = null;
501  Long dateTime = Long.valueOf(0);
502  Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
503  String sourceFile = null;
504  String comment = null;
505  AbstractFile absFile = null;
506 
507  try {
508  for (BlackboardAttribute bba : bbattributes) {
509  switch (bba.getAttributeType().getTypeName()) {
510  case "TSK_GEO_LATITUDE":
511  latitude = bba.getValueDouble();
512  break;
513  case "TSK_GEO_LONGITUDE":
514  longitude = bba.getValueDouble();
515  break;
516  case "TSK_GEO_ALTITUDE":
517  altitude = bba.getValueDouble();
518  break;
519  case "TSK_DATETIME":
520  dateTime = bba.getValueLong();
521  break;
522  case "TSK_NAME":
523  segmentName = bba.getValueString();
524  break;
525  case "TSK_TEXT_FILE":
526  sourceFile = bba.getValueString();
527  break;
528  case "TSK_COMMENT":
529  comment = bba.getValueString();
530  otherAttributes.add(bba);
531  break;
532  default:
533  otherAttributes.add(bba);
534  break;
535  }
536  }
537  absFile = findAbstractFile(dataSource, sourceFile);
538  if (absFile == null) {
539  absFile = (AbstractFile) dataSource;
540  }
541  if ((trackpointSegmentName == null) || (trackpointSegmentName.equals(segmentName))) {
542  pointList.addPoint(new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime));
543  } else {
544  (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, comment, absFile, context.getJobId())).addTrack(segmentName, pointList, new ArrayList<>());
545  pointList.addPoint(new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime));
546 
547  }
548  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
549  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_trackpoint_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
550  }
551 
552  return absFile;
553 
554  }
555 
556  @NbBundle.Messages({
557  "LeappFileProcessor.cannot.create.message.relationship=Cannot create TSK_MESSAGE Relationship."
558  })
559  private void createMessageRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
560 
561  String messageType = null;
562  String alternateId = null;
563  CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN;
564  String senderId = null;
565  String receipentId = null;
566  String[] receipentIdList = null;
567  Long dateTime = Long.valueOf(0);
568  MessageReadStatus messageStatus = MessageReadStatus.UNKNOWN;
569  String subject = null;
570  String messageText = null;
571  String threadId = null;
572  List<BlackboardAttribute> otherAttributes = new ArrayList<>();
573  List<FileAttachment> fileAttachments = new ArrayList<>();
574  String sourceFile = null;
575  MessageAttachments messageAttachments;
576 
577  try {
578  for (BlackboardAttribute bba : bbattributes) {
579  switch (bba.getAttributeType().getTypeName()) {
580  case "TSK_DIRECTION":
581  if (bba.getValueString().toLowerCase().equals("outgoing")) {
582  communicationDirection = CommunicationDirection.OUTGOING;
583  } else if (bba.getValueString().toLowerCase().equals("incoming")) {
584  communicationDirection = CommunicationDirection.INCOMING;
585  }
586  break;
587  case "TSK_PHONE_NUMBER_FROM":
588  if (!bba.getValueString().isEmpty()) {
589  senderId = bba.getValueString();
590  }
591  break;
592  case "TSK_PHONE_NUMBER_TO":
593  if (!bba.getValueString().isEmpty()) {
594  receipentIdList = bba.getValueString().split(",", 0);
595  }
596  break;
597  case "TSK_DATETIME":
598  dateTime = bba.getValueLong();
599  break;
600  case "TSK_COMMENT":
601  messageType = bba.getValueString();
602  break;
603  case "TSK_ATTACHMENTS":
604  if (!bba.getValueString().isEmpty()) {
605  fileAttachments.add(new FileAttachment(Case.getCurrentCaseThrows().getSleuthkitCase(), dataSource, bba.getValueString()));
606  }
607  break;
608  case "TSK_TEXT_FILE":
609  sourceFile = bba.getValueString();
610  break;
611  case "TSK_READ_STATUS":
612  if (bba.getValueInt() == 1) {
613  messageStatus = MessageReadStatus.READ;
614  } else {
615  messageStatus = MessageReadStatus.UNREAD;
616  }
617  break;
618  case "TSK_TEXT":
619  messageText = bba.getValueString();
620  break;
621  case "TSK_SUBJECT":
622  subject = bba.getValueString();
623  break;
624  case "TSK_ID":
625  alternateId = bba.getValueString();
626  otherAttributes.add(bba);
627  break;
628  default:
629  otherAttributes.add(bba);
630  break;
631  }
632  }
633  AbstractFile absFile = findAbstractFile(dataSource, sourceFile);
634  if (absFile == null) {
635  absFile = (AbstractFile) dataSource;
636  }
637  CommunicationArtifactsHelper accountHelper;
638  Account.Type accountType = getAccountType(fileName);
639  if (alternateId == null) {
640  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
641  moduleName, absFile, accountType, context.getJobId());
642  } else {
643  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
644  moduleName, absFile, accountType, accountType, alternateId, context.getJobId());
645  }
646  BlackboardArtifact messageArtifact = accountHelper.addMessage(messageType, communicationDirection, senderId,
647  receipentId, dateTime, messageStatus, subject,
648  messageText, threadId, otherAttributes);
649  if (!fileAttachments.isEmpty()) {
650  messageAttachments = new MessageAttachments(fileAttachments, new ArrayList<>());
651  accountHelper.addAttachments(messageArtifact, messageAttachments);
652  }
653  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
654  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
655  }
656 
657  }
658 
659  @NbBundle.Messages({
660  "LeappFileProcessor.cannot.create.contact.relationship=Cannot create TSK_CONTACT Relationship."
661  })
662  private void createContactRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
663 
664  String alternateId = null;
665  String contactName = null;
666  String phoneNumber = null;
667  String homePhoneNumber = null;
668  String mobilePhoneNumber = null;
669  String emailAddr = null;
670  List<BlackboardAttribute> otherAttributes = new ArrayList<>();
671  String sourceFile = null;
672 
673  try {
674  for (BlackboardAttribute bba : bbattributes) {
675  switch (bba.getAttributeType().getTypeName()) {
676  case "TSK_PHONE_NUMBER":
677  if (!bba.getValueString().isEmpty()) {
678  phoneNumber = bba.getValueString();
679  }
680  break;
681  case "TSK_NAME":
682  if (!bba.getValueString().isEmpty()) {
683  contactName = bba.getValueString();
684  }
685  break;
686  case "TSK_TEXT_FILE":
687  sourceFile = bba.getValueString();
688  break;
689  case "TSK_PHONE_NUMBER_HOME":
690  homePhoneNumber = bba.getValueString();
691  break;
692  case "TSK_PHONE_NUMBER_MOBILE":
693  mobilePhoneNumber = bba.getValueString();
694  break;
695  case "TSK_EMAIL":
696  emailAddr = bba.getValueString();
697  break;
698  case "TSK_ID":
699  alternateId = bba.getValueString();
700  otherAttributes.add(bba);
701  break;
702  default:
703  otherAttributes.add(bba);
704  break;
705  }
706  }
707  AbstractFile absFile = findAbstractFile(dataSource, sourceFile);
708  if (absFile == null) {
709  absFile = (AbstractFile) dataSource;
710  }
711  Account.Type accountType = getAccountType(fileName);
712  if (accountType != null) {
713 
714  CommunicationArtifactsHelper accountHelper;
715  if (alternateId == null) {
716  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
717  moduleName, absFile, accountType, context.getJobId());
718  } else {
719  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
720  moduleName, absFile, accountType, accountType, alternateId, context.getJobId());
721  }
722  BlackboardArtifact messageArtifact = accountHelper.addContact(contactName, phoneNumber, homePhoneNumber, mobilePhoneNumber, emailAddr, otherAttributes);
723  }
724  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
725  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_contact_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
726  }
727  }
728 
729  @NbBundle.Messages({
730  "LeappFileProcessor.cannot.create.calllog.relationship=Cannot create TSK_CALLLOG Relationship."
731  })
732  private void createCalllogRelationship(Collection<BlackboardAttribute> bbattributes, Content dataSource, String fileName) throws IngestModuleException {
733 
734  String callerId = null;
735  String alternateId = null;
736  List<String> calleeId = Arrays.asList();
737  CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN;
738  Long startDateTime = Long.valueOf(0);
739  Long endDateTime = Long.valueOf(0);
740  CallMediaType mediaType = CallMediaType.UNKNOWN;
741  List<BlackboardAttribute> otherAttributes = new ArrayList<>();
742  String sourceFile = null;
743 
744  try {
745  for (BlackboardAttribute bba : bbattributes) {
746  switch (bba.getAttributeType().getTypeName()) {
747  case "TSK_TEXT_FILE":
748  sourceFile = bba.getValueString();
749  break;
750  case "TSK_DATETIME_START":
751  startDateTime = bba.getValueLong();
752  break;
753  case "TSK_DATETIME_END":
754  startDateTime = bba.getValueLong();
755  break;
756  case "TSK_DIRECTION":
757  if (bba.getValueString().toLowerCase().equals("outgoing")) {
758  communicationDirection = CommunicationDirection.OUTGOING;
759  } else if (bba.getValueString().toLowerCase().equals("incoming")) {
760  communicationDirection = CommunicationDirection.INCOMING;
761  }
762  break;
763  case "TSK_PHONE_NUMBER_FROM":
764  if (!bba.getValueString().isEmpty()) {
765  callerId = bba.getValueString();
766  }
767  break;
768  case "TSK_PHONE_NUMBER_TO":
769  if (!bba.getValueString().isEmpty()) {
770  String[] calleeTempList = bba.getValueString().split(",", 0);
771  calleeId = Arrays.asList(calleeTempList);
772  }
773  break;
774  case "TSK_ID":
775  alternateId = bba.getValueString();
776  otherAttributes.add(bba);
777  break;
778  default:
779  otherAttributes.add(bba);
780  break;
781  }
782  }
783 
784  if (calleeId.isEmpty() && communicationDirection == CommunicationDirection.OUTGOING && callerId != null) {
785  String[] calleeTempList = callerId.split(",", 0);
786  calleeId = Arrays.asList(calleeTempList);
787  callerId = null;
788  }
789  AbstractFile absFile = findAbstractFile(dataSource, sourceFile);
790  if (absFile == null) {
791  absFile = (AbstractFile) dataSource;
792  }
793  Account.Type accountType = getAccountType(fileName);
794  CommunicationArtifactsHelper accountHelper;
795  if (accountType != null) {
796  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
797  moduleName, absFile, accountType, context.getJobId());
798  } else {
799  accountHelper = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(),
800  moduleName, absFile, accountType, accountType, alternateId, context.getJobId());
801  }
802  accountHelper.addCalllog(communicationDirection, callerId, calleeId, startDateTime, endDateTime, mediaType, otherAttributes);
803  } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) {
804  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_calllog_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
805  }
806 
807  }
808 
809  private Account.Type getAccountType(String AccountTypeName) {
810  switch (AccountTypeName.toLowerCase()) {
811  case "zapya.tsv":
812  return Account.Type.ZAPYA;
813  case "sms messages.tsv":
814  return Account.Type.PHONE;
815  case "contacts.tsv":
816  return Account.Type.PHONE;
817  case "imo - accountid.tsv":
818  return Account.Type.IMO;
819  case "imo - messages.tsv":
820  return Account.Type.IMO;
821  case "textnow - contacts.tsv":
822  return Account.Type.TEXTNOW;
823  case "textnow - messages.tsv":
824  return Account.Type.TEXTNOW;
825  case "mms messages.tsv":
826  return Account.Type.PHONE;
827  case "viber - call logs.tsv":
828  return Account.Type.VIBER;
829  case "viber - contacts.tsv":
830  return Account.Type.VIBER;
831  case "viber - messages.tsv":
832  return Account.Type.VIBER;
833  case "xender file transfer - messages.tsv":
834  return Account.Type.XENDER;
835  case "xender file transfer - contacts.tsv":
836  return Account.Type.XENDER;
837  case "whatsapp - single call logs.tsv":
838  return Account.Type.WHATSAPP;
839  case "whatsapp - messages logs.tsv":
840  return Account.Type.WHATSAPP;
841  case "whatsapp - group call logs.tsv":
842  return Account.Type.WHATSAPP;
843  case "whatsapp - contacts.tsv":
844  return Account.Type.WHATSAPP;
845  case "tangomessages messages.tsv":
846  return Account.Type.TANGO;
847  case "shareit file transfer.tsv":
848  return Account.Type.SHAREIT;
849  case "line - calllogs.tsv":
850  return Account.Type.LINE;
851  case "line - contacts.tsv":
852  return Account.Type.LINE;
853  case "line - messages.tsv":
854  return Account.Type.LINE;
855  case "skype - call logs.tsv":
856  return Account.Type.SKYPE;
857  case "skype - contacts.tsv":
858  return Account.Type.SKYPE;
859  case "skype - messages logs.tsv":
860  return Account.Type.SKYPE;
861  case "facebook messenger - calls.tsv":
862  return Account.Type.FACEBOOK;
863  case "facebook messenger - contacts.tsv":
864  return Account.Type.FACEBOOK;
865  case "facebook messenger - chats.tsv":
866  return Account.Type.FACEBOOK;
867  case "call logs2.tsv":
868  return Account.Type.PHONE;
869  case "call logs.tsv":
870  return Account.Type.PHONE;
871  case "sms - imessage.tsv":
872  return Account.Type.PHONE;
873  default:
874  return Account.Type.PHONE;
875  }
876  }
877 
895  private Collection<BlackboardAttribute> processReadLine(List<String> lineValues, Map<String, Integer> columnIndexes,
896  List<TsvColumn> attrList, String fileName, int lineNum) throws IngestModuleException {
897 
898  if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues)
899  || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) {
900  return Collections.emptyList();
901  } else if (lineValues.size() != columnIndexes.size()) {
902  logger.log(Level.WARNING, String.format(
903  "Row at line number %d in file %s has %d columns when %d were expected based on the header row.",
904  lineNum, fileName, lineValues.size(), columnIndexes.size()));
905  return Collections.emptyList();
906  }
907 
908  List<BlackboardAttribute> attrsToRet = new ArrayList<>();
909  for (TsvColumn colAttr : attrList) {
910  if (colAttr.getAttributeType() == null) {
911  // this handles columns that are currently ignored.
912  continue;
913  }
914 
915  Integer columnIdx = columnIndexes.get(colAttr.getColumnName());
916  if (columnIdx == null) {
917  logger.log(Level.WARNING, String.format("No column mapping found for %s in file %s. Omitting column.", colAttr.getColumnName(), fileName));
918  continue;
919  }
920 
921  String value = (columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx);
922  if (value == null) {
923  logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName));
924  return Collections.emptyList();
925  }
926 
927  String formattedValue = formatValueBasedOnAttrType(colAttr, value);
928 
929  BlackboardAttribute attr = getAttribute(colAttr.getAttributeType(), formattedValue, fileName);
930  if (attr == null) {
931  logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName));
932  return Collections.emptyList();
933  }
934  attrsToRet.add(attr);
935  }
936 
937  if (tsvFileArtifactComments.containsKey(fileName)) {
938  attrsToRet.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, tsvFileArtifactComments.get(fileName)));
939  }
940 
941  return attrsToRet;
942  }
943 
953  private String formatValueBasedOnAttrType(TsvColumn colAttr, String value) {
954  if (colAttr.getAttributeType().getTypeName().equals("TSK_DOMAIN")) {
955  return NetworkUtils.extractDomain(value);
956  }
957 
958  return value;
959  }
960 
964  private static final DateFormat TIMESTAMP_FORMAT = new SimpleDateFormat("yyyy-MM-d HH:mm:ss", US);
965 
977  private BlackboardAttribute getAttribute(BlackboardAttribute.Type attrType, String value, String fileName) {
978  if (attrType == null || value == null) {
979  logger.log(Level.WARNING, String.format("Unable to parse attribute type %s for value '%s' in fileName %s",
980  attrType == null ? "<null>" : attrType.toString(),
981  value == null ? "<null>" : value,
982  fileName == null ? "<null>" : fileName));
983  return null;
984  }
985 
986  switch (attrType.getValueType()) {
987  case JSON:
988  case STRING:
989  return parseAttrValue(value, attrType, fileName, false, false,
990  (v) -> new BlackboardAttribute(attrType, moduleName, v));
991  case INTEGER:
992  return parseAttrValue(value.trim(), attrType, fileName, true, false,
993  (v) -> new BlackboardAttribute(attrType, moduleName, Double.valueOf(v).intValue()));
994  case LONG:
995  return parseAttrValue(value.trim(), attrType, fileName, true, false,
996  (v) -> new BlackboardAttribute(attrType, moduleName, Double.valueOf(v).longValue()));
997  case DOUBLE:
998  return parseAttrValue(value.trim(), attrType, fileName, true, false,
999  (v) -> new BlackboardAttribute(attrType, moduleName, Double.valueOf(v)));
1000  case BYTE:
1001  return parseAttrValue(value.trim(), attrType, fileName, true, false,
1002  (v) -> new BlackboardAttribute(attrType, moduleName, new byte[]{Byte.valueOf(v)}));
1003  case DATETIME:
1004  return parseAttrValue(value.trim(), attrType, fileName, true, true,
1005  (v) -> new BlackboardAttribute(attrType, moduleName, TIMESTAMP_FORMAT.parse(v).getTime() / 1000));
1006  default:
1007  // Log this and continue on with processing
1008  logger.log(Level.WARNING, String.format("Attribute Type %s for file %s not defined.", attrType, fileName)); //NON-NLS
1009  return null;
1010 
1011  }
1012  }
1013 
1017  private interface ParseExceptionFunction {
1018 
1029  BlackboardAttribute apply(String orig) throws ParseException, NumberFormatException;
1030  }
1031 
1047  private BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter) {
1048  // remove non-printable characters from tsv input
1049  // https://stackoverflow.com/a/6199346
1050  String sanitizedValue = value.replaceAll("\\p{C}", "");
1051 
1052  if (blankIsNull && StringUtils.isBlank(sanitizedValue)) {
1053  return null;
1054  }
1055 
1056  if (zeroIsNull && sanitizedValue.matches("^\\s*[0\\.]*\\s*$")) {
1057  return null;
1058  }
1059 
1060  try {
1061  return valueConverter.apply(sanitizedValue);
1062  } catch (NumberFormatException | ParseException ex) {
1063  logger.log(Level.WARNING, String.format("Unable to format '%s' as value type %s while converting to attributes from %s.", sanitizedValue, attrType.getValueType().getLabel(), fileName), ex);
1064  return null;
1065  }
1066  }
1067 
1071  @NbBundle.Messages({
1072  "LeappFileProcessor.cannot.load.artifact.xml=Cannot load xml artifact file.",
1073  "LeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.",
1074  "LeappFileProcessor_cannotParseXml=Cannot Parse XML file.",
1075  "LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact",
1076  "LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts."
1077  })
1078  private void loadConfigFile() throws IngestModuleException {
1079  Document xmlinput;
1080  try {
1081  String path = PlatformUtil.getUserConfigDirectory() + File.separator + xmlFile;
1082  File f = new File(path);
1083  DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
1084  DocumentBuilder db = dbf.newDocumentBuilder();
1085  xmlinput = db.parse(f);
1086 
1087  } catch (IOException e) {
1088  throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_load_artifact_xml() + e.getLocalizedMessage(), e); //NON-NLS
1089  } catch (ParserConfigurationException pce) {
1090  throw new IngestModuleException(Bundle.LeappFileProcessor_cannotBuildXmlParser() + pce.getLocalizedMessage(), pce); //NON-NLS
1091  } catch (SAXException sxe) {
1092  throw new IngestModuleException(Bundle.LeappFileProcessor_cannotParseXml() + sxe.getLocalizedMessage(), sxe); //NON-NLS
1093  }
1094 
1095  getFileNode(xmlinput);
1096  getArtifactNode(xmlinput);
1097  getAttributeNodes(xmlinput);
1098 
1099  }
1100 
1101  private void getFileNode(Document xmlinput) {
1102 
1103  NodeList nlist = xmlinput.getElementsByTagName("FileName"); //NON-NLS
1104 
1105  for (int i = 0; i < nlist.getLength(); i++) {
1106  NamedNodeMap nnm = nlist.item(i).getAttributes();
1107  tsvFiles.put(nnm.getNamedItem("filename").getNodeValue().toLowerCase(), nnm.getNamedItem("description").getNodeValue());
1108 
1109  }
1110 
1111  }
1112 
1113  private void getArtifactNode(Document xmlinput) {
1114 
1115  NodeList artifactNlist = xmlinput.getElementsByTagName("ArtifactName"); //NON-NLS
1116  for (int k = 0; k < artifactNlist.getLength(); k++) {
1117  NamedNodeMap nnm = artifactNlist.item(k).getAttributes();
1118  String artifactName = nnm.getNamedItem("artifactname").getNodeValue();
1119  String comment = nnm.getNamedItem("comment").getNodeValue();
1120  String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
1121 
1122  BlackboardArtifact.Type foundArtifactType = null;
1123  try {
1124  foundArtifactType = Case.getCurrentCase().getSleuthkitCase().getBlackboard().getArtifactType(artifactName);
1125  } catch (TskCoreException ex) {
1126  logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex);
1127  }
1128 
1129  if (foundArtifactType == null) {
1130  logger.log(Level.SEVERE, String.format("No known artifact mapping found for [artifact: %s, %s]",
1131  artifactName, getXmlFileIdentifier(parentName)));
1132  } else {
1133  tsvFileArtifacts.put(parentName, foundArtifactType);
1134  }
1135 
1136  if (!comment.toLowerCase().matches("null")) {
1137  tsvFileArtifactComments.put(parentName, comment);
1138  }
1139  }
1140 
1141  }
1142 
1143  private String getXmlFileIdentifier(String fileName) {
1144  return String.format("file: %s, filename: %s",
1145  this.xmlFile == null ? "<null>" : this.xmlFile,
1146  fileName == null ? "<null>" : fileName);
1147  }
1148 
1149  private String getXmlAttrIdentifier(String fileName, String attributeName) {
1150  return String.format("attribute: %s %s",
1151  attributeName == null ? "<null>" : attributeName,
1152  getXmlFileIdentifier(fileName));
1153  }
1154 
1155  private void getAttributeNodes(Document xmlinput) {
1156 
1157  NodeList attributeNlist = xmlinput.getElementsByTagName("AttributeName"); //NON-NLS
1158  for (int k = 0; k < attributeNlist.getLength(); k++) {
1159  NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
1160  String attributeName = nnm.getNamedItem("attributename").getNodeValue();
1161 
1162  if (!attributeName.toLowerCase().matches("null")) {
1163  String columnName = nnm.getNamedItem("columnName").getNodeValue();
1164  String required = nnm.getNamedItem("required").getNodeValue();
1165  String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
1166 
1167  BlackboardAttribute.Type foundAttrType = null;
1168  try {
1169  foundAttrType = Case.getCurrentCase().getSleuthkitCase().getBlackboard().getAttributeType(attributeName.toUpperCase());
1170  } catch (TskCoreException ex) {
1171  logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch attribute type for %s.", attributeName), ex);
1172  }
1173 
1174  if (foundAttrType == null) {
1175  logger.log(Level.SEVERE, String.format("No known attribute mapping found for [%s]", getXmlAttrIdentifier(parentName, attributeName)));
1176  }
1177 
1178  if (required != null && required.compareToIgnoreCase("yes") != 0 && required.compareToIgnoreCase("no") != 0) {
1179  logger.log(Level.SEVERE, String.format("Required value %s did not match 'yes' or 'no' for [%s]",
1180  required, getXmlAttrIdentifier(parentName, attributeName)));
1181  }
1182 
1183  if (columnName == null) {
1184  logger.log(Level.SEVERE, String.format("No column name provided for [%s]", getXmlAttrIdentifier(parentName, attributeName)));
1185  continue;
1186  } else if (columnName.trim().length() != columnName.length()) {
1187  logger.log(Level.SEVERE, String.format("Column name '%s' starts or ends with whitespace for [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName)));
1188  continue;
1189  } else if (columnName.matches("[^ \\S]")) {
1190  logger.log(Level.SEVERE, String.format("Column name '%s' contains invalid characters [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName)));
1191  continue;
1192  }
1193 
1194  TsvColumn thisCol = new TsvColumn(
1195  foundAttrType,
1196  columnName.trim().toLowerCase(),
1197  "yes".compareToIgnoreCase(required) == 0);
1198 
1199  if (tsvFileAttributes.containsKey(parentName)) {
1200  List<TsvColumn> attrList = tsvFileAttributes.get(parentName);
1201  attrList.add(thisCol);
1202  tsvFileAttributes.replace(parentName, attrList);
1203  } else {
1204  List<TsvColumn> attrList = new ArrayList<>();
1205  attrList.add(thisCol);
1206  tsvFileAttributes.put(parentName, attrList);
1207  }
1208  }
1209 
1210  }
1211  }
1212 
1225  private BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.Type artType, Content dataSource, Collection<BlackboardAttribute> bbattributes) {
1226  try {
1227  switch (artType.getCategory()) {
1228  case DATA_ARTIFACT:
1229  return dataSource.newDataArtifact(artType, bbattributes);
1230  case ANALYSIS_RESULT:
1231  return dataSource.newAnalysisResult(artType, Score.SCORE_UNKNOWN, null, null, null, bbattributes).getAnalysisResult();
1232  default:
1233  logger.log(Level.SEVERE, String.format("Unknown category type: %s", artType.getCategory().getDisplayName()));
1234  return null;
1235  }
1236  } catch (TskException ex) {
1237  logger.log(Level.WARNING, Bundle.LeappFileProcessor_error_creating_new_artifacts(), ex); //NON-NLS
1238  }
1239  return null;
1240  }
1241 
1248  void postArtifacts(Collection<BlackboardArtifact> artifacts) {
1249  if (artifacts == null || artifacts.isEmpty()) {
1250  return;
1251  }
1252 
1253  try {
1254  Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifacts(artifacts, moduleName, context.getJobId());
1255  } catch (Blackboard.BlackboardException ex) {
1256  logger.log(Level.SEVERE, Bundle.LeappFileProcessor_postartifacts_error(), ex); //NON-NLS
1257  }
1258  }
1259 
1265  private void configExtractor() throws IOException {
1267  xmlFile, true);
1268  }
1269 
1270  private static final Set<String> ALLOWED_EXTENSIONS = new HashSet<>(Arrays.asList("zip", "tar", "tgz"));
1271 
1279  static List<AbstractFile> findLeappFilesToProcess(Content dataSource) {
1280 
1281  List<AbstractFile> leappFiles = new ArrayList<>();
1282 
1283  FileManager fileManager = getCurrentCase().getServices().getFileManager();
1284 
1285  // findFiles use the SQL wildcard % in the file name
1286  try {
1287  leappFiles = fileManager.findFiles(dataSource, "%", "/"); //NON-NLS
1288  } catch (TskCoreException ex) {
1289  logger.log(Level.WARNING, "No files found to process"); //NON-NLS
1290  return leappFiles;
1291  }
1292 
1293  List<AbstractFile> leappFilesToProcess = new ArrayList<>();
1294  for (AbstractFile leappFile : leappFiles) {
1295  if (((leappFile.getLocalAbsPath() != null)
1296  && !leappFile.isVirtual())
1297  && leappFile.getNameExtension() != null
1298  && ALLOWED_EXTENSIONS.contains(leappFile.getNameExtension().toLowerCase())) {
1299  leappFilesToProcess.add(leappFile);
1300  }
1301  }
1302 
1303  return leappFilesToProcess;
1304  }
1305 
1310  private void createCustomArtifacts(Blackboard blkBoard) {
1311 
1312  for (Map.Entry<String, String> customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) {
1313  String artifactName = customArtifact.getKey();
1314  String artifactDescription = customArtifact.getValue();
1315 
1316  try {
1317  BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(artifactName, artifactDescription);
1318  } catch (Blackboard.BlackboardException ex) {
1319  logger.log(Level.WARNING, String.format("Failed to create custom artifact type %s.", artifactName), ex);
1320  }
1321 
1322  }
1323  }
1324 
1325  private AbstractFile findAbstractFile(Content dataSource, String fileNamePath) {
1326  if (fileNamePath == null) {
1327  return null;
1328  }
1329 
1330  List<AbstractFile> files;
1331 
1332  String fileName = FilenameUtils.getName(fileNamePath);
1333  String filePath = FilenameUtils.normalize(FilenameUtils.getPath(fileNamePath), true);
1334 
1336 
1337  try {
1338  files = fileManager.findFiles(dataSource, fileName); //NON-NLS
1339 
1340  } catch (TskCoreException ex) {
1341  logger.log(Level.WARNING, "Unable to find prefetch files.", ex); //NON-NLS
1342  return null; // No need to continue
1343  }
1344 
1345  for (AbstractFile pFile : files) {
1346 
1347  if (pFile.getParentPath().toLowerCase().endsWith(filePath.toLowerCase())) {
1348  return pFile;
1349  }
1350  }
1351 
1352  return null;
1353 
1354  }
1355 }
void createMessageRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.Type artType, Content dataSource, Collection< BlackboardAttribute > bbattributes)
LeappFileProcessor(String xmlFile, String moduleName, IngestJobContext context)
void createContactRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
static String extractDomain(String urlString)
List< AbstractFile > findFiles(String fileName)
void createCalllogRelationship(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
void processFile(File LeappFile, List< TsvColumn > attrList, String fileName, BlackboardArtifact.Type artifactType, Content dataSource)
final Map< String, BlackboardArtifact.Type > tsvFileArtifacts
AbstractFile findAbstractFile(Content dataSource, String fileNamePath)
void createRoute(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName)
Collection< BlackboardAttribute > processReadLine(List< String > lineValues, Map< String, Integer > columnIndexes, List< TsvColumn > attrList, String fileName, int lineNum)
BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter)
void processLeappFiles(List< String > LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress)
static< T > boolean extractResourceToUserConfigDir(final Class< T > resourceClass, final String resourceFileName, boolean overWrite)
BlackboardAttribute getAttribute(BlackboardAttribute.Type attrType, String value, String fileName)
ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
AbstractFile createTrackpoint(Collection< BlackboardAttribute > bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList)
String getXmlAttrIdentifier(String fileName, String attributeName)
ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress)
String formatValueBasedOnAttrType(TsvColumn colAttr, String value)

Copyright © 2012-2022 Basis Technology. Generated on: Thu Dec 8 2022
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.