Autopsy  4.21.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestModuleTierBuilder.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2021-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.LinkedHashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.regex.Matcher;
26 import java.util.regex.Pattern;
27 import java.util.stream.Stream;
28 
33 class IngestModuleTierBuilder {
34 
35  private static final String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy";
36  private static final Pattern JYTHON_MODULE_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$");
37 
50  static List<IngestModuleTier> buildIngestModuleTiers(IngestJobSettings settings, IngestJobExecutor executor) throws InterruptedException {
51  /*
52  * Get the enabled ingest module templates from the ingest job settings.
53  */
54  List<IngestModuleTemplate> enabledTemplates = settings.getEnabledIngestModuleTemplates();
55 
62  Map<String, IngestModuleTemplate> javaDataSourceModuleTemplates = new LinkedHashMap<>();
63  Map<String, IngestModuleTemplate> jythonDataSourceModuleTemplates = new LinkedHashMap<>();
64  Map<String, IngestModuleTemplate> javaFileModuleTemplates = new LinkedHashMap<>();
65  Map<String, IngestModuleTemplate> jythonFileModuleTemplates = new LinkedHashMap<>();
66  Map<String, IngestModuleTemplate> javaArtifactModuleTemplates = new LinkedHashMap<>();
67  Map<String, IngestModuleTemplate> jythonArtifactModuleTemplates = new LinkedHashMap<>();
68  Map<String, IngestModuleTemplate> javaResultModuleTemplates = new LinkedHashMap<>();
69  Map<String, IngestModuleTemplate> jythonResultModuleTemplates = new LinkedHashMap<>();
70  for (IngestModuleTemplate template : enabledTemplates) {
71  if (template.isDataSourceIngestModuleTemplate()) {
72  addModuleTemplateToSortingMap(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template);
73  }
74  if (template.isFileIngestModuleTemplate()) {
75  addModuleTemplateToSortingMap(javaFileModuleTemplates, jythonFileModuleTemplates, template);
76  }
77  if (template.isDataArtifactIngestModuleTemplate()) {
78  addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template);
79  }
80  if (template.isAnalysisResultIngestModuleTemplate()) {
81  addModuleTemplateToSortingMap(javaResultModuleTemplates, jythonResultModuleTemplates, template);
82  }
83  }
84 
92  IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance();
93  List<IngestModuleTemplate> firstStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig());
94  List<IngestModuleTemplate> secondStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig());
95  List<IngestModuleTemplate> filePipelineTemplate = createIngestPipelineTemplate(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig());
96  List<IngestModuleTemplate> artifactPipelineTemplate = new ArrayList<>();
97  List<IngestModuleTemplate> resultsPipelineTemplate = new ArrayList<>();
98 
107  addToIngestPipelineTemplate(firstStageDataSourcePipelineTemplate, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates);
108  addToIngestPipelineTemplate(filePipelineTemplate, javaFileModuleTemplates, jythonFileModuleTemplates);
109  addToIngestPipelineTemplate(artifactPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates);
110  addToIngestPipelineTemplate(resultsPipelineTemplate, javaResultModuleTemplates, jythonResultModuleTemplates);
111 
116  List<IngestModuleTier> moduleTiers = new ArrayList<>();
117  IngestModuleTier firstTier = new IngestModuleTier();
118  int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
119  List<FileIngestPipeline> fileIngestPipelines = new ArrayList<>();
120  for (int i = 0; i < numberOfFileIngestThreads; ++i) {
121  fileIngestPipelines.add(new FileIngestPipeline(executor, filePipelineTemplate));
122  }
123  firstTier.setsFileIngestPipelines(fileIngestPipelines);
124  firstTier.setDataSourceIngestPipeline(new DataSourceIngestPipeline(executor, firstStageDataSourcePipelineTemplate));
125  firstTier.setDataArtifactIngestPipeline(new DataArtifactIngestPipeline(executor, artifactPipelineTemplate));
126  firstTier.setAnalysisResultIngestPipeline(new AnalysisResultIngestPipeline(executor, resultsPipelineTemplate));
127  moduleTiers.add(firstTier);
128  IngestModuleTier secondTier = new IngestModuleTier();
129  secondTier.setDataSourceIngestPipeline(new DataSourceIngestPipeline(executor, secondStageDataSourcePipelineTemplate));
130 
131  // RJCTODO: Remove test
132 // List<FileIngestPipeline> fileIngestPipelines2 = new ArrayList<>();
133 // for (int i = 0; i < numberOfFileIngestThreads; ++i) {
134 // fileIngestPipelines2.add(new FileIngestPipeline(executor, filePipelineTemplate));
135 // }
136 // secondTier.setsFileIngestPipelines(fileIngestPipelines2);
137 
138  moduleTiers.add(secondTier);
139  return moduleTiers;
140  }
141 
152  private static void addModuleTemplateToSortingMap(Map<String, IngestModuleTemplate> mapping, Map<String, IngestModuleTemplate> jythonMapping, IngestModuleTemplate template) {
153  String className = template.getModuleFactory().getClass().getCanonicalName();
154  String jythonName = getModuleNameFromJythonClassName(className);
155  if (jythonName != null) {
156  jythonMapping.put(jythonName, template);
157  } else {
158  mapping.put(className, template);
159  }
160  }
161 
173  private static String getModuleNameFromJythonClassName(String className) {
174  Matcher m = JYTHON_MODULE_REGEX.matcher(className);
175  if (m.find()) {
176  return String.format("%s.%s", m.group(1), m.group(2)); //NON-NLS
177  } else {
178  return null;
179  }
180  }
181 
197  private static List<IngestModuleTemplate> createIngestPipelineTemplate(Map<String, IngestModuleTemplate> javaIngestModuleTemplates, Map<String, IngestModuleTemplate> jythonIngestModuleTemplates, List<String> pipelineConfig) {
198  List<IngestModuleTemplate> pipelineTemplate = new ArrayList<>();
199  for (String moduleClassName : pipelineConfig) {
200  if (javaIngestModuleTemplates.containsKey(moduleClassName)) {
201  pipelineTemplate.add(javaIngestModuleTemplates.remove(moduleClassName));
202  } else if (jythonIngestModuleTemplates.containsKey(moduleClassName)) {
203  pipelineTemplate.add(jythonIngestModuleTemplates.remove(moduleClassName));
204  }
205  }
206  return pipelineTemplate;
207  }
208 
220  private static void addToIngestPipelineTemplate(final List<IngestModuleTemplate> sortedModules, final Map<String, IngestModuleTemplate> javaModules, final Map<String, IngestModuleTemplate> jythonModules) {
221  final List<IngestModuleTemplate> autopsyModules = new ArrayList<>();
222  final List<IngestModuleTemplate> thirdPartyModules = new ArrayList<>();
223  Stream.concat(javaModules.entrySet().stream(), jythonModules.entrySet().stream()).forEach((templateEntry) -> {
224  if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) {
225  autopsyModules.add(templateEntry.getValue());
226  } else {
227  thirdPartyModules.add(templateEntry.getValue());
228  }
229  });
230  sortedModules.addAll(autopsyModules);
231  sortedModules.addAll(thirdPartyModules);
232  }
233 
237  IngestModuleTierBuilder() {
238  }
239 
240 }

Copyright © 2012-2022 Basis Technology. Generated on: Tue Feb 6 2024
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.