19 package org.sleuthkit.autopsy.keywordsearch;
21 import java.io.BufferedReader;
22 import java.util.HashMap;
24 import java.util.logging.Level;
25 import org.apache.solr.client.solrj.SolrServerException;
26 import org.apache.solr.common.SolrInputDocument;
27 import org.openide.util.NbBundle;
51 private static final Logger logger = Logger.getLogger(Ingester.class.getName());
52 private volatile boolean uncommitedIngests =
false;
53 private final Server solrServer = KeywordSearch.getServer();
54 private static final SolrFieldsVisitor SOLR_FIELDS_VISITOR =
new SolrFieldsVisitor();
55 private static Ingester instance;
56 private static final int SINGLE_READ_CHARS = 512;
61 public static synchronized Ingester getDefault() {
62 if (instance == null) {
63 instance =
new Ingester();
70 @SuppressWarnings(
"FinalizeDeclaration")
71 protected
void finalize() throws Throwable {
75 if (uncommitedIngests) {
76 logger.warning(
"Ingester was used to add files that it never committed.");
90 void indexMetaDataOnly(AbstractFile file)
throws IngesterException {
91 indexChunk(
"", file.getName(), getContentFields(file));
104 void indexMetaDataOnly(BlackboardArtifact artifact)
throws IngesterException {
105 indexChunk(
"",
new ArtifactTextExtractor().getName(artifact), getContentFields(artifact));
116 private Map<String, String> getContentFields(SleuthkitVisitableItem item) {
117 return item.accept(SOLR_FIELDS_VISITOR);
140 < T extends SleuthkitVisitableItem>
boolean indexText(TextExtractor< T> extractor, T source, IngestJobContext context)
throws Ingester.IngesterException {
141 final long sourceID = extractor.getID(source);
142 final String sourceName = extractor.getName(source);
146 if (extractor.isDisabled()) {
152 Map<String, String> fields = getContentFields(source);
154 try (BufferedReader reader =
new BufferedReader(extractor.getReader(source));) {
155 Chunker chunker =
new Chunker(reader);
156 for (Chunk chunk : chunker) {
157 String chunkId = Server.getChunkIdString(sourceID, numChunks + 1);
158 fields.put(Server.Schema.ID.toString(), chunkId);
159 fields.put(Server.Schema.CHUNK_SIZE.toString(), String.valueOf(chunk.getBaseChunkLength()));
162 indexChunk(chunk.toString(), sourceName, fields);
164 }
catch (Ingester.IngesterException ingEx) {
165 extractor.logWarning(
"Ingester had a problem with extracted string from file '"
166 + sourceName +
"' (id: " + sourceID +
").", ingEx);
171 if (chunker.hasException()) {
172 extractor.logWarning(
"Error chunking content from " + sourceID +
": " + sourceName, chunker.getException());
175 }
catch (Exception ex) {
176 extractor.logWarning(
"Unexpected error, can't read content stream from " + sourceID +
": " + sourceName, ex);
180 fields.put(Server.Schema.NUM_CHUNKS.toString(), Integer.toString(numChunks));
182 fields.put(Server.Schema.ID.toString(), Long.toString(sourceID));
184 fields.remove(Server.Schema.CHUNK_SIZE.toString());
185 indexChunk(null, sourceName, fields);
204 private void indexChunk(String chunk, String sourceName, Map<String, String> fields)
throws IngesterException {
205 if (fields.get(Server.Schema.IMAGE_ID.toString()) == null) {
210 String msg = NbBundle.getMessage(Ingester.class,
211 "Ingester.ingest.exception.unknownImgId.msg", sourceName);
212 logger.log(Level.SEVERE, msg);
213 throw new IngesterException(msg);
217 SolrInputDocument updateDoc =
new SolrInputDocument();
218 for (String key : fields.keySet()) {
219 updateDoc.addField(key, fields.get(key));
223 updateDoc.addField(Server.Schema.CONTENT.toString(), chunk);
227 solrServer.addDocument(updateDoc);
228 uncommitedIngests =
true;
230 }
catch (KeywordSearchModuleException | NoOpenCoreException ex) {
232 throw new IngesterException(
233 NbBundle.getMessage(Ingester.class,
"Ingester.ingest.exception.err.msg", sourceName), ex);
244 uncommitedIngests =
false;
245 }
catch (NoOpenCoreException | SolrServerException ex) {
246 logger.log(Level.WARNING,
"Error commiting index", ex);
254 static private class SolrFieldsVisitor extends SleuthkitItemVisitor.Default<Map<String, String>> {
257 protected Map<String, String>
defaultVisit(SleuthkitVisitableItem svi) {
258 return new HashMap<>();
262 public Map<String, String>
visit(File f) {
267 public Map<String, String>
visit(DerivedFile df) {
272 public Map<String, String>
visit(Directory d) {
277 public Map<String, String>
visit(LocalDirectory ld){
282 public Map<String, String>
visit(LayoutFile lf) {
288 public Map<String, String>
visit(LocalFile lf) {
293 public Map<String, String>
visit(SlackFile f) {
324 Map<String, String> params =
new HashMap<>();
325 params.put(
Server.
Schema.ID.toString(), Long.toString(file.getId()));
327 params.put(
Server.
Schema.IMAGE_ID.toString(), Long.toString(file.getDataSource().getId()));
328 }
catch (TskCoreException ex) {
329 logger.log(Level.SEVERE,
"Could not get data source id to properly index the file " + file.getId(), ex);
330 params.put(
Server.
Schema.IMAGE_ID.toString(), Long.toString(-1));
332 params.put(
Server.
Schema.FILE_NAME.toString(), file.getName());
344 public Map<String, String>
visit(BlackboardArtifact artifact) {
345 Map<String, String> params =
new HashMap<>();
346 params.put(
Server.
Schema.ID.toString(), Long.toString(artifact.getArtifactID()));
348 params.put(
Server.
Schema.IMAGE_ID.toString(), Long.toString(ArtifactTextExtractor.getDataSource(artifact).getId()));
349 }
catch (TskCoreException ex) {
350 logger.log(Level.SEVERE,
"Could not get data source id to properly index the artifact " + artifact.getArtifactID(), ex);
351 params.put(
Server.
Schema.IMAGE_ID.toString(), Long.toString(-1));
361 static class IngesterException
extends Exception {
363 private static final long serialVersionUID = 1L;
365 IngesterException(String message, Throwable ex) {
369 IngesterException(String message) {
Map< String, String > visit(LayoutFile lf)
Map< String, String > visit(File f)
Map< String, String > visit(LocalDirectory ld)
Map< String, String > getCommonAndMACTimeFields(AbstractFile file)
Map< String, String > visit(SlackFile f)
Map< String, String > visit(Directory d)
static String getStringTimeISO8601(long epochSeconds, TimeZone tzone)
Map< String, String > getCommonFields(AbstractFile file)
Map< String, String > visit(DerivedFile df)
Map< String, String > visit(BlackboardArtifact artifact)
Map< String, String > visit(LocalFile lf)
Map< String, String > defaultVisit(SleuthkitVisitableItem svi)