LanguageSpecificContentIndexingHelper.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2019 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.keywordsearch;

import org.apache.commons.lang3.math.NumberUtils;
import org.apache.solr.common.SolrInputDocument;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.healthmonitor.HealthMonitor;
import org.sleuthkit.autopsy.healthmonitor.TimingMetric;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;

class LanguageSpecificContentIndexingHelper {

    private final LanguageDetector languageDetector = new LanguageDetector();

    Optional<Language> detectLanguageIfNeeded(Chunker.Chunk chunk) throws NoOpenCoreException {
        double indexSchemaVersion = NumberUtils.toDouble(KeywordSearch.getServer().getIndexInfo().getSchemaVersion());
        if (2.2 <= indexSchemaVersion) {
            return languageDetector.detect(chunk.toString());
        } else {
            return Optional.empty();
        }
    }

    void updateLanguageSpecificFields(Map<String, Object> fields, Chunker.Chunk chunk, Language language) {
        List<String> values = new ArrayList<>();
        values.add(chunk.toString());
        if (fields.containsKey(Server.Schema.FILE_NAME.toString())) {
            values.add(fields.get(Server.Schema.FILE_NAME.toString()).toString());
        }

        // index the chunk to a language specific field
        fields.put(Server.Schema.CONTENT_JA.toString(), values);
        fields.put(Server.Schema.LANGUAGE.toString(), language.getValue());
    }

    void indexMiniChunk(Chunker.Chunk chunk, String sourceName, Map<String, Object> fields, String baseChunkID, Language language)
        throws Ingester.IngesterException {
        //Make a SolrInputDocument out of the field map
        SolrInputDocument updateDoc = new SolrInputDocument();
        for (String key : fields.keySet()) {
            updateDoc.addField(key, fields.get(key));
        }

        try {
            updateDoc.setField(Server.Schema.ID.toString(), MiniChunkHelper.getChunkIdString(baseChunkID));

            // index the chunk to a language specific field
            updateDoc.addField(Server.Schema.CONTENT_JA.toString(), chunk.toString().substring(chunk.getBaseChunkLength()));
            updateDoc.addField(Server.Schema.LANGUAGE.toString(), language.getValue());

            TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk");

            KeywordSearch.getServer().addDocument(updateDoc);
            HealthMonitor.submitTimingMetric(metric);

        } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
            throw new Ingester.IngesterException(
                NbBundle.getMessage(Ingester.class, "Ingester.ingest.exception.err.msg", sourceName), ex);
        }
    }
}