CommonAttributeCountSearchResults.java

Go to the documentation of this file.

/*
 *
 * Autopsy Forensic Browser
 *
 * Copyright 2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.commonfilesearch;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;

final public class CommonAttributeCountSearchResults {

    private static final Logger LOGGER = Logger.getLogger(CommonAttributeCountSearchResults.class.getName());

    // maps instance count to list of attribute values. 
    private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
    private final Set<String> mimeTypesToInclude;
    private final int percentageThreshold;
    private final int resultTypeId;

    CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
        //wrap in a new object in case any client code has used an unmodifiable collection
        this.instanceCountToAttributeValues = new HashMap<>(metadata);
        this.percentageThreshold = percentageThreshold;
        this.resultTypeId = resultType.getId();
        this.mimeTypesToInclude = mimeTypesToFilterOn;
    }

    CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold) {
        //wrap in a new object in case any client code has used an unmodifiable collection
        this.instanceCountToAttributeValues = new HashMap<>(metadata);
        this.percentageThreshold = percentageThreshold;
        this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID;
        this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes
    }

    CommonAttributeValueList getAttributeValuesForInstanceCount(Integer instanceCount) {
        return this.instanceCountToAttributeValues.get(instanceCount);
    }

    public Map<Integer, CommonAttributeValueList> getMetadata() {
        return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
    }

    public void filterMetadata() throws EamDbException {
        filterMetadata(this.percentageThreshold);
    }

    private void filterMetadata(int maximumPercentageThreshold) throws EamDbException {
        CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
                .getDefaultCorrelationTypes()
                .stream()
                .filter(filterType -> filterType.getId() == this.resultTypeId)
                .findFirst().get();

        EamDb eamDb = EamDb.getInstance();

        Map<Integer, List<CommonAttributeValue>> itemsToRemove = new HashMap<>();
        //Call countUniqueDataSources once to reduce the number of DB queries needed to get
        //the frequencyPercentage
        Double uniqueCaseDataSourceTuples = eamDb.getCountUniqueDataSources().doubleValue();

        for (Entry<Integer, CommonAttributeValueList> listOfValues : Collections.unmodifiableMap(this.instanceCountToAttributeValues).entrySet()) {

            final Integer key = listOfValues.getKey();
            final CommonAttributeValueList values = listOfValues.getValue();

            for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata

                //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list 
                //because when performing intra case search this filtering will have been done during the query of the case database 
                boolean mimeTypeToRemove = false;  //allow code to be more efficient by not attempting to remove the same value multiple times
                if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
                    for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
                        AbstractFile abstractFile = commonAttr.getAbstractFile();
                        if (abstractFile != null) {
                            String mimeType = commonAttr.getAbstractFile().getMIMEType();
                            if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
                                if (itemsToRemove.containsKey(key)) {
                                    itemsToRemove.get(key).add(value);
                                } else {
                                    List<CommonAttributeValue> toRemove = new ArrayList<>();
                                    toRemove.add(value);
                                    itemsToRemove.put(key, toRemove);
                                }
                                //value will be removed as the mime type existed and was not in the set to be included
                                //because value is removed this value does not need to be checked further
                                mimeTypeToRemove = true;
                                break;
                            }
                        }
                        if (mimeTypeToRemove) {
                            break;
                        }
                    }
                }
                if (!mimeTypeToRemove && maximumPercentageThreshold != 0) {  //only do the frequency filtering when a max % was set
                    try {
                        Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue(
                                attributeType, value.getValue()).doubleValue();
                        Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100;
                        int frequencyPercentage = commonalityPercentage.intValue();
                        if (frequencyPercentage > maximumPercentageThreshold) {
                            if (itemsToRemove.containsKey(key)) {
                                itemsToRemove.get(key).add(value);
                            } else {
                                List<CommonAttributeValue> toRemove = new ArrayList<>();
                                toRemove.add(value);
                                itemsToRemove.put(key, toRemove);
                            }
                        }
                    } catch (CorrelationAttributeNormalizationException ex) {
                        LOGGER.log(Level.WARNING, "Unable to determine frequency percentage attribute - frequency filter may not be accurate for these results.", ex);
                    }
                }
            }
        }
        for (Entry<Integer, List<CommonAttributeValue>> valuesToRemove : itemsToRemove.entrySet()) {
            final Integer key = valuesToRemove.getKey();
            final List<CommonAttributeValue> values = valuesToRemove.getValue();
            for (CommonAttributeValue value : values) {
                final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key);
                if (instanceCountValue != null) {
                    instanceCountValue.removeMetaData(value);
                    if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata
                        this.instanceCountToAttributeValues.remove(key);
                    }
                }
            }
        }
    }

    public int size() {

        int count = 0;
        for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) {
            for (CommonAttributeValue md5 : data.getDelayedMetadataList()) {
                count += md5.getInstanceCount();
            }
        }
        return count;
    }
}