CorrelationAttributeNormalizer.java

Go to the documentation of this file.

/*
 *
 * Autopsy Forensic Browser
 *
 * Copyright 2019 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.centralrepository.datamodel;

import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.validator.routines.DomainValidator;
import org.apache.commons.validator.routines.EmailValidator;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;

final public class CorrelationAttributeNormalizer {

    //common seperators that may be removed for normalizing
    private static final String SEPERATORS_REGEX = "[\\s-:]";

    public static String normalize(CorrelationAttributeInstance.Type attributeType, String data) throws CorrelationAttributeNormalizationException {

        if (attributeType == null) {
            throw new CorrelationAttributeNormalizationException("Attribute type was null.");
        }
        if (data == null) {
            throw new CorrelationAttributeNormalizationException("Correlation value was null.");
        }

        String trimmedData = data.trim();

        switch (attributeType.getId()) {
            case CorrelationAttributeInstance.FILES_TYPE_ID:
                return normalizeMd5(trimmedData);
            case CorrelationAttributeInstance.DOMAIN_TYPE_ID:
                return normalizeDomain(trimmedData);
            case CorrelationAttributeInstance.EMAIL_TYPE_ID:
                return normalizeEmail(trimmedData);
            case CorrelationAttributeInstance.PHONE_TYPE_ID:
                return normalizePhone(trimmedData);
            case CorrelationAttributeInstance.USBID_TYPE_ID:
                return normalizeUsbId(trimmedData);
            case CorrelationAttributeInstance.SSID_TYPE_ID:
                return verifySsid(trimmedData);
            case CorrelationAttributeInstance.MAC_TYPE_ID:
                return normalizeMac(trimmedData);
            case CorrelationAttributeInstance.IMEI_TYPE_ID:
                return normalizeImei(trimmedData);
            case CorrelationAttributeInstance.IMSI_TYPE_ID:
                return normalizeImsi(trimmedData);
            case CorrelationAttributeInstance.ICCID_TYPE_ID:
                return normalizeIccid(trimmedData);

            default:
                try {
                    // If the atttribute is not one of the above 
                    // but is one of the other default correlation types, then let the data go as is
                    List<CorrelationAttributeInstance.Type> defaultCorrelationTypes = CorrelationAttributeInstance.getDefaultCorrelationTypes();
                    for (CorrelationAttributeInstance.Type defaultCorrelationType : defaultCorrelationTypes) {
                        if (defaultCorrelationType.getId() == attributeType.getId()) {
                            return trimmedData;
                        }
                    }
                    final String errorMessage = String.format(
                            "Validator function not found for attribute type: %s",
                            attributeType.getDisplayName());
                    throw new CorrelationAttributeNormalizationException(errorMessage);
                } catch (CentralRepoException ex) {
                    throw new CorrelationAttributeNormalizationException("Failed to get default correlation types.", ex);
                }
        }
    }

    public static String normalize(int attributeTypeId, String data) throws CorrelationAttributeNormalizationException {
        try {
            List<CorrelationAttributeInstance.Type> defaultTypes = CorrelationAttributeInstance.getDefaultCorrelationTypes();
            Optional<CorrelationAttributeInstance.Type> typeOption = defaultTypes.stream().filter(attributeType -> attributeType.getId() == attributeTypeId).findAny();

            if (typeOption.isPresent()) {
                CorrelationAttributeInstance.Type type = typeOption.get();
                return CorrelationAttributeNormalizer.normalize(type, data);
            } else {
                throw new CorrelationAttributeNormalizationException(String.format("Given attributeTypeId did not correspond to any known Attribute: %s", attributeTypeId));
            }
        } catch (CentralRepoException ex) {
            throw new CorrelationAttributeNormalizationException(ex);
        }
    }

    private static String normalizeMd5(String data) throws CorrelationAttributeNormalizationException {
        final String validMd5Regex = "^[a-f0-9]{32}$";
        final String dataLowered = data.toLowerCase();
        if (dataLowered.matches(validMd5Regex)) {
            return dataLowered;
        } else {
            throw new CorrelationAttributeNormalizationException(String.format("Data purporting to be an MD5 was found not to comform to expected format: %s", data));
        }
    }

    private static String normalizeDomain(String data) throws CorrelationAttributeNormalizationException {
        DomainValidator validator = DomainValidator.getInstance(true);
        if (validator.isValid(data)) {
            return NetworkUtils.extractDomain(data.toLowerCase());
        } else {
            final String validIpAddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$";
            if (data.matches(validIpAddressRegex)) {
                return NetworkUtils.extractDomain(data);
            } else {
                throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid domain: %s", data));
            }
        }
    }

    static String normalizeEmail(String emailAddress) throws CorrelationAttributeNormalizationException {
        if (isValidEmailAddress(emailAddress)) {
            return emailAddress.toLowerCase().trim();
        } else {
            throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid email address: %s", emailAddress));
        }
    }

    static String normalizePhone(String phoneNumber) throws CorrelationAttributeNormalizationException {
        if (isValidPhoneNumber(phoneNumber)) {
            String normalizedNumber = phoneNumber.replaceAll("\\s+", ""); // remove spaces.     
            normalizedNumber = normalizedNumber.replaceAll("[\\-()]", ""); // remove parens & dashes.

            // ensure a min length
            if (normalizedNumber.length() < MIN_PHONENUMBER_LEN) {
                throw new CorrelationAttributeNormalizationException(String.format("Phone number string %s is too short ", phoneNumber));
            }
            return normalizedNumber;

        } else {
            throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid phone number: %s", phoneNumber));
        }
    }

    private static String normalizeUsbId(String data) throws CorrelationAttributeNormalizationException {
        //TODO replace with correct usb id validation at a later date
        return data;
    }

    private static String verifySsid(String data) throws CorrelationAttributeNormalizationException {
        if (data.length() <= 32) {
            return data;
        } else {
            throw new CorrelationAttributeNormalizationException("Name provided was longer than the maximum valid SSID (32 characters). Name: " + data);
        }
    }

    private static String normalizeIccid(String data) throws CorrelationAttributeNormalizationException {
        final String validIccidRegex = "^89[f0-9]{17,22}$";
        final String iccidWithoutSeperators = data.toLowerCase().replaceAll(SEPERATORS_REGEX, "");
        if (iccidWithoutSeperators.matches(validIccidRegex)) {
            return iccidWithoutSeperators;
        } else {
            throw new CorrelationAttributeNormalizationException("Data provided was not a valid ICCID. : " + data);
        }
    }

    private static String normalizeImsi(String data) throws CorrelationAttributeNormalizationException {
        final String validImsiRegex = "^[0-9]{14,15}$";
        final String imsiWithoutSeperators = data.replaceAll(SEPERATORS_REGEX, "");
        if (imsiWithoutSeperators.matches(validImsiRegex)) {
            return imsiWithoutSeperators;
        } else {
            throw new CorrelationAttributeNormalizationException("Data provided was not a valid IMSI. : " + data);
        }
    }

    private static String normalizeMac(String data) throws CorrelationAttributeNormalizationException {
        final String validMacRegex = "^([a-f0-9]{12}|[a-f0-9]{16})$";
        final String macWithoutSeperators = data.toLowerCase().replaceAll(SEPERATORS_REGEX, "");
        if (macWithoutSeperators.matches(validMacRegex)) {
            return macWithoutSeperators;
        } else {
            throw new CorrelationAttributeNormalizationException("Data provided was not a valid MAC address. : " + data);
        }
    }

    private static String normalizeImei(String data) throws CorrelationAttributeNormalizationException {
        final String validImeiRegex = "^[0-9]{14,16}$";
        final String imeiWithoutSeperators = data.replaceAll(SEPERATORS_REGEX, "");
        if (imeiWithoutSeperators.matches(validImeiRegex)) {
            return imeiWithoutSeperators;
        } else {
            throw new CorrelationAttributeNormalizationException("Data provided was not a valid IMEI. : " + data);
        }
    }

    // These symbols are allowed in written form of phone numbers.
    // A '+' is allowed only as a leading digit and hence not inlcuded here.
    // While a dialed sequence may have additonal special characters, such as #, * or ',',
    // CR attributes represent accounts and hence those chatracter are not allowed.
    private static final Set<String> PHONENUMBER_CHARS = new HashSet<>(Arrays.asList(
            "-", "(", ")"
    ));

    private static final int MIN_PHONENUMBER_LEN = 5;

    static boolean isValidPhoneNumber(String phoneNumber) {

        // A phone number may have a leading '+', special telephony chars, or digits.
        // Anything else implies an invalid phone number.
        for (int i = 0; i < phoneNumber.length(); i++) {
            if ( !((i == 0 && phoneNumber.charAt(i) == '+')
                    || Character.isSpaceChar(phoneNumber.charAt(i))
                    || Character.isDigit(phoneNumber.charAt(i))
                    || PHONENUMBER_CHARS.contains(String.valueOf(phoneNumber.charAt(i))))) {
                return false;
            } 
        }

        // ensure a min length
        return phoneNumber.length() >= MIN_PHONENUMBER_LEN;
    }

    static boolean isValidEmailAddress(String emailAddress) {
        if (!StringUtils.isEmpty(emailAddress)) {
            EmailValidator validator = EmailValidator.getInstance(true, true);
            return validator.isValid(emailAddress);
        }

        return false;
    }

    private CorrelationAttributeNormalizer() {
        //Empty constructor
    }
}