JniDbHelper.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2020 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.datamodel;

import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;

class JniDbHelper {
    
    private static final Logger logger = Logger.getLogger(JniDbHelper.class.getName());
    
    private final SleuthkitCase caseDb;
    private CaseDbTransaction trans = null;
    private final AddDataSourceCallbacks addDataSourceCallbacks;
    
    private final Map<Long, Long> fsIdToRootDir = new HashMap<>();
    private final Map<Long, TskData.TSK_FS_TYPE_ENUM> fsIdToFsType = new HashMap<>();
    private final Map<ParentCacheKey, Long> parentDirCache = new HashMap<>();
    
    private static final long BATCH_FILE_THRESHOLD = 500;
    private final Queue<FileInfo> batchedFiles = new LinkedList<>();
    private final Queue<LayoutRangeInfo> batchedLayoutRanges = new LinkedList<>();
    private final List<Long> layoutFileIds = new ArrayList<>();
    
    JniDbHelper(SleuthkitCase caseDb, AddDataSourceCallbacks addDataSourceCallbacks) {
        this.caseDb = caseDb;
        this.addDataSourceCallbacks = addDataSourceCallbacks;
        trans = null;
    }
    
    private void beginTransaction() throws TskCoreException {
        trans = caseDb.beginTransaction();
    }
    
    private void commitTransaction() throws TskCoreException {
        trans.commit();
        trans = null;
    }
    
    private void revertTransaction() {
        try {
            if (trans != null) {
                trans.rollback();
                trans = null;
            }
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error rolling back transaction", ex);
        }
    }        
    
    void finish() {
        addBatchedFilesToDb();
        addBatchedLayoutRangesToDb();
        processLayoutFiles();
    }
    
    long addImageInfo(int type, long ssize, String timezone, 
            long size, String md5, String sha1, String sha256, String deviceId, 
            String collectionDetails, String[] paths) {    
        try {
            beginTransaction();
            long objId = caseDb.addImageJNI(TskData.TSK_IMG_TYPE_ENUM.valueOf(type), ssize, size,
                    timezone, md5, sha1, sha256, deviceId, collectionDetails, trans);
            for (int i = 0;i < paths.length;i++) {
                caseDb.addImageNameJNI(objId, paths[i], i, trans);
            }
            commitTransaction();
            return objId;
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding image to the database", ex);
            revertTransaction();
            return -1;
        }
    }
        
        void addAcquisitionDetails(long imgId, String details) {
        try {
            beginTransaction();
            caseDb.setAcquisitionDetails(imgId, details, trans);
            commitTransaction();
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding image details \"" + details + "\" to image with ID " + imgId, ex);
            revertTransaction();
        }               
        }
    
    long addVsInfo(long parentObjId, int vsType, long imgOffset, long blockSize) {
        try {
            beginTransaction();
            VolumeSystem vs = caseDb.addVolumeSystem(parentObjId, TskData.TSK_VS_TYPE_ENUM.valueOf(vsType), imgOffset, blockSize, trans);
            commitTransaction();
            return vs.getId();
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding volume system to the database - parent obj ID: " + parentObjId 
                    + ", image offset: " + imgOffset, ex);
            revertTransaction();
            return -1;
        }
    }
    
    long addVolume(long parentObjId, long addr, long start, long length, String desc,
            long flags) {
        try {
            beginTransaction();
            Volume vol = caseDb.addVolume(parentObjId, addr, start, length, desc, flags, trans);
            commitTransaction();
            return vol.getId();
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding volume to the database - parent object ID: " + parentObjId
                + ", addr: " + addr, ex);
            revertTransaction();
            return -1;
        }
    }

    long addPool(long parentObjId, int poolType) {
        try {
            beginTransaction();
            Pool pool = caseDb.addPool(parentObjId, TskData.TSK_POOL_TYPE_ENUM.valueOf(poolType), trans);
            commitTransaction();
            return pool.getId();
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding pool to the database - parent object ID: " + parentObjId, ex);
            revertTransaction();
            return -1;
        }
    }

    long addFileSystem(long parentObjId, long imgOffset, int fsType, long blockSize, long blockCount,
            long rootInum, long firstInum, long lastInum) {
        try {
            beginTransaction();
            FileSystem fs = caseDb.addFileSystem(parentObjId, imgOffset, TskData.TSK_FS_TYPE_ENUM.valueOf(fsType), blockSize, blockCount,
                    rootInum, firstInum, lastInum, null, trans);
            commitTransaction();
            fsIdToFsType.put(fs.getId(), TskData.TSK_FS_TYPE_ENUM.valueOf(fsType));
            return fs.getId();
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding file system to the database - parent object ID: " + parentObjId
                    + ", offset: " + imgOffset, ex);
            revertTransaction();
            return -1;
        }
    }

    long addFile(long parentObjId, 
        long fsObjId, long dataSourceObjId,
        int fsType,
        int attrType, int attrId, String name,
        long metaAddr, long metaSeq,
        int dirType, int metaType, int dirFlags, int metaFlags,
        long size,
        long crtime, long ctime, long atime, long mtime,
        int meta_mode, int gid, int uid,
        String escaped_path, String extension, 
        long seq, long parMetaAddr, long parSeq) {
        
        // Add the new file to the list
        batchedFiles.add(new FileInfo(parentObjId,
                fsObjId, dataSourceObjId,
                fsType,
                attrType, attrId, name,
                metaAddr, metaSeq,
                dirType, metaType, dirFlags, metaFlags,
                size,
                crtime, ctime, atime, mtime,
                meta_mode, gid, uid,
                escaped_path, extension,
                seq, parMetaAddr, parSeq));
        
        // Add the current files to the database if we've exceeded the threshold or if we
        // have the root folder.
        if ((fsObjId == parentObjId)
                || (batchedFiles.size() > BATCH_FILE_THRESHOLD)) {
            return addBatchedFilesToDb();
        }
        return 0;
    }
    
    private long addBatchedFilesToDb() {
        List<Long> newObjIds = new ArrayList<>();
        try {
            beginTransaction();
            FileInfo fileInfo;
            while ((fileInfo = batchedFiles.poll()) != null) {
                long computedParentObjId = fileInfo.parentObjId;
                try {
                    // If we weren't given the parent object ID, look it up
                    if (fileInfo.parentObjId == 0) {
                        computedParentObjId = getParentObjId(fileInfo);
                    }

                    long objId = caseDb.addFileJNI(computedParentObjId, 
                        fileInfo.fsObjId, fileInfo.dataSourceObjId,
                        fileInfo.fsType,
                        fileInfo.attrType, fileInfo.attrId, fileInfo.name,
                        fileInfo.metaAddr, fileInfo.metaSeq,
                        fileInfo.dirType, fileInfo.metaType, fileInfo.dirFlags, fileInfo.metaFlags,
                        fileInfo.size,
                        fileInfo.crtime, fileInfo.ctime, fileInfo.atime, fileInfo.mtime,
                        fileInfo.meta_mode, fileInfo.gid, fileInfo.uid,
                        null, TskData.FileKnown.UNKNOWN,
                        fileInfo.escaped_path, fileInfo.extension, 
                        false, trans);
                    if (fileInfo.fsObjId != fileInfo.parentObjId) {
                        // Add new file ID to the list to send to ingest unless it is the root folder
                        newObjIds.add(objId);
                    }

                    // If we're adding the root directory for the file system, cache it
                    if (fileInfo.parentObjId == fileInfo.fsObjId) {
                        fsIdToRootDir.put(fileInfo.fsObjId, objId);
                    }

                    // If the file is a directory, cache the object ID
                    if ((fileInfo.metaType == TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()
                            || (fileInfo.metaType == TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_VIRT_DIR.getValue()))
                            && (fileInfo.name != null)
                            && ! fileInfo.name.equals(".")
                            && ! fileInfo.name.equals("..")) {
                        String dirName = fileInfo.escaped_path + fileInfo.name;
                        ParentCacheKey key = new ParentCacheKey(fileInfo.fsObjId, fileInfo.metaAddr, fileInfo.seq, dirName);
                        parentDirCache.put(key, objId);
                    }
                } catch (TskCoreException ex) {
                    logger.log(Level.SEVERE, "Error adding file to the database - parent object ID: " + computedParentObjId
                            + ", file system object ID: " + fileInfo.fsObjId + ", name: " + fileInfo.name, ex);
                }
            }
            commitTransaction();
            try {
                addDataSourceCallbacks.onFilesAdded(newObjIds);
                        } catch (Exception ex) {
                // Exception firewall to prevent unexpected return to the native code
                logger.log(Level.SEVERE, "Unexpected error from files added callback", ex);
            }
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding batched files to database", ex);
            revertTransaction();
            return -1;
        }
        return 0;
    }
    
    private long getParentObjId(FileInfo fileInfo) throws TskCoreException {
        // Remove the final slash from the path unless we're in the root folder
        String parentPath = fileInfo.escaped_path;
        if(parentPath.endsWith("/") && ! parentPath.equals("/")) {
            parentPath =  parentPath.substring(0, parentPath.lastIndexOf('/'));
        }

        // Look up the parent
        ParentCacheKey key = new ParentCacheKey(fileInfo.fsObjId, fileInfo.parMetaAddr, fileInfo.parSeq, parentPath);
        if (parentDirCache.containsKey(key)) {
            return parentDirCache.get(key);
        } else {
            // There's no reason to do a database query since every folder added is being
            // stored in the cache.
            throw new TskCoreException("Parent not found in cache (fsObjId: " +fileInfo.fsObjId + ", parMetaAddr: " + fileInfo.parMetaAddr
                + ", parSeq: " + fileInfo.parSeq + ", parentPath: " + parentPath + ")");
        }
    }
    
    long addLayoutFile(long parentObjId, 
        long fsObjId, long dataSourceObjId,
        int fileType,
        String name, long size) {
        try {
            
            // The file system may be null for layout files
            Long fsObjIdForDb = fsObjId;
            if (fsObjId == 0) {
                fsObjIdForDb = null;
            }
            
            beginTransaction();
            long objId = caseDb.addFileJNI(parentObjId, 
                fsObjIdForDb, dataSourceObjId,
                fileType,
                null, null, name,
                null, null,
                TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue(), 
                TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue(), 
                TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC.getValue(), 
                TskData.TSK_FS_META_FLAG_ENUM.UNALLOC.getValue(),
                size,
                null, null, null, null,
                null, null, null,
                null, TskData.FileKnown.UNKNOWN,
                null, null, 
                true, trans);
            commitTransaction();

            // Store the layout file ID for later processing
            layoutFileIds.add(objId);

            return objId;
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding layout file to the database - parent object ID: " + parentObjId
                    + ", file system object ID: " + fsObjId + ", name: " + name, ex);
            revertTransaction();
            return -1;
        }
    }    
    
    long addLayoutFileRange(long objId, long byteStart, long byteLen, long seq) {
        batchedLayoutRanges.add(new LayoutRangeInfo(objId, byteStart, byteLen, seq));
        
        if (batchedLayoutRanges.size() > BATCH_FILE_THRESHOLD) {
            return addBatchedLayoutRangesToDb();
        }
        return 0;
    }
    
    private long addBatchedLayoutRangesToDb() {
        try {
            beginTransaction();
                LayoutRangeInfo range;
            while ((range = batchedLayoutRanges.poll()) != null) {
                try {
                    caseDb.addLayoutFileRangeJNI(range.objId, range.byteStart, range.byteLen, range.seq, trans);
                } catch (TskCoreException ex) {
                    logger.log(Level.SEVERE, "Error adding layout file range to the database - layout file ID: " + range.objId 
                        + ", byte start: " + range.byteStart + ", length: " + range.byteLen + ", seq: " + range.seq, ex);
                }
            }
            commitTransaction();
            return 0;
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error adding batched files to database", ex);
            revertTransaction();
            return -1;
        }
    }
        
    void processLayoutFiles() {
        addDataSourceCallbacks.onFilesAdded(layoutFileIds);
        layoutFileIds.clear();
    }
    
    long addUnallocFsBlockFilesParent(long fsObjId, String name) {
        try {
            if (! fsIdToRootDir.containsKey(fsObjId)) {
                logger.log(Level.SEVERE, "Error - root directory for file system ID {0} not found", fsObjId);
                return -1;
            }
            beginTransaction();
            VirtualDirectory dir = caseDb.addVirtualDirectory(fsIdToRootDir.get(fsObjId), name, trans);
            commitTransaction();
            addDataSourceCallbacks.onFilesAdded(Arrays.asList(dir.getId()));
            return dir.getId();
        } catch (TskCoreException ex) {
            logger.log(Level.SEVERE, "Error creating virtual directory " + name + " under file system ID " + fsObjId, ex);
            revertTransaction();
            return -1;
        }
    }
    
    private class ParentCacheKey {
        long fsObjId;
        long metaAddr;
        long seqNum;
        String path;
        
        ParentCacheKey(long fsObjId, long metaAddr, long seqNum, String path) {
            this.fsObjId = fsObjId;
            this.metaAddr = metaAddr;
            if (fsIdToFsType.containsKey(fsObjId) 
                    && (fsIdToFsType.get(fsObjId).equals(TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS)
                        || fsIdToFsType.get(fsObjId).equals(TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS_DETECT))) {
                this.seqNum = seqNum;
            } else {
                this.seqNum = 0;
            }
            this.path = path;
        }
        
        @Override
        public boolean equals(Object obj) {
            if (! (obj instanceof ParentCacheKey)) {
                return false;
            }
          
            ParentCacheKey otherKey = (ParentCacheKey) obj;
            if (this.fsObjId != otherKey.fsObjId 
                    || this.metaAddr != otherKey.metaAddr
                    || this.seqNum != otherKey.seqNum) {
                return false;
            }
            
            return StringUtils.equals(this.path, otherKey.path);
        }

        @Override
        public int hashCode() {
            int hash = 3;
            hash = 31 * hash + (int) (this.fsObjId ^ (this.fsObjId >>> 32));
            hash = 31 * hash + (int) (this.metaAddr ^ (this.metaAddr >>> 32));
            hash = 31 * hash + (int) (this.seqNum ^ (this.seqNum >>> 32));
            hash = 31 * hash + Objects.hashCode(this.path);
            return hash;
        }
    }
    
    private class LayoutRangeInfo {
        long objId;
        long byteStart;
        long byteLen;
        long seq;
        
        LayoutRangeInfo(long objId, long byteStart, long byteLen, long seq) {
            this.objId = objId;
            this.byteStart = byteStart;
            this.byteLen = byteLen;
            this.seq = seq;
        }
    }
    
    private class FileInfo {
        long parentObjId; 
        long fsObjId;
        long dataSourceObjId;
        int fsType;
        int attrType;
        int attrId;
        String name;
        long metaAddr; 
        long metaSeq;
        int dirType;
        int metaType;
        int dirFlags;
        int metaFlags;
        long size;
        long crtime;
        long ctime;
        long atime;
        long mtime;
        int meta_mode;
        int gid;
        int uid;
        String escaped_path;
        String extension;
        long seq;
        long parMetaAddr;
        long parSeq;
        
        FileInfo(long parentObjId, 
            long fsObjId, long dataSourceObjId,
            int fsType,
            int attrType, int attrId, String name,
            long metaAddr, long metaSeq,
            int dirType, int metaType, int dirFlags, int metaFlags,
            long size,
            long crtime, long ctime, long atime, long mtime,
            int meta_mode, int gid, int uid,
            String escaped_path, String extension, 
            long seq, long parMetaAddr, long parSeq) {
            
            this.parentObjId = parentObjId;
            this.fsObjId = fsObjId;
            this.dataSourceObjId = dataSourceObjId;
            this.fsType = fsType;
            this.attrType = attrType;
            this.attrId = attrId;
            this.name = name;
            this.metaAddr = metaAddr; 
            this.metaSeq = metaSeq;
            this.dirType = dirType;
            this.metaType = metaType;
            this.dirFlags = dirFlags;
            this.metaFlags = metaFlags;
            this.size = size;
            this.crtime = crtime;
            this.ctime = ctime;
            this.atime = atime;
            this.mtime = mtime;
            this.meta_mode = meta_mode;
            this.gid = gid;
            this.uid = uid;
            this.escaped_path = escaped_path;
            this.extension = extension;
            this.seq = seq;
            this.parMetaAddr = parMetaAddr;
            this.parSeq = parSeq;
        }
    }
}