IngestJob.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 * 
 * Copyright 2014-2015 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.sleuthkit.datamodel.Content;

public final class IngestJob {

    private static final AtomicLong nextId = new AtomicLong(0L);
    private final long id;
    private final Map<Long, DataSourceIngestJob> dataSourceJobs;
    private final AtomicInteger incompleteJobsCount;
    private boolean started;  // Guarded by this
    private volatile boolean cancelled;

    IngestJob(Collection<Content> dataSources, IngestJobSettings settings, boolean runInteractively) {
        this.id = IngestJob.nextId.getAndIncrement();
        this.dataSourceJobs = new ConcurrentHashMap<>();
        for (Content dataSource : dataSources) {
            DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, runInteractively);
            this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob);
        }
        incompleteJobsCount = new AtomicInteger(dataSourceJobs.size());
    }

    public long getId() {
        return this.id;
    }

    boolean hasIngestPipeline() {
        for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
            if (dataSourceJob.hasIngestPipeline()) {
                return true;
            }
        }
        return false;
    }

    synchronized List<IngestModuleError> start() {
        List<IngestModuleError> errors = new ArrayList<>();        
        if (started) {
            errors.add(new IngestModuleError("IngestJob", new IllegalStateException("Job already started")));
            return errors;
        }
        started = true;
                
        for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
            errors.addAll(dataSourceJob.start());
            if (!errors.isEmpty()) {
                break;
            }
        }

        if (!errors.isEmpty()) {
            for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
                dataSourceJob.cancel();
            }
        }

        return errors;
    }
    
    public ProgressSnapshot getSnapshot() {
        return new ProgressSnapshot(true);
    }

    public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) {
        return new ProgressSnapshot(getIngestTasksSnapshot);
    }
    
    List<DataSourceIngestJob.Snapshot> getDataSourceIngestJobSnapshots() {
        List<DataSourceIngestJob.Snapshot> snapshots = new ArrayList<>();
        for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
            snapshots.add(dataSourceJob.getSnapshot(true));
        }
        return snapshots;
    }

    public void cancel() {
        for (DataSourceIngestJob job : this.dataSourceJobs.values()) {
            job.cancel();
        }
        this.cancelled = true;
    }

    public boolean isCancelled() {
        return this.cancelled;
    }

    void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) {
        if (incompleteJobsCount.decrementAndGet() == 0) {
            IngestManager.getInstance().finishIngestJob(this);
        }
    }

    public final class ProgressSnapshot {

        private final List<DataSourceProcessingSnapshot> dataSourceProcessingSnapshots;
        private DataSourceIngestModuleHandle dataSourceModule;
        private boolean fileIngestRunning;
        private Date fileIngestStartTime;
        private final boolean jobCancelled;

        public final class DataSourceProcessingSnapshot {

            private final DataSourceIngestJob.Snapshot snapshot;

            private DataSourceProcessingSnapshot(DataSourceIngestJob.Snapshot snapshot) {
                this.snapshot = snapshot;
            }

            public String getDataSource() {
                return snapshot.getDataSource();
            }

            public boolean isCancelled() {
                return snapshot.isCancelled();
            }

            public List<String> getCancelledDataSourceIngestModules() {
                return snapshot.getCancelledDataSourceIngestModules();
            }

        }

        private ProgressSnapshot(boolean getIngestTasksSnapshot) {
            dataSourceModule = null;
            fileIngestRunning = false;
            fileIngestStartTime = null;
            dataSourceProcessingSnapshots = new ArrayList<>();
            for (DataSourceIngestJob dataSourceJob : dataSourceJobs.values()) {
                DataSourceIngestJob.Snapshot snapshot = dataSourceJob.getSnapshot(getIngestTasksSnapshot);
                dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
                if (null == dataSourceModule) {
                    DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule();
                    if (null != module) {
                        dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module);
                    }
                }
                if (snapshot.fileIngestIsRunning()) {
                    fileIngestRunning = true;
                }
                Date childFileIngestStartTime = snapshot.fileIngestStartTime();
                if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
                    fileIngestStartTime = childFileIngestStartTime;
                }
            }
            this.jobCancelled = cancelled;
        }

        public DataSourceIngestModuleHandle runningDataSourceIngestModule() {
            return this.dataSourceModule;
        }

        public boolean fileIngestIsRunning() {
            return this.fileIngestRunning;
        }

        public Date fileIngestStartTime() {
            return new Date(this.fileIngestStartTime.getTime());
        }

        public boolean isCancelled() {
            return this.jobCancelled;
        }

        public List<DataSourceProcessingSnapshot> getDataSourceSnapshots() {
            return Collections.unmodifiableList(this.dataSourceProcessingSnapshots);
        }

    }

    public static class DataSourceIngestModuleHandle {

        private final DataSourceIngestJob job;
        private final DataSourceIngestPipeline.PipelineModule module;
        private final boolean cancelled;

        private DataSourceIngestModuleHandle(DataSourceIngestJob job, DataSourceIngestPipeline.PipelineModule module) {
            this.job = job;
            this.module = module;
            this.cancelled = job.currentDataSourceIngestModuleIsCancelled();
        }

        public String displayName() {
            return this.module.getDisplayName();
        }

        public Date startTime() {
            return this.module.getProcessingStartTime();
        }

        public boolean isCancelled() {
            return this.cancelled;
        }

        public void cancel() {
            if (this.job.getCurrentDataSourceIngestModule() == this.module) {
                this.job.cancelCurrentDataSourceIngestModule();
            }
        }

    }

}