Autopsy  4.21.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
Server.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.awt.event.ActionEvent;
23 import java.beans.PropertyChangeListener;
24 import java.io.BufferedReader;
25 import java.io.BufferedWriter;
26 import java.io.File;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStream;
32 import java.io.OutputStreamWriter;
33 import java.net.ConnectException;
34 import java.net.DatagramSocket;
35 import java.net.ServerSocket;
36 import java.net.SocketException;
37 import java.nio.charset.Charset;
38 import java.nio.file.Files;
39 import java.nio.file.OpenOption;
40 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
41 import java.nio.file.Path;
42 import java.nio.file.Paths;
43 import java.util.ArrayList;
44 import java.util.Arrays;
45 import java.util.Collections;
46 import java.util.Iterator;
47 import java.util.List;
48 import java.util.Random;
49 import java.util.concurrent.ScheduledThreadPoolExecutor;
50 import java.util.concurrent.TimeUnit;
51 import java.util.concurrent.locks.ReentrantReadWriteLock;
52 import java.util.logging.Level;
53 import javax.swing.AbstractAction;
54 import org.apache.commons.io.FileUtils;
55 import java.util.concurrent.TimeoutException;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.stream.Collectors;
59 import static java.util.stream.Collectors.toList;
60 import javax.swing.JOptionPane;
61 import org.apache.solr.client.solrj.SolrQuery;
62 import org.apache.solr.client.solrj.SolrRequest;
63 import org.apache.solr.client.solrj.SolrServerException;
64 import org.apache.solr.client.solrj.SolrClient;
65 import org.apache.solr.client.solrj.impl.HttpSolrClient;
66 import org.apache.solr.client.solrj.impl.CloudSolrClient;
67 import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
68 import org.apache.solr.client.solrj.impl.XMLResponseParser;
69 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
70 import org.apache.solr.client.solrj.response.CollectionAdminResponse;
71 import org.apache.solr.client.solrj.request.CoreAdminRequest;
72 import org.apache.solr.client.solrj.response.CoreAdminResponse;
73 import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException;
74 import org.apache.solr.client.solrj.request.QueryRequest;
75 import org.apache.solr.client.solrj.response.QueryResponse;
76 import org.apache.solr.client.solrj.response.TermsResponse;
77 import org.apache.solr.client.solrj.response.TermsResponse.Term;
78 import org.apache.solr.common.SolrDocument;
79 import org.apache.solr.common.SolrDocumentList;
80 import org.apache.solr.common.SolrException;
81 import org.apache.solr.common.SolrInputDocument;
82 import org.apache.solr.common.util.NamedList;
83 import org.openide.modules.InstalledFileLocator;
84 import org.openide.modules.Places;
85 import org.openide.util.NbBundle;
86 import org.openide.util.NbBundle.Messages;
87 import org.openide.windows.WindowManager;
104 import org.sleuthkit.datamodel.Content;
105 
110 public class Server {
111 
115  public static enum Schema {
116 
117  ID {
118  @Override
119  public String toString() {
120  return "id"; //NON-NLS
121  }
122  },
123  IMAGE_ID {
124  @Override
125  public String toString() {
126  return "image_id"; //NON-NLS
127  }
128  },
129  // This is not stored or indexed. it is copied to text by the schema
130  CONTENT {
131  @Override
132  public String toString() {
133  return "content"; //NON-NLS
134  }
135  },
136  // String representation for regular expression searching
137  CONTENT_STR {
138  @Override
139  public String toString() {
140  return "content_str"; //NON-NLS
141  }
142  },
143  // default search field. Populated by schema
144  TEXT {
145  @Override
146  public String toString() {
147  return "text"; //NON-NLS
148  }
149  },
150  // no longer populated. Was used for regular expression searching.
151  // Should not be used.
152  CONTENT_WS {
153  @Override
154  public String toString() {
155  return "content_ws"; //NON-NLS
156  }
157  },
158  CONTENT_JA {
159  @Override
160  public String toString() {
161  return "content_ja"; //NON-NLS
162  }
163  },
164  LANGUAGE {
165  @Override
166  public String toString() {
167  return "language"; //NON-NLS
168  }
169  },
170  FILE_NAME {
171  @Override
172  public String toString() {
173  return "file_name"; //NON-NLS
174  }
175  },
176  // note that we no longer store or index this field
177  CTIME {
178  @Override
179  public String toString() {
180  return "ctime"; //NON-NLS
181  }
182  },
183  // note that we no longer store or index this field
184  ATIME {
185  @Override
186  public String toString() {
187  return "atime"; //NON-NLS
188  }
189  },
190  // note that we no longer store or index this field
191  MTIME {
192  @Override
193  public String toString() {
194  return "mtime"; //NON-NLS
195  }
196  },
197  // note that we no longer store or index this field
198  CRTIME {
199  @Override
200  public String toString() {
201  return "crtime"; //NON-NLS
202  }
203  },
204  NUM_CHUNKS {
205  @Override
206  public String toString() {
207  return "num_chunks"; //NON-NLS
208  }
209  },
210  CHUNK_SIZE {
211  @Override
212  public String toString() {
213  return "chunk_size"; //NON-NLS
214  }
215  },
221  TERMFREQ {
222  @Override
223  public String toString() {
224  return "termfreq"; //NON-NLS
225  }
226  }
227  };
228 
229  public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
230  //max content size we can send to Solr
231  public static final long MAX_CONTENT_SIZE = 1L * 31 * 1024 * 1024;
232  private static final Logger logger = Logger.getLogger(Server.class.getName());
233  public static final String CORE_EVT = "CORE_EVT"; //NON-NLS
234  @Deprecated
235  public static final char ID_CHUNK_SEP = '_';
236  public static final String CHUNK_ID_SEPARATOR = "_";
237  private String javaPath = "java";
238  public static final Charset DEFAULT_INDEXED_TEXT_CHARSET = Charset.forName("UTF-8");
239  private Process curSolrProcess = null;
240  static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
241  static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
242  static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
243  private static final String KEY = "jjk#09s"; //NON-NLS
244  static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
245  static final int DEFAULT_SOLR_SERVER_PORT = 23232;
246  static final int DEFAULT_SOLR_STOP_PORT = 34343;
247  private int localSolrServerPort = 0;
248  private int localSolrStopPort = 0;
249  private File localSolrFolder;
250  private static final String SOLR = "solr";
251  private static final String CORE_PROPERTIES = "core.properties";
252  private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT);
253  private static final int NUM_COLLECTION_CREATION_RETRIES = 5;
254  private static final int NUM_EMBEDDED_SERVER_RETRIES = 12; // attempt to connect to embedded Solr server for 1 minute
255  private static final int EMBEDDED_SERVER_RETRY_WAIT_SEC = 5;
256 
257  public enum CORE_EVT_STATES {
258 
259  STOPPED, STARTED
260  };
261 
262  private enum SOLR_VERSION {
263 
264  SOLR8, SOLR4
265  };
266 
267  // A reference to the locally running Solr instance.
268  private HttpSolrClient localSolrServer = null;
269  private SOLR_VERSION localServerVersion = SOLR_VERSION.SOLR8; // start local Solr 8 by default
270 
271  // A reference to the remote/network running Solr instance.
272  private HttpSolrClient remoteSolrServer;
273 
274  private Collection currentCollection;
275  private final ReentrantReadWriteLock currentCoreLock;
276 
277  private final ServerAction serverAction;
279 
284  Server() {
285  initSettings();
286 
287  localSolrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr");
288 
289  serverAction = new ServerAction();
290  File solr8Folder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
291  File solr4Folder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS
292 
293  // Figure out where Java is located. The Java home location
294  // will be passed as the SOLR_JAVA_HOME environment
295  // variable to the Solr script but it can be overridden by the user in
296  // either autopsy-solr.cmd or autopsy-solr-in.cmd.
297  javaPath = PlatformUtil.getJavaPath();
298 
299  Path solr8Home = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr"); //NON-NLS
300  try {
301  // Always copy the config files, as they may have changed. Otherwise potentially stale Solr configuration is being used.
302  if (!solr8Home.toFile().exists()) {
303  Files.createDirectory(solr8Home);
304  } else {
305  // delete the configsets directory as the Autopsy configset could have changed
306  FileUtil.deleteDir(solr8Home.resolve("configsets").toFile());
307  }
308  Files.copy(Paths.get(solr8Folder.getAbsolutePath(), "server", "solr", "solr.xml"), solr8Home.resolve("solr.xml"), REPLACE_EXISTING); //NON-NLS
309  Files.copy(Paths.get(solr8Folder.getAbsolutePath(), "server", "solr", "zoo.cfg"), solr8Home.resolve("zoo.cfg"), REPLACE_EXISTING); //NON-NLS
310  FileUtils.copyDirectory(Paths.get(solr8Folder.getAbsolutePath(), "server", "solr", "configsets").toFile(), solr8Home.resolve("configsets").toFile()); //NON-NLS
311  } catch (IOException ex) {
312  logger.log(Level.SEVERE, "Failed to create Solr 8 home folder:", ex); //NON-NLS
313  }
314 
315  Path solr4Home = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr4"); //NON-NLS
316  try {
317  // Always copy the config files, as they may have changed. Otherwise potentially stale Solr configuration is being used.
318  if (!solr4Home.toFile().exists()) {
319  Files.createDirectory(solr4Home);
320  }
321  Files.copy(Paths.get(solr4Folder.getAbsolutePath(), "solr", "solr.xml"), solr4Home.resolve("solr.xml"), REPLACE_EXISTING); //NON-NLS
322  Files.copy(Paths.get(solr4Folder.getAbsolutePath(), "solr", "zoo.cfg"), solr4Home.resolve("zoo.cfg"), REPLACE_EXISTING); //NON-NLS
323  } catch (IOException ex) {
324  logger.log(Level.SEVERE, "Failed to create Solr 4 home folder:", ex); //NON-NLS
325  }
326 
327  currentCoreLock = new ReentrantReadWriteLock(true);
328 
329  logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
330  }
331 
332  private void initSettings() {
333 
334  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)) {
335  try {
336  localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
337  } catch (NumberFormatException nfe) {
338  logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
339  localSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
340  }
341  } else {
342  localSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
343  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(localSolrServerPort));
344  }
345 
346  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT)) {
347  try {
348  localSolrStopPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT));
349  } catch (NumberFormatException nfe) {
350  logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
351  localSolrStopPort = DEFAULT_SOLR_STOP_PORT;
352  }
353  } else {
354  localSolrStopPort = DEFAULT_SOLR_STOP_PORT;
355  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(localSolrStopPort));
356  }
357  }
358 
359  private HttpSolrClient getSolrClient(String solrUrl) {
360  int connectionTimeoutMs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getConnectionTimeout();
361  return new HttpSolrClient.Builder(solrUrl)
362  .withSocketTimeout(connectionTimeoutMs)
363  .withConnectionTimeout(connectionTimeoutMs)
364  .withResponseParser(new XMLResponseParser())
365  .build();
366  }
367 
368  private ConcurrentUpdateSolrClient getConcurrentClient(String solrUrl) {
369  int numThreads = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getNumThreads();
370  int numDocs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getDocumentsQueueSize();
371  int connectionTimeoutMs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getConnectionTimeout();
372  logger.log(Level.INFO, "Creating new ConcurrentUpdateSolrClient: {0}", solrUrl); //NON-NLS
373  logger.log(Level.INFO, "Queue size = {0}, Number of threads = {1}, Connection Timeout (ms) = {2}", new Object[]{numDocs, numThreads, connectionTimeoutMs}); //NON-NLS
374  ConcurrentUpdateSolrClient client = new ConcurrentUpdateSolrClient.Builder(solrUrl)
375  .withQueueSize(numDocs)
376  .withThreadCount(numThreads)
377  .withSocketTimeout(connectionTimeoutMs)
378  .withConnectionTimeout(connectionTimeoutMs)
379  .withResponseParser(new XMLResponseParser())
380  .build();
381 
382  return client;
383  }
384 
385  private CloudSolrClient getCloudSolrClient(String host, String port, String defaultCollectionName) throws KeywordSearchModuleException {
386  List<String> solrServerList = getSolrServerList(host, port);
387  List<String> solrUrls = new ArrayList<>();
388  for (String server : solrServerList) {
389  solrUrls.add("http://" + server + "/solr");
390  logger.log(Level.INFO, "Using Solr server: {0}", server);
391  }
392 
393  logger.log(Level.INFO, "Creating new CloudSolrClient"); //NON-NLS
394  int connectionTimeoutMs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getConnectionTimeout();
395  CloudSolrClient client = new CloudSolrClient.Builder(solrUrls)
396  .withConnectionTimeout(connectionTimeoutMs)
397  .withSocketTimeout(connectionTimeoutMs)
398  .withResponseParser(new XMLResponseParser())
399  .build();
400  if (!defaultCollectionName.isEmpty()) {
401  client.setDefaultCollection(defaultCollectionName);
402  }
403  client.connect();
404  return client;
405  }
406 
407  @Override
408  public void finalize() throws java.lang.Throwable {
409  stop();
410  super.finalize();
411  }
412 
413  public void addServerActionListener(PropertyChangeListener l) {
414  serverAction.addPropertyChangeListener(l);
415  }
416 
417  int getLocalSolrServerPort() {
418  return localSolrServerPort;
419  }
420 
421  int getLocalSolrStopPort() {
422  return localSolrStopPort;
423  }
424 
428  private static class InputStreamPrinterThread extends Thread {
429 
430  InputStream stream;
431  OutputStream out;
432  volatile boolean doRun = true;
433 
434  InputStreamPrinterThread(InputStream stream, String type) {
435  this.stream = stream;
436  try {
437  final String log = Places.getUserDirectory().getAbsolutePath()
438  + File.separator + "var" + File.separator + "log" //NON-NLS
439  + File.separator + "solr.log." + type; //NON-NLS
440  File outputFile = new File(log.concat(".0"));
441  File first = new File(log.concat(".1"));
442  File second = new File(log.concat(".2"));
443  if (second.exists()) {
444  second.delete();
445  }
446  if (first.exists()) {
447  first.renameTo(second);
448  }
449  if (outputFile.exists()) {
450  outputFile.renameTo(first);
451  } else {
452  outputFile.createNewFile();
453  }
454  out = new FileOutputStream(outputFile);
455 
456  } catch (Exception ex) {
457  logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
458  }
459  }
460 
461  void stopRun() {
462  doRun = false;
463  }
464 
465  @Override
466  public void run() {
467 
468  try (InputStreamReader isr = new InputStreamReader(stream);
469  BufferedReader br = new BufferedReader(isr);
470  OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset());
471  BufferedWriter bw = new BufferedWriter(osw);) {
472 
473  String line = null;
474  while (doRun && (line = br.readLine()) != null) {
475  bw.write(line);
476  bw.newLine();
477  if (DEBUG) {
478  //flush buffers if dev version for debugging
479  bw.flush();
480  }
481  }
482  bw.flush();
483  } catch (IOException ex) {
484  logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
485  }
486  }
487  }
488 
498  private Process runLocalSolr8ControlCommand(List<String> solrArguments) throws IOException {
499  final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + UserPreferences.getMaxSolrVMSize() + "m"; //NON-NLS
500 
501  // This is our customized version of the Solr batch script to start/stop Solr.
502  File solr8Folder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
503  Path solr8CmdPath;
505  solr8CmdPath = Paths.get(solr8Folder.getAbsolutePath(), "bin", "autopsy-solr.cmd"); //NON-NLS
506  } else {
507  solr8CmdPath = Paths.get(solr8Folder.getAbsolutePath(), "bin", "autopsy-solr"); //NON-NLS
508  }
509  Path solr8Home = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr"); //NON-NLS
510 
511  List<String> commandLine = new ArrayList<>();
512  commandLine.add(solr8CmdPath.toString());
513  commandLine.addAll(solrArguments);
514 
515  ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
516  solrProcessBuilder.directory(solr8Folder);
517 
518  // Redirect stdout and stderr to files to prevent blocking.
519  Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
520  solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
521 
522  Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
523  solrProcessBuilder.redirectError(solrStderrPath.toFile());
524 
525  // get the path to the JRE folder. That's what Solr needs as SOLR_JAVA_HOME
526  String jreFolderPath = new File(javaPath).getParentFile().getParentFile().getAbsolutePath();
527 
528  solrProcessBuilder.environment().put("SOLR_JAVA_HOME", jreFolderPath); // NON-NLS
529  solrProcessBuilder.environment().put("SOLR_HOME", solr8Home.toString()); // NON-NLS
530  solrProcessBuilder.environment().put("STOP_KEY", KEY); // NON-NLS
531  solrProcessBuilder.environment().put("SOLR_JAVA_MEM", MAX_SOLR_MEM_MB_PAR); // NON-NLS
532  logger.log(Level.INFO, "Setting Solr 8 directory: {0}", solr8Folder.toString()); //NON-NLS
533  logger.log(Level.INFO, "Running Solr 8 command: {0} from {1}", new Object[]{solrProcessBuilder.command(), solr8Folder.toString()}); //NON-NLS
534  Process process = solrProcessBuilder.start();
535  logger.log(Level.INFO, "Finished running Solr 8 command"); //NON-NLS
536  return process;
537  }
538 
548  private Process runLocalSolr4ControlCommand(List<String> solrArguments) throws IOException {
549  final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + UserPreferences.getMaxSolrVMSize() + "m"; //NON-NLS
550  File solr4Folder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS
551 
552  List<String> commandLine = new ArrayList<>();
553  commandLine.add(javaPath);
554  commandLine.add(MAX_SOLR_MEM_MB_PAR);
555  commandLine.add("-DSTOP.PORT=" + localSolrStopPort); //NON-NLS
556  commandLine.add("-Djetty.port=" + localSolrServerPort); //NON-NLS
557  commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
558  commandLine.add("-jar"); //NON-NLS
559  commandLine.add("start.jar"); //NON-NLS
560 
561  commandLine.addAll(solrArguments);
562 
563  ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
564  solrProcessBuilder.directory(solr4Folder);
565 
566  // Redirect stdout and stderr to files to prevent blocking.
567  Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
568  solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
569 
570  Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
571  solrProcessBuilder.redirectError(solrStderrPath.toFile());
572 
573  logger.log(Level.INFO, "Running Solr 4 command: {0}", solrProcessBuilder.command()); //NON-NLS
574  Process process = solrProcessBuilder.start();
575  logger.log(Level.INFO, "Finished running Solr 4 command"); //NON-NLS
576  return process;
577  }
578 
584  List<Long> getSolrPIDs() {
585  List<Long> pids = new ArrayList<>();
586 
587  //NOTE: these needs to be in sync with process start string in start()
588  final String pidsQuery = "-DSTOP.KEY=" + KEY + "%start.jar"; //NON-NLS
589 
590  long[] pidsArr = PlatformUtil.getJavaPIDs(pidsQuery);
591  if (pidsArr != null) {
592  for (int i = 0; i < pidsArr.length; ++i) {
593  pids.add(pidsArr[i]);
594  }
595  }
596 
597  return pids;
598  }
599 
604  void killSolr() {
605  List<Long> solrPids = getSolrPIDs();
606  for (long pid : solrPids) {
607  logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
608  PlatformUtil.killProcess(pid);
609  }
610  }
611 
612  void start() throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException {
613  startLocalSolr(SOLR_VERSION.SOLR8);
614  }
615 
616  @Messages({
617  "# {0} - indexVersion",
618  "Server_configureSolrConnection_illegalSolrVersion=The solr version in the case: {0}, is not supported."
619  })
620  private void configureSolrConnection(Case theCase, Index index) throws KeywordSearchModuleException, SolrServerNoPortException {
621 
622  try {
623  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
624 
625  // makes sure the proper local Solr server is running
626  if (!IndexFinder.getCurrentSolrVersion().equals(index.getSolrVersion())) {
627  throw new KeywordSearchModuleException(Bundle.Server_configureSolrConnection_illegalSolrVersion(index.getSolrVersion()));
628  }
629 
630  startLocalSolr(SOLR_VERSION.SOLR8);
631 
632  // check if the local Solr server is running
633  if (!this.isLocalSolrRunning()) {
634  logger.log(Level.SEVERE, "Local Solr server is not running"); //NON-NLS
635  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
636  }
637  } else {
638  // create SolrJ client to connect to remore Solr server
639  remoteSolrServer = configureMultiUserConnection(theCase, index, "");
640 
641  // test the connection
642  connectToSolrServer(remoteSolrServer);
643  }
644  } catch (SolrServerException | IOException ex) {
645  throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
646  }
647  }
648 
663  private HttpSolrClient configureMultiUserConnection(Case theCase, Index index, String name) throws KeywordSearchModuleException {
664 
665  // read Solr connection info from user preferences, unless "solrserver.txt" is present
667  if (properties.host.isEmpty() || properties.port.isEmpty()) {
668  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.connectionInfoMissing.exception.msg", index.getSolrVersion()));
669  }
670  String solrUrl = "http://" + properties.host + ":" + properties.port + "/solr";
671 
672  if (!name.isEmpty()) {
673  solrUrl = solrUrl + "/" + name;
674  }
675 
676  // create SolrJ client to connect to remore Solr server
677  return getSolrClient(solrUrl);
678  }
679 
685  @NbBundle.Messages({
686  "Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize.",})
687  synchronized void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException {
688 
689  logger.log(Level.INFO, "Starting local Solr " + version + " server"); //NON-NLS
690  if (version == SOLR_VERSION.SOLR8) {
691  localSolrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
692  } else {
693  throw new KeywordSearchModuleException(Bundle.Server_configureSolrConnection_illegalSolrVersion(version.name()));
694  }
695 
696  if (isLocalSolrRunning()) {
697  if (localServerVersion.equals(version)) {
698  // this version of local server is already running
699  logger.log(Level.INFO, "Local Solr " + version + " server is already running"); //NON-NLS
700  return;
701  } else {
702  // wrong version of local server is running, stop it
703  stop();
704  }
705  }
706 
707  // set which version of local server is currently running
708  localServerVersion = version;
709 
710  if (!isPortAvailable(localSolrServerPort)) {
711  // There is something already listening on our port. Let's see if
712  // this is from an earlier run that didn't successfully shut down
713  // and if so kill it.
714  final List<Long> pids = this.getSolrPIDs();
715 
716  // If the culprit listening on the port is not a Solr process
717  // we refuse to start.
718  if (pids.isEmpty()) {
719  throw new SolrServerNoPortException(localSolrServerPort);
720  }
721 
722  // Ok, we've tried to stop it above but there still appears to be
723  // a Solr process listening on our port so we forcefully kill it.
724  killSolr();
725 
726  // If either of the ports are still in use after our attempt to kill
727  // previously running processes we give up and throw an exception.
728  if (!isPortAvailable(localSolrServerPort)) {
729  throw new SolrServerNoPortException(localSolrServerPort);
730  }
731  if (!isPortAvailable(localSolrStopPort)) {
732  throw new SolrServerNoPortException(localSolrStopPort);
733  }
734  }
735 
736  if (isPortAvailable(localSolrServerPort)) {
737  logger.log(Level.INFO, "Port [{0}] available, starting Solr", localSolrServerPort); //NON-NLS
738  try {
739  if (version == SOLR_VERSION.SOLR8) {
740  logger.log(Level.INFO, "Starting Solr 8 server"); //NON-NLS
741  curSolrProcess = runLocalSolr8ControlCommand(new ArrayList<>(Arrays.asList("start", "-p", //NON-NLS
742  Integer.toString(localSolrServerPort)))); //NON-NLS
743  } else {
744  // solr4
745  logger.log(Level.INFO, "Starting Solr 4 server"); //NON-NLS
746  curSolrProcess = runLocalSolr4ControlCommand(new ArrayList<>(
747  Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
748  "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
749  }
750 
751  // Wait for the Solr server to start and respond to a statusRequest request.
752  for (int numRetries = 0; numRetries < NUM_EMBEDDED_SERVER_RETRIES; numRetries++) {
753  if (isLocalSolrRunning()) {
754  final List<Long> pids = this.getSolrPIDs();
755  logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
756  return;
757  }
758 
759  // Local Solr server did not respond so we sleep for
760  // 5 seconds before trying again.
761  try {
762  TimeUnit.SECONDS.sleep(EMBEDDED_SERVER_RETRY_WAIT_SEC);
763  } catch (InterruptedException ex) {
764  logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
765  }
766  }
767 
768  // If we get here the Solr server has not responded to connection
769  // attempts in a timely fashion.
770  logger.log(Level.WARNING, "Local Solr server failed to respond to status requests.");
771  WindowManager.getDefault().invokeWhenUIReady(new Runnable() {
772  @Override
773  public void run() {
774  MessageNotifyUtil.Notify.error(
775  NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"),
776  Bundle.Server_status_failed_msg());
777  }
778  });
779  } catch (SecurityException ex) {
780  throw new KeywordSearchModuleException(
781  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
782  } catch (IOException ex) {
783  throw new KeywordSearchModuleException(
784  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
785  }
786  }
787  }
788 
795  static boolean isPortAvailable(int port) {
796  // implementation taken from https://stackoverflow.com/a/435579
797  if (port < 1 || port > 65535) {
798  throw new IllegalArgumentException("Invalid start port: " + port);
799  }
800 
801  ServerSocket ss = null;
802  DatagramSocket ds = null;
803  try {
804  ss = new ServerSocket(port);
805  ss.setReuseAddress(true);
806  ds = new DatagramSocket(port);
807  ds.setReuseAddress(true);
808  return true;
809  } catch (IOException e) {
810  } finally {
811  if (ds != null) {
812  ds.close();
813  }
814 
815  if (ss != null) {
816  try {
817  ss.close();
818  } catch (IOException e) {
819  /* should not be thrown */
820  }
821  }
822  }
823 
824  return false;
825  }
826 
827 
833  void changeSolrServerPort(int port) {
834  localSolrServerPort = port;
835  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
836  }
837 
843  void changeSolrStopPort(int port) {
844  localSolrStopPort = port;
845  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
846  }
847 
853  synchronized void stop() {
854 
855  try {
856  // Close any open core before stopping server
857  closeCore();
858  } catch (KeywordSearchModuleException e) {
859  logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
860  }
861 
862  stopLocalSolr();
863  }
864 
868  private void stopLocalSolr() {
869  try {
870  //try graceful shutdown
871  Process process;
872  if (localServerVersion == SOLR_VERSION.SOLR8) {
873  logger.log(Level.INFO, "Stopping Solr 8 server"); //NON-NLS
874  process = runLocalSolr8ControlCommand(new ArrayList<>(Arrays.asList("stop", "-k", KEY, "-p", Integer.toString(localSolrServerPort)))); //NON-NLS
875  } else {
876  // solr 4
877  logger.log(Level.INFO, "Stopping Solr 4 server"); //NON-NLS
878  process = runLocalSolr4ControlCommand(new ArrayList<>(Arrays.asList("--stop"))); //NON-NLS
879  }
880 
881  logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
882  process.waitFor();
883 
884  //if still running, forcefully stop it
885  if (curSolrProcess != null) {
886  curSolrProcess.destroy();
887  curSolrProcess = null;
888  }
889 
890  } catch (IOException | InterruptedException ex) {
891  logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
892  } finally {
893  //stop Solr stream -> log redirect threads
894  try {
895  if (errorRedirectThread != null) {
896  errorRedirectThread.stopRun();
897  errorRedirectThread = null;
898  }
899  } finally {
900  //if still running, kill it
901  killSolr();
902  }
903 
904  logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
905  }
906  }
907 
915  synchronized boolean isLocalSolrRunning() throws KeywordSearchModuleException {
916  try {
917 
918  if (isPortAvailable(localSolrServerPort)) {
919  return false;
920  }
921 
922  // making a statusRequest request here instead of just doing solrServer.ping(), because
923  // that doesn't work when there are no cores
924  //TODO handle timeout in cases when some other type of server on that port
926 
927  logger.log(Level.INFO, "Solr server is running"); //NON-NLS
928  } catch (SolrServerException ex) {
929 
930  Throwable cause = ex.getRootCause();
931 
932  // TODO: check if SocketExceptions should actually happen (is
933  // probably caused by starting a connection as the server finishes
934  // shutting down)
935  if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
936  logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
937  return false;
938  } else {
939  throw new KeywordSearchModuleException(
940  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
941  }
942  } catch (SolrException ex) {
943  // Just log 404 errors for now...
944  logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
945  return false;
946  } catch (IOException ex) {
947  throw new KeywordSearchModuleException(
948  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
949  }
950 
951  return true;
952  }
953 
954  /*
955  * ** Convenience methods for use while we only open one case at a time ***
956  */
966  void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
967  currentCoreLock.writeLock().lock();
968  try {
969  currentCollection = openCore(theCase, index);
970 
971  try {
972  // execute a test query. if it fails, an exception will be thrown
974  } catch (NoOpenCoreException ex) {
975  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
976  }
977 
978  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
979  } finally {
980  currentCoreLock.writeLock().unlock();
981  }
982  }
983 
989  boolean coreIsOpen() {
990  currentCoreLock.readLock().lock();
991  try {
992  return (null != currentCollection);
993  } finally {
994  currentCoreLock.readLock().unlock();
995  }
996  }
997 
998  Index getIndexInfo() throws NoOpenCoreException {
999  currentCoreLock.readLock().lock();
1000  try {
1001  if (null == currentCollection) {
1002  throw new NoOpenCoreException();
1003  }
1004  return currentCollection.getIndexInfo();
1005  } finally {
1006  currentCoreLock.readLock().unlock();
1007  }
1008  }
1009 
1010  void closeCore() throws KeywordSearchModuleException {
1011  currentCoreLock.writeLock().lock();
1012  try {
1013  if (null != currentCollection) {
1014  currentCollection.close();
1015  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
1016  }
1017  } finally {
1018  currentCollection = null;
1019  currentCoreLock.writeLock().unlock();
1020  }
1021  }
1022 
1023  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
1024  currentCoreLock.readLock().lock();
1025  try {
1026  if (null == currentCollection) {
1027  throw new NoOpenCoreException();
1028  }
1029  TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk");
1030  currentCollection.addDocument(doc);
1031  HealthMonitor.submitTimingMetric(metric);
1032  } finally {
1033  currentCoreLock.readLock().unlock();
1034  }
1035  }
1036 
1045  @NbBundle.Messages({
1046  "# {0} - colelction name", "Server.deleteCore.exception.msg=Failed to delete Solr colelction {0}",})
1047  void deleteCollection(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException, KeywordSearchModuleException {
1048  try {
1049  HttpSolrClient solrServer;
1050  if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
1051  solrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
1052  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
1053  if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
1054  /*
1055  * Send a core unload request to the Solr server, with the
1056  * parameter set that request deleting the index and the
1057  * instance directory (deleteInstanceDir = true). Note that
1058  * this removes everything related to the core on the server
1059  * (the index directory, the configuration files, etc.), but
1060  * does not delete the actual Solr text index because it is
1061  * currently stored in the case directory.
1062  */
1063  org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName, true, true, solrServer);
1064  }
1065  } else {
1066  IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory());
1067  solrServer = getSolrClient("http://" + properties.getHost() + ":" + properties.getPort() + "/solr");
1068  connectToSolrServer(solrServer);
1069 
1070  CollectionAdminRequest.Delete deleteCollectionRequest = CollectionAdminRequest.deleteCollection(coreName);
1071  CollectionAdminResponse response = deleteCollectionRequest.process(solrServer);
1072  if (response.isSuccess()) {
1073  logger.log(Level.INFO, "Deleted collection {0}", coreName); //NON-NLS
1074  } else {
1075  logger.log(Level.WARNING, "Unable to delete collection {0}", coreName); //NON-NLS
1076  }
1077  }
1078  } catch (SolrServerException | IOException ex) {
1079  // We will get a RemoteSolrException with cause == null and detailsMessage
1080  // == "Already closed" if the core is not loaded. This is not an error in this scenario.
1081  if (!ex.getMessage().equals("Already closed")) { // NON-NLS
1082  throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
1083  }
1084  }
1085  }
1086 
1098  @NbBundle.Messages({
1099  "Server.exceptionMessage.unableToCreateCollection=Unable to create Solr collection",
1100  "Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection",
1101  "Server.exceptionMessage.unableToRestoreCollection=Unable to restore Solr collection",
1102  })
1103  private Collection openCore(Case theCase, Index index) throws KeywordSearchModuleException {
1104 
1105  int numShardsToUse = 1;
1106  try {
1107  // connect to proper Solr server
1108  configureSolrConnection(theCase, index);
1109 
1110  if (theCase.getCaseType() == CaseType.MULTI_USER_CASE) {
1111  // select number of shards to use
1112  numShardsToUse = getNumShardsToUse();
1113  }
1114  } catch (Exception ex) {
1115  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1116  throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
1117  }
1118 
1119  try {
1120  String collectionName = index.getIndexName();
1121 
1122  if (theCase.getCaseType() == CaseType.MULTI_USER_CASE) {
1123  if (!collectionExists(collectionName)) {
1124  /*
1125  * The collection does not exist. Make a request that will cause the colelction to be created.
1126  */
1127  boolean doRetry = false;
1128  for (int reTryAttempt = 0; reTryAttempt < NUM_COLLECTION_CREATION_RETRIES; reTryAttempt++) {
1129  try {
1130  doRetry = false;
1131  createMultiUserCollection(collectionName, numShardsToUse);
1132  } catch (Exception ex) {
1133  if (reTryAttempt >= NUM_COLLECTION_CREATION_RETRIES) {
1134  logger.log(Level.SEVERE, "Unable to create Solr collection " + collectionName, ex); //NON-NLS
1135  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
1136  } else {
1137  logger.log(Level.SEVERE, "Unable to create Solr collection " + collectionName + ". Re-trying...", ex); //NON-NLS
1138  Thread.sleep(1000L);
1139  doRetry = true;
1140  }
1141  }
1142  if (!doRetry) {
1143  break;
1144  }
1145  }
1146  }
1147  } else {
1148  if (!coreIsLoaded(collectionName)) {
1149  // In single user mode, the index is stored in case output directory
1150  File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
1151  if (!dataDir.exists()) {
1152  dataDir.mkdirs();
1153  }
1154 
1155  // In single user mode, if there is a core.properties file already,
1156  // we've hit a solr bug. Compensate by deleting it.
1157  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
1158  Path corePropertiesFile = Paths.get(localSolrFolder.toString(), SOLR, collectionName, CORE_PROPERTIES);
1159  if (corePropertiesFile.toFile().exists()) {
1160  try {
1161  corePropertiesFile.toFile().delete();
1162  } catch (Exception ex) {
1163  logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
1164  }
1165  }
1166  }
1167 
1168  // for single user cases, we unload the core when we close the case. So we have to load the core again.
1169  CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
1170  createCoreRequest.setDataDir(dataDir.getAbsolutePath());
1171  createCoreRequest.setCoreName(collectionName);
1172  createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
1173  createCoreRequest.setIsLoadOnStartup(false);
1174  createCoreRequest.setIsTransient(true);
1175  localSolrServer.request(createCoreRequest);
1176 
1177  if (!coreIndexFolderExists(collectionName)) {
1178  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
1179  }
1180  }
1181  }
1182 
1183  return new Collection(collectionName, theCase, index);
1184 
1185  } catch (Exception ex) {
1186  logger.log(Level.SEVERE, "Exception during Solr collection creation.", ex); //NON-NLS
1187  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
1188  }
1189  }
1190 
1191  private int getNumShardsToUse() throws KeywordSearchModuleException {
1192 
1193  // if we want to use a specific sharding strategy, use that
1194  if (org.sleuthkit.autopsy.keywordsearch.UserPreferences.getMaxNumShards() > 0) {
1195  return org.sleuthkit.autopsy.keywordsearch.UserPreferences.getMaxNumShards();
1196  }
1197 
1198  // otherwise get list of all live Solr servers in the cluster
1199  List<String> solrServerList = getSolrServerList(remoteSolrServer);
1200  // shard across all available servers
1201  return solrServerList.size();
1202  }
1203 
1204  /*
1205  * Poll the remote Solr server for list of existing collections, and check if
1206  * the collection of interest exists.
1207  *
1208  * @param collectionName The name of the collection.
1209  *
1210  * @return True if the collection exists, false otherwise.
1211  *
1212  * @throws SolrServerException If there is a problem communicating with the
1213  * Solr server.
1214  * @throws IOException If there is a problem communicating with the Solr
1215  * server.
1216  */
1217  private boolean collectionExists(String collectionName) throws SolrServerException, IOException {
1218  CollectionAdminRequest.List req = new CollectionAdminRequest.List();
1219  CollectionAdminResponse response = req.process(remoteSolrServer);
1220  List<?> existingCollections = (List<?>) response.getResponse().get("collections");
1221  if (existingCollections == null) {
1222  existingCollections = new ArrayList<>();
1223  }
1224  return existingCollections.contains(collectionName);
1225  }
1226 
1227  /* NOTE: Keeping this code for reference, since it works.
1228  private boolean collectionExists(String collectionName) throws SolrServerException, IOException {
1229 
1230  // TODO we could potentially use this API. Currently set exception "Solr instance is not running in SolrCloud mode"
1231  // List<String> list = CollectionAdminRequest.listCollections(localSolrServer);
1232 
1233  CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus().setCollectionName(collectionName);
1234  CollectionAdminResponse statusResponse;
1235  try {
1236  statusResponse = statusRequest.process(remoteSolrServer);
1237  } catch (RemoteSolrException ex) {
1238  // collection doesn't exist
1239  return false;
1240  }
1241 
1242  if (statusResponse == null) {
1243  return false;
1244  }
1245 
1246  NamedList error = (NamedList) statusResponse.getResponse().get("error");
1247  if (error != null) {
1248  return false;
1249  }
1250 
1251  // For some reason this returns info about all collections even though it's supposed to only return about the one we are requesting
1252  NamedList cluster = (NamedList) statusResponse.getResponse().get("cluster");
1253  NamedList collections = (NamedList) cluster.get("collections");
1254  if (collections != null) {
1255  Object collection = collections.get(collectionName);
1256  return (collection != null);
1257  } else {
1258  return false;
1259  }
1260  }*/
1261 
1262  private void createMultiUserCollection(String collectionName, int numShardsToUse) throws KeywordSearchModuleException, SolrServerException, IOException {
1263  /*
1264  * The core either does not exist or it is not loaded. Make a
1265  * request that will cause the core to be created if it does not
1266  * exist or loaded if it already exists.
1267  */
1268 
1269  Integer numShards = numShardsToUse;
1270  logger.log(Level.INFO, "numShardsToUse: {0}", numShardsToUse);
1271  Integer numNrtReplicas = 1;
1272  Integer numTlogReplicas = 0;
1273  Integer numPullReplicas = 0;
1274  CollectionAdminRequest.Create createCollectionRequest = CollectionAdminRequest.createCollection(collectionName, "AutopsyConfig", numShards, numNrtReplicas, numTlogReplicas, numPullReplicas);
1275 
1276  CollectionAdminResponse createResponse = createCollectionRequest.process(remoteSolrServer);
1277  if (createResponse.isSuccess()) {
1278  logger.log(Level.INFO, "Collection {0} successfully created.", collectionName);
1279  } else {
1280  logger.log(Level.SEVERE, "Unable to create Solr collection {0}", collectionName); //NON-NLS
1281  throw new KeywordSearchModuleException(Bundle.Server_exceptionMessage_unableToCreateCollection());
1282  }
1283 
1284  /* If we need core name:
1285  Map<String, NamedList<Integer>> status = createResponse.getCollectionCoresStatus();
1286  existingCoreName = status.keySet().iterator().next();*/
1287  if (!collectionExists(collectionName)) {
1288  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
1289  }
1290  }
1291 
1292  private void backupCollection(String collectionName, String backupName, String pathToBackupLocation) throws SolrServerException, IOException, KeywordSearchModuleException {
1293  CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
1294  .setLocation(pathToBackupLocation);
1295 
1296  CollectionAdminResponse backupResponse = backup.process(remoteSolrServer);
1297  if (backupResponse.isSuccess()) {
1298  logger.log(Level.INFO, "Collection {0} successfully backep up.", collectionName);
1299  } else {
1300  logger.log(Level.SEVERE, "Unable to back up Solr collection {0}", collectionName); //NON-NLS
1301  throw new KeywordSearchModuleException(Bundle.Server_exceptionMessage_unableToBackupCollection());
1302  }
1303  }
1304 
1305  private void restoreCollection(String backupName, String restoreCollectionName, String pathToBackupLocation) throws SolrServerException, IOException, KeywordSearchModuleException {
1306 
1307  CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
1308  .setLocation(pathToBackupLocation);
1309 
1310  CollectionAdminResponse restoreResponse = restore.process(remoteSolrServer);
1311  if (restoreResponse.isSuccess()) {
1312  logger.log(Level.INFO, "Collection {0} successfully resored.", restoreCollectionName);
1313  } else {
1314  logger.log(Level.SEVERE, "Unable to restore Solr collection {0}", restoreCollectionName); //NON-NLS
1315  throw new KeywordSearchModuleException(Bundle.Server_exceptionMessage_unableToRestoreCollection());
1316  }
1317  }
1318 
1333  private boolean coreIsLoaded(String coreName) throws SolrServerException, IOException {
1334  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, localSolrServer);
1335  return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1336  }
1337 
1350  private boolean coreIndexFolderExists(String coreName) throws SolrServerException, IOException {
1351  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, localSolrServer);
1352  Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1353  if (null != dataDirPath) {
1354  File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1355  return indexDir.exists();
1356  } else {
1357  return false;
1358  }
1359  }
1360 
1372  public static IndexingServerProperties getMultiUserServerProperties(String caseDirectory) {
1373 
1374  // if "solrserver.txt" is present, use those connection settings
1375  Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt"); //NON-NLS
1376  if (serverFilePath.toFile().exists()) {
1377  try {
1378  List<String> lines = Files.readAllLines(serverFilePath);
1379  if (lines.isEmpty()) {
1380  logger.log(Level.SEVERE, "solrserver.txt file does not contain any data"); //NON-NLS
1381  } else if (!lines.get(0).contains(",")) {
1382  logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0)); //NON-NLS
1383  } else {
1384  String[] parts = lines.get(0).split(",");
1385  if (parts.length != 2) {
1386  logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0)); //NON-NLS
1387  } else {
1388  return new IndexingServerProperties(parts[0], parts[1]);
1389  }
1390  }
1391  } catch (IOException ex) {
1392  logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex); //NON-NLS
1393  }
1394  }
1395 
1396  // otherwise (or if an error occurred) determine Solr version of the current case
1397  List<Index> indexes = new ArrayList<>();
1398  try {
1399  IndexMetadata indexMetadata = new IndexMetadata(caseDirectory);
1400  indexes = indexMetadata.getIndexes();
1401  } catch (IndexMetadata.TextIndexMetadataException ex) {
1402  logger.log(Level.SEVERE, "Unable to read text index metadata file: " + caseDirectory, ex);
1403 
1404  // default to using the latest Solr version settings
1405  String host = UserPreferences.getIndexingServerHost();
1406  String port = UserPreferences.getIndexingServerPort();
1407  return new IndexingServerProperties(host, port);
1408  }
1409 
1410  // select which index to use. In practice, all cases always have only one
1411  // index but there is support for having multiple indexes.
1412  Index indexToUse = IndexFinder.identifyIndexToUse(indexes);
1413  if (indexToUse == null) {
1414  // unable to find index that can be used
1415  logger.log(Level.SEVERE, "Unable to find index that can be used for case: {0}", caseDirectory);
1416 
1417  // default to using the latest Solr version settings
1418  String host = UserPreferences.getIndexingServerHost();
1419  String port = UserPreferences.getIndexingServerPort();
1420  return new IndexingServerProperties(host, port);
1421  }
1422 
1423  // return connection info for the Solr version of the current index
1424  if (IndexFinder.getCurrentSolrVersion().equals(indexToUse.getSolrVersion())) {
1425  // Solr 8
1426  String host = UserPreferences.getIndexingServerHost();
1427  String port = UserPreferences.getIndexingServerPort();
1428  return new IndexingServerProperties(host, port);
1429  } else {
1430  // Solr 4
1431  String host = UserPreferences.getSolr4ServerHost().trim();
1432  String port = UserPreferences.getSolr4ServerPort().trim();
1433  return new IndexingServerProperties(host, port);
1434  }
1435  }
1436 
1448  public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath) throws KeywordSearchModuleException {
1449  // Look for the solr server list file
1450  String serverListName = "solrServerList.txt"; //NON-NLS
1451  Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
1452  if (serverListPath.toFile().exists()) {
1453 
1454  // Read the list of solr servers
1455  List<String> lines;
1456  try {
1457  lines = Files.readAllLines(serverListPath);
1458  } catch (IOException ex) {
1459  throw new KeywordSearchModuleException(serverListName + " could not be read", ex); //NON-NLS
1460  }
1461 
1462  // Remove any lines that don't contain a comma (these are likely just whitespace)
1463  for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
1464  String line = iterator.next();
1465  if (!line.contains(",")) {
1466  // Remove the current element from the iterator and the list.
1467  iterator.remove();
1468  }
1469  }
1470  if (lines.isEmpty()) {
1471  throw new KeywordSearchModuleException(serverListName + " had no valid server information"); //NON-NLS
1472  }
1473 
1474  // Choose which server to use
1475  int rnd = new Random().nextInt(lines.size());
1476  String[] parts = lines.get(rnd).split(",");
1477  if (parts.length != 2) {
1478  throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd)); //NON-NLS
1479  }
1480 
1481  // Split it up just to do a sanity check on the data
1482  String host = parts[0];
1483  String port = parts[1];
1484  if (host.isEmpty() || port.isEmpty()) {
1485  throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd)); //NON-NLS
1486  }
1487 
1488  // Write the server data to a file
1489  Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt"); //NON-NLS
1490  try {
1491  caseDirectoryPath.toFile().mkdirs();
1492  if (!caseDirectoryPath.toFile().exists()) {
1493  throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist"); //NON-NLS
1494  }
1495  Files.write(serverFile, lines.get(rnd).getBytes());
1496  } catch (IOException ex) {
1497  throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex); //NON-NLS
1498  }
1499  }
1500  }
1501 
1505  public static class IndexingServerProperties {
1506 
1507  private final String host;
1508  private final String port;
1509 
1510  IndexingServerProperties(String host, String port) {
1511  this.host = host;
1512  this.port = port;
1513  }
1514 
1520  public String getHost() {
1521  return host;
1522  }
1523 
1529  public String getPort() {
1530  return port;
1531  }
1532  }
1533 
1539  void commit() throws SolrServerException, NoOpenCoreException {
1540  currentCoreLock.readLock().lock();
1541  try {
1542  if (null == currentCollection) {
1543  throw new NoOpenCoreException();
1544  }
1545  currentCollection.commit();
1546  } finally {
1547  currentCoreLock.readLock().unlock();
1548  }
1549  }
1550 
1551  NamedList<Object> request(SolrRequest<?> request) throws SolrServerException, RemoteSolrException, NoOpenCoreException {
1552  currentCoreLock.readLock().lock();
1553  try {
1554  if (null == currentCollection) {
1555  throw new NoOpenCoreException();
1556  }
1557  return currentCollection.request(request);
1558  } finally {
1559  currentCoreLock.readLock().unlock();
1560  }
1561  }
1562 
1573  public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException {
1574  currentCoreLock.readLock().lock();
1575  try {
1576  if (null == currentCollection) {
1577  throw new NoOpenCoreException();
1578  }
1579  try {
1580  return currentCollection.queryNumIndexedFiles();
1581  } catch (Exception ex) {
1582  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1583  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
1584  }
1585  } finally {
1586  currentCoreLock.readLock().unlock();
1587  }
1588  }
1589 
1599  public int queryNumIndexedChunks() throws KeywordSearchModuleException, NoOpenCoreException {
1600  currentCoreLock.readLock().lock();
1601  try {
1602  if (null == currentCollection) {
1603  throw new NoOpenCoreException();
1604  }
1605  try {
1606  return currentCollection.queryNumIndexedChunks();
1607  } catch (Exception ex) {
1608  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1609  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
1610  }
1611  } finally {
1612  currentCoreLock.readLock().unlock();
1613  }
1614  }
1615 
1625  public int queryNumIndexedDocuments() throws KeywordSearchModuleException, NoOpenCoreException {
1626  currentCoreLock.readLock().lock();
1627  try {
1628  if (null == currentCollection) {
1629  throw new NoOpenCoreException();
1630  }
1631  try {
1632  return currentCollection.queryNumIndexedDocuments();
1633  } catch (Exception ex) {
1634  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1635  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
1636  }
1637  } finally {
1638  currentCoreLock.readLock().unlock();
1639  }
1640  }
1641 
1652  public boolean queryIsFullyIndexed(long contentID) throws KeywordSearchModuleException, NoOpenCoreException {
1653  currentCoreLock.readLock().lock();
1654  try {
1655  if (null == currentCollection) {
1656  throw new NoOpenCoreException();
1657  }
1658  try {
1659  int totalNumChunks = currentCollection.queryTotalNumFileChunks(contentID);
1660  if (totalNumChunks == 0) {
1661  return false;
1662  }
1663 
1664  int numIndexedChunks = currentCollection.queryNumIndexedChunks(contentID);
1665  return numIndexedChunks == totalNumChunks;
1666  } catch (Exception ex) {
1667  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1668  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
1669  }
1670 
1671  } finally {
1672  currentCoreLock.readLock().unlock();
1673  }
1674  }
1675 
1687  public int queryNumFileChunks(long fileID) throws KeywordSearchModuleException, NoOpenCoreException {
1688  currentCoreLock.readLock().lock();
1689  try {
1690  if (null == currentCollection) {
1691  throw new NoOpenCoreException();
1692  }
1693  try {
1694  return currentCollection.queryTotalNumFileChunks(fileID);
1695  } catch (Exception ex) {
1696  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1697  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
1698  }
1699  } finally {
1700  currentCoreLock.readLock().unlock();
1701  }
1702  }
1703 
1714  public QueryResponse query(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1715  currentCoreLock.readLock().lock();
1716  try {
1717  if (null == currentCollection) {
1718  throw new NoOpenCoreException();
1719  }
1720  try {
1721  return currentCollection.query(sq);
1722  } catch (Exception ex) {
1723  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1724  logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1725  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1726  }
1727  } finally {
1728  currentCoreLock.readLock().unlock();
1729  }
1730  }
1731 
1743  public QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1744  currentCoreLock.readLock().lock();
1745  try {
1746  if (null == currentCollection) {
1747  throw new NoOpenCoreException();
1748  }
1749  try {
1750  return currentCollection.query(sq, method);
1751  } catch (Exception ex) {
1752  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1753  logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1754  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1755  }
1756  } finally {
1757  currentCoreLock.readLock().unlock();
1758  }
1759  }
1760 
1771  public TermsResponse queryTerms(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException {
1772  currentCoreLock.readLock().lock();
1773  try {
1774  if (null == currentCollection) {
1775  throw new NoOpenCoreException();
1776  }
1777  try {
1778  return currentCollection.queryTerms(sq);
1779  } catch (Exception ex) {
1780  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1781  logger.log(Level.SEVERE, "Solr terms query failed: " + sq.getQuery(), ex); //NON-NLS
1782  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1783  }
1784  } finally {
1785  currentCoreLock.readLock().unlock();
1786  }
1787  }
1788 
1796  void deleteDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException {
1797  try {
1798  currentCoreLock.writeLock().lock();
1799  if (null == currentCollection) {
1800  throw new NoOpenCoreException();
1801  }
1802  currentCollection.deleteDataSource(dataSourceId);
1803  currentCollection.commit();
1804  } finally {
1805  currentCoreLock.writeLock().unlock();
1806  }
1807  }
1808 
1817  @NbBundle.Messages({
1818  "Server.getAllTerms.error=Extraction of all unique Solr terms failed:"})
1819  void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws KeywordSearchModuleException, NoOpenCoreException {
1820  try {
1821  currentCoreLock.writeLock().lock();
1822  if (null == currentCollection) {
1823  throw new NoOpenCoreException();
1824  }
1825  try {
1826  currentCollection.extractAllTermsForDataSource(outputFile, progressPanel);
1827  } catch (Exception ex) {
1828  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1829  logger.log(Level.SEVERE, "Extraction of all unique Solr terms failed: ", ex); //NON-NLS
1830  throw new KeywordSearchModuleException(Bundle.Server_getAllTerms_error(), ex);
1831  }
1832  } finally {
1833  currentCoreLock.writeLock().unlock();
1834  }
1835  }
1836 
1846  public String getSolrContent(final Content content) throws NoOpenCoreException {
1847  currentCoreLock.readLock().lock();
1848  try {
1849  if (null == currentCollection) {
1850  throw new NoOpenCoreException();
1851  }
1852  return currentCollection.getSolrContent(content.getId(), 0);
1853  } finally {
1854  currentCoreLock.readLock().unlock();
1855  }
1856  }
1857 
1870  public String getSolrContent(final Content content, int chunkID) throws NoOpenCoreException {
1871  currentCoreLock.readLock().lock();
1872  try {
1873  if (null == currentCollection) {
1874  throw new NoOpenCoreException();
1875  }
1876  return currentCollection.getSolrContent(content.getId(), chunkID);
1877  } finally {
1878  currentCoreLock.readLock().unlock();
1879  }
1880  }
1881 
1891  public String getSolrContent(final long objectID) throws NoOpenCoreException {
1892  currentCoreLock.readLock().lock();
1893  try {
1894  if (null == currentCollection) {
1895  throw new NoOpenCoreException();
1896  }
1897  return currentCollection.getSolrContent(objectID, 0);
1898  } finally {
1899  currentCoreLock.readLock().unlock();
1900  }
1901  }
1902 
1913  public String getSolrContent(final long objectID, final int chunkID) throws NoOpenCoreException {
1914  currentCoreLock.readLock().lock();
1915  try {
1916  if (null == currentCollection) {
1917  throw new NoOpenCoreException();
1918  }
1919  return currentCollection.getSolrContent(objectID, chunkID);
1920  } finally {
1921  currentCoreLock.readLock().unlock();
1922  }
1923  }
1924 
1934  public static String getChunkIdString(long parentID, int childID) {
1935  return Long.toString(parentID) + Server.CHUNK_ID_SEPARATOR + Integer.toString(childID);
1936  }
1937 
1944  private void connectToEmbeddedSolrServer() throws SolrServerException, IOException {
1945  TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
1946  CoreAdminRequest.getStatus(null, localSolrServer);
1948  }
1949 
1961  void connectToSolrServer(String host, String port) throws SolrServerException, IOException {
1962  try (HttpSolrClient solrServer = getSolrClient("http://" + host + ":" + port + "/solr")) {
1963  connectToSolrServer(solrServer);
1964  }
1965  }
1966 
1976  private void connectToSolrServer(HttpSolrClient solrServer) throws SolrServerException, IOException {
1977  TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
1978  CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus();
1979  CollectionAdminResponse statusResponse = statusRequest.process(solrServer);
1980  int statusCode = Integer.valueOf(((NamedList) statusResponse.getResponse().get("responseHeader")).get("status").toString());
1981  if (statusCode != 0) {
1982  logger.log(Level.WARNING, "Could not connect to Solr server "); //NON-NLS
1983  } else {
1984  logger.log(Level.INFO, "Connected to Solr server "); //NON-NLS
1985  }
1987  }
1988 
1989  private List<String> getSolrServerList(String host, String port) throws KeywordSearchModuleException {
1990  HttpSolrClient solrServer = getSolrClient("http://" + host + ":" + port + "/solr");
1991  return getSolrServerList(solrServer);
1992  }
1993 
1994  private List<String> getSolrServerList(HttpSolrClient solrServer) throws KeywordSearchModuleException {
1995 
1996  try {
1997  CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus();
1998  CollectionAdminResponse statusResponse;
1999  try {
2000  statusResponse = statusRequest.process(solrServer);
2001  } catch (RemoteSolrException ex) {
2002  // collection doesn't exist
2003  return Collections.emptyList();
2004  }
2005 
2006  if (statusResponse == null) {
2007  return Collections.emptyList();
2008  }
2009 
2010  NamedList<?> error = (NamedList) statusResponse.getResponse().get("error");
2011  if (error != null) {
2012  return Collections.emptyList();
2013  }
2014 
2015  NamedList<?> cluster = (NamedList) statusResponse.getResponse().get("cluster");
2016  @SuppressWarnings("unchecked")
2017  List<String> liveNodes = (ArrayList) cluster.get("live_nodes");
2018 
2019  if (liveNodes != null) {
2020  liveNodes = liveNodes.stream()
2021  .map(serverStr -> serverStr.endsWith("_solr")
2022  ? serverStr.substring(0, serverStr.length() - "_solr".length())
2023  : serverStr)
2024  .collect(Collectors.toList());
2025  }
2026  return liveNodes;
2027  } catch (Exception ex) {
2028  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2029  throw new KeywordSearchModuleException(
2030  NbBundle.getMessage(this.getClass(), "Server.serverList.exception.msg", solrServer.getBaseURL()));
2031  }
2032  }
2033 
2034  class Collection {
2035 
2036  // handle to the collection in Solr
2037  private final String name;
2038 
2039  private final CaseType caseType;
2040 
2041  private final Index textIndex;
2042 
2043  // We use different Solr clients for different operations. HttpSolrClient is geared towards query performance.
2044  // ConcurrentUpdateSolrClient is geared towards batching solr documents for better indexing throughput. We
2045  // have implemented our own batching algorithm so we will probably not use ConcurrentUpdateSolrClient.
2046  // CloudSolrClient is geared towards SolrCloud deployments. These are only good for collection-specific operations.
2047  private HttpSolrClient queryClient;
2048  private SolrClient indexingClient;
2049 
2050  private final int maxBufferSize;
2051  private final List<SolrInputDocument> buffer;
2052  private final Object bufferLock;
2053 
2054  /* (JIRA-7521) Sometimes we get into a situation where Solr server is no longer able to index new data.
2055  * Typically main reason for this is Solr running out of memory. In this case we will stop trying to send new
2056  * data to Solr (for this collection) after certain number of consecutive batches have failed. */
2057  private static final int MAX_NUM_CONSECUTIVE_FAILURES = 5;
2058  private AtomicInteger numConsecutiveFailures = new AtomicInteger(0);
2059  private AtomicBoolean skipIndexing = new AtomicBoolean(false);
2060 
2061  private final ScheduledThreadPoolExecutor periodicTasksExecutor;
2062  private static final long PERIODIC_BATCH_SEND_INTERVAL_MINUTES = 10;
2063  private static final int NUM_BATCH_UPDATE_RETRIES = 10;
2064  private static final long SLEEP_BETWEEN_RETRIES_MS = 10000; // 10 seconds
2065 
2066  private Collection(String name, Case theCase, Index index) throws TimeoutException, InterruptedException, KeywordSearchModuleException {
2067  this.name = name;
2068  this.caseType = theCase.getCaseType();
2069  this.textIndex = index;
2070  bufferLock = new Object();
2071 
2072  if (caseType == CaseType.SINGLE_USER_CASE) {
2073  // get SolrJ client
2074  queryClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
2075  indexingClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
2076  } else {
2077  // read Solr connection info from user preferences, unless "solrserver.txt" is present
2078  queryClient = configureMultiUserConnection(theCase, index, name);
2079 
2080  // for MU cases, use CloudSolrClient for indexing. Indexing is only supported for Solr 8.
2081  if (IndexFinder.getCurrentSolrVersion().equals(index.getSolrVersion())) {
2082  IndexingServerProperties properties = getMultiUserServerProperties(theCase.getCaseDirectory());
2083  indexingClient = getCloudSolrClient(properties.getHost(), properties.getPort(), name); // CloudClient
2084  } else {
2085  indexingClient = configureMultiUserConnection(theCase, index, name); // HttpClient
2086  }
2087  }
2088 
2089  // document batching
2090  maxBufferSize = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getDocumentsQueueSize();
2091  logger.log(Level.INFO, "Using Solr document queue size = {0}", maxBufferSize); //NON-NLS
2092  buffer = new ArrayList<>(maxBufferSize);
2093  periodicTasksExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("periodic-batched-document-task-%d").build()); //NON-NLS
2094  periodicTasksExecutor.scheduleWithFixedDelay(new SendBatchedDocumentsTask(), PERIODIC_BATCH_SEND_INTERVAL_MINUTES, PERIODIC_BATCH_SEND_INTERVAL_MINUTES, TimeUnit.MINUTES);
2095  }
2096 
2103  private final class SendBatchedDocumentsTask implements Runnable {
2104 
2105  @Override
2106  public void run() {
2107 
2108  if (skipIndexing.get()) {
2109  return;
2110  }
2111 
2112  List<SolrInputDocument> clone;
2113  synchronized (bufferLock) {
2114 
2115  if (buffer.isEmpty()) {
2116  return;
2117  }
2118 
2119  // Buffer is full. Make a clone and release the lock, so that we don't
2120  // hold other ingest threads
2121  clone = buffer.stream().collect(toList());
2122  buffer.clear();
2123  }
2124 
2125  try {
2126  // send the cloned list to Solr
2127  sendBufferedDocs(clone);
2128  } catch (KeywordSearchModuleException ex) {
2129  logger.log(Level.SEVERE, "Periodic batched document update failed", ex); //NON-NLS
2130  }
2131  }
2132  }
2133 
2139  String getName() {
2140  return name;
2141  }
2142 
2143  private Index getIndexInfo() {
2144  return this.textIndex;
2145  }
2146 
2147  private QueryResponse query(SolrQuery sq) throws SolrServerException, IOException {
2148  return queryClient.query(sq);
2149  }
2150 
2151  private NamedList<Object> request(SolrRequest<?> request) throws SolrServerException, RemoteSolrException {
2152  try {
2153  return queryClient.request(request);
2154  } catch (Exception e) {
2155  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2156  logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
2157  throw new SolrServerException(
2158  NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
2159  }
2160 
2161  }
2162 
2163  private QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
2164  return queryClient.query(sq, method);
2165  }
2166 
2167  private TermsResponse queryTerms(SolrQuery sq) throws SolrServerException, IOException {
2168  QueryResponse qres = queryClient.query(sq);
2169  return qres.getTermsResponse();
2170  }
2171 
2172  private void commit() throws SolrServerException {
2173  List<SolrInputDocument> clone;
2174  synchronized (bufferLock) {
2175  // Make a clone and release the lock, so that we don't
2176  // hold other ingest threads
2177  clone = buffer.stream().collect(toList());
2178  buffer.clear();
2179  }
2180 
2181  try {
2182  sendBufferedDocs(clone);
2183  } catch (KeywordSearchModuleException ex) {
2184  throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), ex);
2185  }
2186 
2187  try {
2188  //commit and block
2189  indexingClient.commit(true, true);
2190  } catch (Exception e) {
2191  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2192  logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
2193  throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
2194  }
2195  }
2196 
2197  private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
2198  String dataSourceId = Long.toString(dsObjId);
2199  String deleteQuery = "image_id:" + dataSourceId;
2200 
2201  queryClient.deleteByQuery(deleteQuery);
2202  }
2203 
2215  @NbBundle.Messages({
2216  "# {0} - Number of extracted terms",
2217  "ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms..."
2218  })
2219  private void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException {
2220 
2221  Files.deleteIfExists(outputFile);
2222  OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND };
2223 
2224  // step through the terms
2225  int termStep = 1000;
2226  long numExtractedTerms = 0;
2227  String firstTerm = "";
2228  while (true) {
2229  SolrQuery query = new SolrQuery();
2230  query.setRequestHandler("/terms");
2231  query.setTerms(true);
2232  query.setTermsLimit(termStep);
2233  query.setTermsLower(firstTerm);
2234  query.setTermsLowerInclusive(false);
2235 
2236  // Returned terms sorted by "index" order, which is the fastest way. Per Solr documentation:
2237  // "Retrieving terms in index order is very fast since the implementation directly uses Lucene‚Äôs TermEnum to iterate over the term dictionary."
2238  // All other sort criteria return very inconsistent and overlapping resuts.
2239  query.setTermsSortString("index");
2240 
2241  // "text" field is the schema field that we populate with (lowercased) terms
2242  query.addTermsField(Server.Schema.TEXT.toString());
2243  query.setTermsMinCount(0);
2244 
2245  // Unfortunatelly Solr "terms queries" do not support any filtering so we can't filter by data source this way.
2246  // query.addFilterQuery(Server.Schema.IMAGE_ID.toString() + ":" + dataSourceId);
2247 
2248  QueryRequest request = new QueryRequest(query);
2249  TermsResponse response = request.process(queryClient).getTermsResponse();
2250  List<Term> terms = response.getTerms(Server.Schema.TEXT.toString());
2251 
2252  if (terms == null || terms.isEmpty()) {
2253  numExtractedTerms += terms.size();
2254  progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms));
2255  break;
2256  }
2257 
2258  // set the first term for the next query
2259  firstTerm = terms.get(terms.size()-1).getTerm();
2260 
2261  List<String> listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList());
2262  Files.write(outputFile, listTerms, options);
2263 
2264  numExtractedTerms += termStep;
2265  progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms));
2266  }
2267  }
2268 
2277  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
2278 
2279  if (skipIndexing.get()) {
2280  return;
2281  }
2282 
2283  List<SolrInputDocument> clone;
2284  synchronized (bufferLock) {
2285  buffer.add(doc);
2286  // buffer documents if the buffer is not full
2287  if (buffer.size() < maxBufferSize) {
2288  return;
2289  }
2290 
2291  // Buffer is full. Make a clone and release the lock, so that we don't
2292  // hold other ingest threads
2293  clone = buffer.stream().collect(toList());
2294  buffer.clear();
2295  }
2296 
2297  // send the cloned list to Solr
2298  sendBufferedDocs(clone);
2299  }
2300 
2308  @NbBundle.Messages({
2309  "Collection.unableToIndexData.error=Unable to add data to text index. All future text indexing for the current case will be skipped.",
2310 
2311  })
2312  private void sendBufferedDocs(List<SolrInputDocument> docBuffer) throws KeywordSearchModuleException {
2313 
2314  if (docBuffer.isEmpty()) {
2315  return;
2316  }
2317 
2318  try {
2319  boolean success = true;
2320  for (int reTryAttempt = 0; reTryAttempt < NUM_BATCH_UPDATE_RETRIES; reTryAttempt++) {
2321  try {
2322  success = true;
2323  indexingClient.add(docBuffer);
2324  } catch (Exception ex) {
2325  success = false;
2326  if (reTryAttempt < NUM_BATCH_UPDATE_RETRIES - 1) {
2327  logger.log(Level.WARNING, "Unable to send document batch to Solr. Re-trying...", ex); //NON-NLS
2328  try {
2329  Thread.sleep(SLEEP_BETWEEN_RETRIES_MS);
2330  } catch (InterruptedException ignore) {
2331  throw new KeywordSearchModuleException(
2332  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
2333  }
2334  }
2335  }
2336  if (success) {
2337  numConsecutiveFailures.set(0);
2338  if (reTryAttempt > 0) {
2339  logger.log(Level.INFO, "Batch update suceeded after {0} re-try", reTryAttempt); //NON-NLS
2340  }
2341  return;
2342  }
2343  }
2344  // if we are here, it means all re-try attempts failed
2345  logger.log(Level.SEVERE, "Unable to send document batch to Solr. All re-try attempts failed!"); //NON-NLS
2346  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg")); //NON-NLS
2347  } catch (Exception ex) {
2348  // Solr throws a lot of unexpected exception types
2349  numConsecutiveFailures.incrementAndGet();
2350  logger.log(Level.SEVERE, "Could not add batched documents to index", ex); //NON-NLS
2351 
2352  // display message to user that that a document batch is missing from the index
2353  MessageNotifyUtil.Notify.error(
2354  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"),
2355  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"));
2356  throw new KeywordSearchModuleException(
2357  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
2358  } finally {
2359  if (numConsecutiveFailures.get() >= MAX_NUM_CONSECUTIVE_FAILURES) {
2360  // skip all future indexing
2361  skipIndexing.set(true);
2362  logger.log(Level.SEVERE, "Unable to add data to text index. All future text indexing for the current case will be skipped!"); //NON-NLS
2363 
2364  // display message to user that no more data will be added to the index
2365  MessageNotifyUtil.Notify.error(
2366  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"),
2367  Bundle.Collection_unableToIndexData_error());
2368  if (RuntimeProperties.runningWithGUI()) {
2369  MessageNotifyUtil.Message.error(Bundle.Collection_unableToIndexData_error());
2370  }
2371  }
2372  docBuffer.clear();
2373  }
2374  }
2375 
2386  private String getSolrContent(long contentID, int chunkID) {
2387  final SolrQuery q = new SolrQuery();
2388  q.setQuery("*:*");
2389  String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
2390  if (chunkID != 0) {
2391  filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
2392  }
2393  q.addFilterQuery(filterQuery);
2394  q.setFields(Schema.TEXT.toString());
2395  try {
2396  // Get the first result.
2397  SolrDocumentList solrDocuments = queryClient.query(q).getResults();
2398 
2399  if (!solrDocuments.isEmpty()) {
2400  SolrDocument solrDocument = solrDocuments.get(0);
2401  if (solrDocument != null) {
2402  java.util.Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
2403  if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
2404  {
2405  return fieldValues.toArray(new String[0])[0];
2406  } else // The indexed text for files has 2 values, the file name and the file content.
2407  // We return the file content value.
2408  {
2409  return fieldValues.toArray(new String[0])[1];
2410  }
2411  }
2412  }
2413  } catch (Exception ex) {
2414  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2415  logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", chunk id " + chunkID + ", query: " + filterQuery, ex); //NON-NLS
2416  return null;
2417  }
2418 
2419  return null;
2420  }
2421 
2422  synchronized void close() throws KeywordSearchModuleException {
2423  try {
2424 
2425  // stop the periodic batch update task. If the task is already running,
2426  // allow it to finish.
2427  ThreadUtils.shutDownTaskExecutor(periodicTasksExecutor);
2428 
2429  // We only unload cores for "single-user" cases.
2430  if (this.caseType == CaseType.MULTI_USER_CASE) {
2431  return;
2432  }
2433 
2434  CoreAdminRequest.unloadCore(this.name, localSolrServer);
2435  } catch (Exception ex) {
2436  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2437  throw new KeywordSearchModuleException(
2438  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
2439  } finally {
2440  try {
2441  queryClient.close();
2442  queryClient = null;
2443  indexingClient.close();
2444  indexingClient = null;
2445  } catch (IOException ex) {
2446  throw new KeywordSearchModuleException(
2447  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
2448  }
2449  }
2450  }
2451 
2461  private int queryNumIndexedFiles() throws SolrServerException, IOException {
2463  }
2464 
2474  private int queryNumIndexedChunks() throws SolrServerException, IOException {
2475  SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
2476  q.setRows(0);
2477  int numChunks = (int) query(q).getResults().getNumFound();
2478  return numChunks;
2479  }
2480 
2491  private int queryNumIndexedDocuments() throws SolrServerException, IOException {
2492  SolrQuery q = new SolrQuery("*:*");
2493  q.setRows(0);
2494  return (int) query(q).getResults().getNumFound();
2495  }
2496 
2506  private boolean queryIsIndexed(long contentID) throws SolrServerException, IOException {
2507  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
2508  SolrQuery q = new SolrQuery("*:*");
2509  q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
2510  //q.setFields(Server.Schema.ID.toString());
2511  q.setRows(0);
2512  return (int) query(q).getResults().getNumFound() != 0;
2513  }
2514 
2529  private int queryTotalNumFileChunks(long contentID) throws SolrServerException, IOException {
2530  final SolrQuery q = new SolrQuery();
2531  q.setQuery("*:*");
2532  String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
2533  q.addFilterQuery(filterQuery);
2534  q.setFields(Schema.NUM_CHUNKS.toString());
2535  try {
2536  SolrDocumentList solrDocuments = query(q).getResults();
2537  if (!solrDocuments.isEmpty()) {
2538  SolrDocument solrDocument = solrDocuments.get(0);
2539  if (solrDocument != null && !solrDocument.isEmpty()) {
2540  Object fieldValue = solrDocument.getFieldValue(Schema.NUM_CHUNKS.toString());
2541  return (Integer)fieldValue;
2542  }
2543  }
2544  } catch (Exception ex) {
2545  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2546  logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", query: " + filterQuery, ex); //NON-NLS
2547  return 0;
2548  }
2549  // File not indexed
2550  return 0;
2551  }
2552 
2564  int queryNumIndexedChunks(long contentID) throws SolrServerException, IOException {
2565  SolrQuery q = new SolrQuery(Server.Schema.ID + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID)) + Server.CHUNK_ID_SEPARATOR + "*");
2566  q.setRows(0);
2567  int numChunks = (int) query(q).getResults().getNumFound();
2568  return numChunks;
2569  }
2570  }
2571 
2572  class ServerAction extends AbstractAction {
2573 
2574  private static final long serialVersionUID = 1L;
2575 
2576  @Override
2577  public void actionPerformed(ActionEvent e) {
2578  logger.log(Level.INFO, e.paramString().trim());
2579  }
2580  }
2581 
2585  class SolrServerNoPortException extends SocketException {
2586 
2587  private static final long serialVersionUID = 1L;
2588 
2592  private final int port;
2593 
2594  SolrServerNoPortException(int port) {
2595  super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
2596  Server.PROPERTIES_CURRENT_SERVER_PORT));
2597  this.port = port;
2598  }
2599 
2600  int getPortNumber() {
2601  return port;
2602  }
2603  }
2604 }
static synchronized String getConfigSetting(String moduleName, String settingName)
String getSolrContent(final long objectID)
Definition: Server.java:1891
static synchronized long[] getJavaPIDs(String argsSubQuery)
final ReentrantReadWriteLock currentCoreLock
Definition: Server.java:275
ConcurrentUpdateSolrClient getConcurrentClient(String solrUrl)
Definition: Server.java:368
boolean queryIsFullyIndexed(long contentID)
Definition: Server.java:1652
static IndexingServerProperties getMultiUserServerProperties(String caseDirectory)
Definition: Server.java:1372
Collection openCore(Case theCase, Index index)
Definition: Server.java:1103
void connectToSolrServer(HttpSolrClient solrServer)
Definition: Server.java:1976
boolean coreIsLoaded(String coreName)
Definition: Server.java:1333
Process runLocalSolr8ControlCommand(List< String > solrArguments)
Definition: Server.java:498
static final int EMBEDDED_SERVER_RETRY_WAIT_SEC
Definition: Server.java:255
List< String > getSolrServerList(String host, String port)
Definition: Server.java:1989
static final int NUM_COLLECTION_CREATION_RETRIES
Definition: Server.java:253
void backupCollection(String collectionName, String backupName, String pathToBackupLocation)
Definition: Server.java:1292
void configureSolrConnection(Case theCase, Index index)
Definition: Server.java:620
CloudSolrClient getCloudSolrClient(String host, String port, String defaultCollectionName)
Definition: Server.java:385
boolean collectionExists(String collectionName)
Definition: Server.java:1217
static TimingMetric getTimingMetric(String name)
static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
Definition: Server.java:1448
void addServerActionListener(PropertyChangeListener l)
Definition: Server.java:413
static synchronized boolean settingExists(String moduleName, String settingName)
static final String HL_ANALYZE_CHARS_UNLIMITED
Definition: Server.java:229
String getSolrContent(final Content content)
Definition: Server.java:1846
boolean coreIndexFolderExists(String coreName)
Definition: Server.java:1350
void restoreCollection(String backupName, String restoreCollectionName, String pathToBackupLocation)
Definition: Server.java:1305
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
HttpSolrClient configureMultiUserConnection(Case theCase, Index index, String name)
Definition: Server.java:663
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
Definition: Server.java:238
InputStreamPrinterThread errorRedirectThread
Definition: Server.java:278
Process runLocalSolr4ControlCommand(List< String > solrArguments)
Definition: Server.java:548
List< String > getSolrServerList(HttpSolrClient solrServer)
Definition: Server.java:1994
QueryResponse query(SolrQuery sq)
Definition: Server.java:1714
static synchronized String getJavaPath()
static void submitTimingMetric(TimingMetric metric)
TermsResponse queryTerms(SolrQuery sq)
Definition: Server.java:1771
void createMultiUserCollection(String collectionName, int numShardsToUse)
Definition: Server.java:1262
String getSolrContent(final Content content, int chunkID)
Definition: Server.java:1870
String getSolrContent(final long objectID, final int chunkID)
Definition: Server.java:1913
HttpSolrClient getSolrClient(String solrUrl)
Definition: Server.java:359
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static String getChunkIdString(long parentID, int childID)
Definition: Server.java:1934
static boolean deleteDir(File dirPath)
Definition: FileUtil.java:47
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)
Definition: Server.java:1743

Copyright © 2012-2022 Basis Technology. Generated on: Tue Feb 6 2024
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.