ENH: New server property "SERVER_SIDE_SCRIPTING_BIN_DIRS" allows multiple directories.

This deprecates the server property "SERVER_SIDE_SCRIPTING_BIN_DIR"  The new server property accepts a comma or space separated list of directories.

CHANGELOG.md

@@ -9,10 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* New server property `SERVER_SIDE_SCRIPTING_BIN_DIRS` which accepts a comma or
+  space separated list as values. The server looks for scripts in all
+  directories in the order or the list and uses the first matching file.
+
 ### Changed
 
 ### Deprecated
 
+* `SERVER_SIDE_SCRIPTING_BIN_DIR` property is deprecated.
+  `SERVER_SIDE_SCRIPTING_BIN_DIRS` should be used instead (note the plural
+  form!)
+
 ### Removed
 
 ### Fixed

conf/core/server.conf

@@ -12,9 +12,10 @@ SERVER_NAME=CaosDB Server
 # The following paths are relative to the working directory of the server.
 # --------------------------------------------------
 
-# The location of the server side scripting binaries.
+# The location(s) of the server side scripting binaries.
 # Put your executable python scripts here, if they need to be called from the scripting API.
-SERVER_SIDE_SCRIPTING_BIN_DIR=./scripting/bin/
+# The value is a comma or space separated list or a single directory
+SERVER_SIDE_SCRIPTING_BIN_DIRS=./scripting/bin/
 
 # Working directory of the server side scripting API.
 # On execution of binaries and scripts the server will create a corresponding working directory in this folder.

src/main/java/org/caosdb/server/ServerProperties.java

@@ -117,6 +117,7 @@ public class ServerProperties extends Properties {
   public static final String KEY_SERVER_OWNER = "SERVER_OWNER";
 
   public static final String KEY_SERVER_SIDE_SCRIPTING_BIN_DIR = "SERVER_SIDE_SCRIPTING_BIN_DIR";
+  public static final String KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS = "SERVER_SIDE_SCRIPTING_BIN_DIRS";
   public static final String KEY_SERVER_SIDE_SCRIPTING_HOME_DIR = "SERVER_SIDE_SCRIPTING_HOME_DIR";
   public static final String KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR =
       "SERVER_SIDE_SCRIPTING_WORKING_DIR";

src/main/java/org/caosdb/server/scripting/ScriptingUtils.java

@@ -23,7 +23,8 @@
 package org.caosdb.server.scripting;
 
 import java.io.File;
-import java.nio.file.Path;
+import java.io.IOException;
+import java.util.ArrayList;
 import org.caosdb.server.CaosDBServer;
 import org.caosdb.server.ServerProperties;
 import org.caosdb.server.entity.Message;
@@ -33,25 +34,48 @@ import org.caosdb.server.utils.Utils;
 
 public class ScriptingUtils {
 
-  private File bin;
+  private File[] bin_dirs;
   private File working;
 
   public ScriptingUtils() {
-    this.bin =
-        new File(
-            CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR));
+    ArrayList<File> new_bin_dirs = new ArrayList<>();
+    String bin_dirs_str =
+        CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS);
+    if (bin_dirs_str == null) {
+      // fall-back for old server property
+      bin_dirs_str =
+          CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR);
+    }
+
+    // split and process
+    if (bin_dirs_str != null) {
+      for (String dir : bin_dirs_str.split("\\s+|\\s*,\\s*")) {
+
+        File bin;
+        try {
+          bin = new File(dir).getCanonicalFile();
+        } catch (IOException e) {
+          throw new ConfigurationException(
+              "Scripting bin dir `" + dir + "` cannot be resolved to a real path.");
+        }
+        if (!bin.exists()) {
+          bin.mkdirs();
+        }
+        if (!bin.isDirectory()) {
+          throw new ConfigurationException(
+              "The ServerProperty `"
+                  + ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS
+                  + "` must point to directories");
+        }
+        new_bin_dirs.add(bin);
+      }
+    }
+
+    bin_dirs = new_bin_dirs.toArray(new File[new_bin_dirs.size()]);
+
     this.working =
         new File(
             CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR));
-    if (!bin.exists()) {
-      bin.mkdirs();
-    }
-    if (!bin.isDirectory()) {
-      throw new ConfigurationException(
-          "The ServerProperty `"
-              + ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR
-              + "` must point to a directory");
-    }
 
     if (!working.exists()) {
       working.mkdirs();
@@ -64,21 +88,46 @@ public class ScriptingUtils {
     }
   }
 
-  public File getScriptFile(final String command) {
-    final Path script = bin.toPath().resolve(command);
-    return script.toFile();
-  }
+  /**
+   * Get the script file by the relative path.
+   *
+   * <p>Run through all registered bin_dirs and try to resolve the command relative to them. The
+   * first matching file is used. When it is not executable throw a
+   * SERVER_SIDE_SCRIPT_NOT_EXECUTABLE message. When no matching file exists throw a
+   * SERVER_SIDE_SCRIPT_DOES_NOT_EXIST message.
+   *
+   * @param command The relative path
+   * @return The script File object.
+   * @throws Message
+   */
+  public File getScriptFile(final String command) throws Message {
+    for (File bin_dir : bin_dirs) {
+      File script = bin_dir.toPath().resolve(command).toFile();
 
-  public void checkScriptExists(final String command) throws Message {
-    if (!getScriptFile(command).exists()) {
-      throw ServerMessages.SERVER_SIDE_SCRIPT_DOES_NOT_EXIST;
-    }
-  }
+      try {
+        script = script.getCanonicalFile();
+        if (!script.toPath().startsWith(bin_dir.toPath())) {
+          // not below the allowed directory tree
+          continue;
+        }
+      } catch (IOException e) {
+        // cannot be resolved to canonical file - we treat it as non-existing.
+        continue;
+      }
+
+      if (!script.exists()) {
+        // doesn't exist.
+        continue;
+      }
+
+      if (!script.canExecute()) {
+        throw ServerMessages.SERVER_SIDE_SCRIPT_NOT_EXECUTABLE;
+      }
 
-  public void checkScriptExecutable(final String command) throws Message {
-    if (!getScriptFile(command).canExecute()) {
-      throw ServerMessages.SERVER_SIDE_SCRIPT_NOT_EXECUTABLE;
+      // we found it!
+      return script;
     }
+    throw ServerMessages.SERVER_SIDE_SCRIPT_DOES_NOT_EXIST;
   }
 
   public File getTmpWorkingDir() {

src/main/java/org/caosdb/server/scripting/ServerSideScriptingCaller.java

@@ -54,6 +54,7 @@ public class ServerSideScriptingCaller {
   public static final Integer STARTED = -1;
   private final String[] commandLine;
   private final int timeoutMs;
+  private String absoluteScriptPath = null;
   private ScriptingUtils utils;
   private List<FileProperties> files;
   private File workingDir;
@@ -122,7 +123,7 @@ public class ServerSideScriptingCaller {
   /** Does some final preparation, then calls the script and cleans up. */
   public int invoke() throws Message {
     try {
-      checkCommandLine(commandLine);
+      this.absoluteScriptPath = getAbsoluteScriptPath(commandLine);
       try {
         createWorkingDir();
         putFilesInWorkingDir(files);
@@ -135,7 +136,8 @@ public class ServerSideScriptingCaller {
       }
 
       try {
-        return callScript();
+        code = callScript(this.absoluteScriptPath, this.commandLine, this.authToken, this.env);
+        return code;
       } catch (TimeoutException e) {
         throw ServerMessages.SERVER_SIDE_SCRIPT_TIMEOUT;
       } catch (final Throwable e) {
@@ -147,9 +149,17 @@ public class ServerSideScriptingCaller {
     }
   }
 
-  void checkCommandLine(String[] commandLine) throws Message {
-    utils.checkScriptExists(commandLine[0]);
-    utils.checkScriptExecutable(commandLine[0]);
+  /**
+   * Returns the absolute script path.
+   *
+   * <p>Throws Message when the script does not exist or when the script is not executable.
+   *
+   * @param commandLine
+   * @return The absolute script path.
+   * @throws Message
+   */
+  String getAbsoluteScriptPath(String[] commandLine) throws Message {
+    return utils.getScriptFile(commandLine[0]).getAbsolutePath();
   }
 
   void putFilesInWorkingDir(final Collection<FileProperties> files)
@@ -243,10 +253,6 @@ public class ServerSideScriptingCaller {
     if (pwd.exists()) FileUtils.forceDelete(pwd);
   }
 
-  String makeCallAbsolute(String call) {
-    return utils.getScriptFile(call).getAbsolutePath();
-  }
-
   /** @fixme Should be injected into environment instead. Will be changed in v0.4 of SSS-API */
   String[] injectAuthToken(String[] commandLine) {
     String[] newCommandLine = new String[commandLine.length + 1];
@@ -258,14 +264,31 @@ public class ServerSideScriptingCaller {
     return newCommandLine;
   }
 
-  int callScript() throws IOException, InterruptedException, TimeoutException {
+  /**
+   * Call the script.
+   *
+   * <p>The absoluteScriptPath is called with all remaining parameters from the commandLine arrays,
+   * an optional additional authToken and environment variables.
+   *
+   * @param absoluteScriptPath
+   * @param commandLine
+   * @param authToken
+   * @param env - environment variables
+   * @return the exit code of the script call
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws TimeoutException
+   */
+  int callScript(
+      String absoluteScriptPath, String[] commandLine, Object authToken, Map<String, String> env)
+      throws IOException, InterruptedException, TimeoutException {
     String[] effectiveCommandLine;
     if (authToken != null) {
-      effectiveCommandLine = injectAuthToken(getCommandLine());
+      effectiveCommandLine = injectAuthToken(commandLine);
     } else {
-      effectiveCommandLine = Arrays.copyOf(getCommandLine(), getCommandLine().length);
+      effectiveCommandLine = Arrays.copyOf(commandLine, commandLine.length);
     }
-    effectiveCommandLine[0] = makeCallAbsolute(effectiveCommandLine[0]);
+    effectiveCommandLine[0] = absoluteScriptPath;
     final ProcessBuilder pb = new ProcessBuilder(effectiveCommandLine);
 
     // inject environment variables
@@ -278,7 +301,7 @@ public class ServerSideScriptingCaller {
     pb.redirectError(Redirect.to(getStdErrFile()));
     pb.directory(getTmpWorkingDir());
 
-    code = STARTED;
+    int code = STARTED;
     final TimeoutProcess process = new TimeoutProcess(pb.start(), getTimeoutMs());
 
     code = process.waitFor();

src/test/java/org/caosdb/server/scripting/TestServerSideScriptingCaller.java

@@ -39,6 +39,7 @@ import org.apache.commons.io.FileUtils;
 import org.caosdb.CaosDBTestClass;
 import org.caosdb.server.CaosDBException;
 import org.caosdb.server.CaosDBServer;
+import org.caosdb.server.ServerProperties;
 import org.caosdb.server.database.exceptions.TransactionException;
 import org.caosdb.server.entity.FileProperties;
 import org.caosdb.server.entity.Message;
@@ -68,6 +69,10 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
 
   @BeforeClass
   public static void setupTestFolder() throws IOException {
+    CaosDBServer.getServerProperties()
+        .setProperty(
+            ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS, testFolder.getAbsolutePath());
+
     FileUtils.forceDeleteOnExit(testFolder);
     FileUtils.forceDeleteOnExit(testFile);
     FileUtils.forceDeleteOnExit(testExecutable);
@@ -305,7 +310,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
 
     testExeExit(1);
 
-    assertEquals(1, caller.callScript());
+    assertEquals(1, caller.callScript(cmd[0], cmd, null, emptyEnv));
   }
 
   @Test
@@ -319,7 +324,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
 
     testExeExit(0);
 
-    assertEquals(0, caller.callScript());
+    assertEquals(0, caller.callScript(cmd[0], cmd, null, emptyEnv));
   }
 
   @Test
@@ -387,7 +392,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
     testSleep(10);
 
     this.exception.expect(TimeoutException.class);
-    caller.callScript();
+    caller.callScript(cmd[0], cmd, null, emptyEnv);
   }
 
   @Test
@@ -418,7 +423,8 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
               throws FileNotFoundException, CaosDBException {}
 
           @Override
-          public int callScript() {
+          public int callScript(
+              String cmd, String[] cmdLine, Object authToken, Map<String, String> env) {
             return 0;
           }
 
@@ -456,7 +462,8 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
           }
 
           @Override
-          public int callScript() {
+          public int callScript(
+              String cmd, String[] cmdLine, Object authToken, Map<String, String> env) {
             return 0;
           }
 
@@ -496,7 +503,9 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
           public void createTmpHomeDir() throws Exception {}
 
           @Override
-          public int callScript() throws IOException {
+          public int callScript(
+              String cmd, String[] cmdLine, Object authToken, Map<String, String> env)
+              throws IOException {
             throw new IOException();
           }
 
@@ -532,7 +541,8 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
               throws FileNotFoundException, CaosDBException {}
 
           @Override
-          public int callScript() {
+          public int callScript(
+              String cmd, String[] cmdLine, Object authToken, Map<String, String> env) {
             return 0;
           }
 
@@ -562,7 +572,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
 
     testPrintArgsToStdErr();
 
-    caller.callScript();
+    caller.callScript(cmd[0], cmd, "authToken", emptyEnv);
 
     assertEquals(
         "--auth-token=authToken opt1 opt2\n", FileUtils.readFileToString(caller.getStdErrFile()));
@@ -579,7 +589,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
 
     testPrintArgsToStdOut();
 
-    caller.callScript();
+    caller.callScript(cmd[0], cmd, "authToken", emptyEnv);
 
     assertEquals(
         "--auth-token=authToken opt1 opt2\n", FileUtils.readFileToString(caller.getStdOutFile()));
@@ -599,7 +609,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
 
     preparePrintEnv("TEST");
 
-    caller.callScript();
+    caller.callScript(cmd[0], cmd, null, env);
 
     assertEquals("-testcontent-\n", caller.getStdOut());
     caller.cleanup();