ENH: server-side scripting

README_SETUP.md

@@ -10,6 +10,12 @@
 * >=Screen 4.01
 * >=MySQL 5.5 (better >=5.6) or >=MariaDB 10.1
 
+### Install the requirements on Debian
+On Debian, the required packages can be installed with:
+
+    apt-get install git make mariadb-server maven openjdk-8-jdk-headless \
+      python3-pip screen
+
 ## System
 
 * >=Linux 4.0.0, x86\_64, e.g. Ubuntu 14.04.1

conf/cache.ccf

+jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
+jcs.default.cacheattributes.MaxObjects=1000
+jcs.default.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
\ No newline at end of file

conf/logging.conf

+org.restlet.handlers = java.util.logging.ConsoleHandler
+caosdb.server.handlers = caosdb.server.logging.BackendLoggingHandler, java.util.logging.ConsoleHandler
+
+java.util.logging.ConsoleHandler.level = INFO
+caosdb.server.logging.BackendLogging.Handler.level = INFO
\ No newline at end of file

src/main/java/caosdb/server/CaosDBException.java

@@ -22,8 +22,8 @@
  */
 package caosdb.server;
 
-public class CaosDBException extends Exception {
-  /** */
+public class CaosDBException extends RuntimeException {
+
   private static final long serialVersionUID = 5317733089121727021L;
 
   public CaosDBException(final String string) {

src/main/java/caosdb/server/CaosDBServer.java

 /*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
+ * ** header v3.0 This file is a part of the CaosDB Project.
  *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2018 Research Group Biomedical Physics, Max-Planck-Institute for Dynamics and
+ * Self-Organization Göttingen
  *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
+ * This program is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+ * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Affero General Public License for more details.
  *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ * You should have received a copy of the GNU Affero General Public License along with this program.
+ * If not, see <https://www.gnu.org/licenses/>.
  *
  * ** end header
  */
@@ -48,6 +45,7 @@ import caosdb.server.resource.InfoResource;
 import caosdb.server.resource.LogoutResource;
 import caosdb.server.resource.PermissionRulesResource;
 import caosdb.server.resource.RolesResource;
+import caosdb.server.resource.ScriptingResource;
 import caosdb.server.resource.ServerLogsResource;
 import caosdb.server.resource.TestCaseFileSystemResource;
 import caosdb.server.resource.TestCaseResource;
@@ -533,6 +531,7 @@ public class CaosDBServer extends Application {
     // After authentication comes authorization...
     authenticator.setNext(authorizer);
 
+    protectedRouter.attach("/scripting", ScriptingResource.class);
     protectedRouter.attach("/Entities", EntityResource.class);
     protectedRouter.attach("/Entities/", EntityResource.class);
     protectedRouter.attach("/Entities/{specifier}", EntityResource.class);
@@ -582,12 +581,10 @@ public class CaosDBServer extends Application {
     protectedRouter.attachDefault(DefaultResource.class);
 
     /*
-     * Dirty Hack - no clean solution found yet (except for patching the
-     * restlet framework). The logging handler causes a NullPointerException
-     * when the Template.match method logs a warning. This warning
-     * (seemingly) always means that the RequestUri cannot be matched
-     * because its to long and causes a StackOverflow. Therefore we want to
-     * generate an HTTP 414 error.
+     * Dirty Hack - no clean solution found yet (except for patching the restlet framework). The
+     * logging handler causes a NullPointerException when the Template.match method logs a warning.
+     * This warning (seemingly) always means that the RequestUri cannot be matched because its to
+     * long and causes a StackOverflow. Therefore we want to generate an HTTP 414 error.
      */
 
     final Handler handler =
@@ -697,6 +694,20 @@ public class CaosDBServer extends Application {
   public static boolean isDebugMode() {
     return DEBUG_MODE;
   }
+
+  /**
+   * Set a server property to a new value. This might not have an immediate effect if classes did
+   * already read an older configuration and stick to that.
+   *
+   * @param key, the server property.
+   * @param value, the new value.
+   */
+  public static void setProperty(String key, String value) {
+    if (SERVER_PROPERTIES == null) {
+      SERVER_PROPERTIES = ServerProperties.initServerProperties();
+    }
+    SERVER_PROPERTIES.setProperty(key, value);
+  }
 }
 
 class CaosDBComponent extends Component {

src/main/java/caosdb/server/ServerProperties.java

@@ -113,6 +113,10 @@ public class ServerProperties extends Properties {
 
   public static final String KEY_SERVER_OWNER = "SERVER_OWNER";
 
+  public static final String KEY_SERVER_SIDE_SCRIPTING_BIN_DIR = "SERVER_SIDE_SCRIPTING_BIN_DIR";
+  public static final String KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR =
+      "SERVER_SIDE_SCRIPTING_WORKING_DIR";
+
   /**
    * This init_server_properties method reads the config file which contains key-value-pairs for
    * such variables like the user name of the database, the port the server will be listening on
@@ -123,6 +127,9 @@ public class ServerProperties extends Properties {
     final String basepath = System.getProperty("user.dir");
     serverProperties.setProperty(KEY_SERVER_OWNER, "");
     serverProperties.setProperty(KEY_SERVER_NAME, "CaosDB Server");
+    serverProperties.setProperty(KEY_SERVER_SIDE_SCRIPTING_BIN_DIR, basepath + "/scripting/bin/");
+    serverProperties.setProperty(
+        KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR, basepath + "/scripting/working/");
     serverProperties.setProperty(KEY_FILE_SYSTEM_ROOT, "CaosDBFileSystem/FileSystemRoot/");
     serverProperties.setProperty(KEY_DROP_OFF_BOX, "CaosDBFileSystem/DropOffBox/");
     serverProperties.setProperty(KEY_TMP_FILES, "CaosDBFileSystem/TMP/");
@@ -165,7 +172,7 @@ public class ServerProperties extends Properties {
     serverProperties.setProperty(KEY_ACTIVATION_TIMEOUT_MS, "604800000"); // 7days
 
     serverProperties.setProperty(KEY_MAIL_HANDLER_CLASS, "caosdb.server.utils.mail.ToFileHandler");
-    serverProperties.setProperty(KEY_MAIL_TO_FILE_HANDLER_LOC, "./OUTBOX");
+    serverProperties.setProperty(KEY_MAIL_TO_FILE_HANDLER_LOC, "./");
 
     serverProperties.setProperty(KEY_ADMIN_NAME, "CaosDB Admin");
     serverProperties.setProperty(KEY_ADMIN_EMAIL, "");

src/main/java/caosdb/server/entity/FileProperties.java

@@ -217,7 +217,7 @@ public class FileProperties {
   }
 
   private static Undoable delete(final File file)
-      throws IOException, InterruptedException, CaosDBException {
+      throws IOException, InterruptedException {
     if (file.getAbsolutePath().startsWith(FileSystem.getBasepath())) {
       final Undoable d;
       final File parent = file.getParentFile();

src/main/java/caosdb/server/resource/AbstractCaosDBServerResource.java

@@ -39,6 +39,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.NoSuchElementException;
+import java.util.logging.Level;
 import org.apache.commons.fileupload.FileUploadException;
 import org.apache.shiro.SecurityUtils;
 import org.apache.shiro.authc.AuthenticationException;
@@ -255,6 +256,10 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
     this.xslScript = s;
   }
 
+  protected JdomRepresentation ok(Element root) {
+    return ok(new Document(root));
+  }
+
   protected JdomRepresentation ok(final Document doc) {
     return new JdomRepresentation(doc, MediaType.TEXT_XML, "  ", getReference(), getXSLScript());
   }
@@ -313,6 +318,7 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
     } catch (final JDOMException e) {
       return noWellFormedNess();
     } catch (final Throwable e) {
+      getLogger().log(Level.SEVERE, "UNKNOWN ERROR", e);
       return error(ServerMessages.UNKNOWN_ERROR(getSRID()));
     }
   }
@@ -337,6 +343,14 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
     return this.flags;
   }
 
+  protected Element generateRootElement(Element... elements) {
+    Element root = generateRootElement();
+    for (Element e : elements) {
+      root.addContent(e);
+    }
+    return root;
+  }
+
   public static class XMLParser {
     private final LinkedList<SAXBuilder> pool = new LinkedList<SAXBuilder>();
     private final int max = 25;

src/main/java/caosdb/server/resource/ScriptingResource.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.resource;
+
+import caosdb.server.FileSystem;
+import caosdb.server.accessControl.Principal;
+import caosdb.server.accessControl.SessionToken;
+import caosdb.server.entity.FileProperties;
+import caosdb.server.entity.Message;
+import caosdb.server.scripting.CallerSerializer;
+import caosdb.server.scripting.ServerSideScriptingCaller;
+import caosdb.server.utils.Serializer;
+import caosdb.server.utils.ServerMessages;
+import caosdb.server.utils.Utils;
+import com.ibm.icu.text.Collator;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.commons.fileupload.FileItemIterator;
+import org.apache.commons.fileupload.FileItemStream;
+import org.apache.commons.fileupload.FileUploadException;
+import org.apache.commons.fileupload.disk.DiskFileItemFactory;
+import org.jdom2.Element;
+import org.restlet.data.CharacterSet;
+import org.restlet.data.Form;
+import org.restlet.data.MediaType;
+import org.restlet.data.Parameter;
+import org.restlet.data.Status;
+import org.restlet.engine.header.ContentType;
+import org.restlet.ext.fileupload.RestletFileUpload;
+import org.restlet.representation.Representation;
+
+public class ScriptingResource extends AbstractCaosDBServerResource {
+
+  private ServerSideScriptingCaller caller;
+  private Collection<FileProperties> deleteFiles = new LinkedList<>();
+
+  @Override
+  public Logger getLogger() {
+    return Logger.getLogger(this.getClass().getName());
+  }
+
+  public Element generateRootElement(ServerSideScriptingCaller caller) {
+    Serializer<ServerSideScriptingCaller, Element> xmlSerializer = new CallerSerializer();
+    Element callerElement = xmlSerializer.serialize(caller);
+
+    return generateRootElement(callerElement);
+  }
+
+  @Override
+  protected Representation httpPostInChildClass(Representation entity) throws Exception {
+
+    MediaType mediaType = entity.getMediaType();
+    try {
+      if (mediaType.equals(MediaType.MULTIPART_FORM_DATA, true)) {
+        handleMultiparts(entity);
+      } else if (mediaType.equals(MediaType.APPLICATION_WWW_FORM)) {
+        handleForm(new Form(entity));
+      } else {
+        getResponse().setStatus(Status.CLIENT_ERROR_UNSUPPORTED_MEDIA_TYPE);
+        return null;
+      }
+    } catch (Message m) {
+      return error(m, Status.valueOf(m.getCode()));
+    } finally {
+      deleteTmpFiles();
+    }
+    return ok(generateRootElement(this.caller));
+  }
+
+  private void deleteTmpFiles() {
+    for (FileProperties p : deleteFiles) {
+      try {
+      p.getFile().delete();
+      } catch (Exception t) {
+        if (getLogger().isLoggable(Level.WARNING)) {
+          getLogger().warning("Could not delete tmp file: " + p.getPath() + "\nException: " + t.toString());
+        }
+      }
+    }
+  }
+
+  public int handleMultiparts(Representation entity)
+      throws FileUploadException, IOException, NoSuchAlgorithmException, Message {
+    final DiskFileItemFactory factory = new DiskFileItemFactory();
+    factory.setSizeThreshold(1000240);
+    final RestletFileUpload upload = new RestletFileUpload(factory);
+    final FileItemIterator iter = upload.getItemIterator(entity);
+
+    List<FileProperties> files = new ArrayList<>();
+    Form form = new Form();
+
+    while (iter.hasNext()) {
+      FileItemStream item = iter.next();
+      if (item.isFormField()) {
+        // put plain text form field into the form
+        CharacterSet characterSet = ContentType.readCharacterSet(item.getContentType());
+        String value =
+            Utils.InputStream2String(
+                item.openStream(),
+                (characterSet != null ? characterSet.toString() : CharacterSet.UTF_8.toString()));
+        form.add(new Parameter(item.getFieldName(), value));
+      } else {
+        // -> this is a file, store in tmp dir
+        final FileProperties file = FileSystem.upload(item, this.getSRID());
+        deleteTmpFileAfterTermination(file);
+        file.setPath(item.getName());
+        file.setTmpIdentifyer(item.getFieldName());
+        files.add(file);
+        form.add(
+            new Parameter(
+                item.getFieldName(),
+                ServerSideScriptingCaller.UPLOAD_FILES_DIR + "/" + item.getName()));
+      }
+    }
+    return callScript(form, files);
+  }
+
+  private void deleteTmpFileAfterTermination(FileProperties file) {
+    deleteFiles.add(file);
+  }
+
+  public int handleForm(Form form) throws Message {
+    return callScript(form, null);
+  }
+
+  public List<String> form2CommandLine(Form form) throws Message {
+    ArrayList<String> commandLine = new ArrayList<>(form.size());
+    ArrayList<Parameter> positionalArgs = new ArrayList<>(form.size() - 1);
+
+    commandLine.add(""); // will be replaced by the "call" value
+
+    for (Parameter p : form) {
+      if (p.getName().startsWith("-p")) {
+        positionalArgs.add(p);
+      } else if (p.getName().startsWith("-O")) {
+        commandLine.add(String.format("--%s=%s", p.getName().substring(2), p.getValue()));
+      } else if (p.getName().equals("call")) {
+        commandLine.set(0, p.getValue());
+      }
+    }
+
+    // sort positional arguments.
+    positionalArgs.sort((o1, o2) -> Collator.getInstance().compare(o1.getName(), o2.getName()));
+    for (Parameter p : positionalArgs) {
+      commandLine.add(p.getValue());
+    }
+
+    if (commandLine.get(0).length() == 0) {
+      // first item still empty
+      throw ServerMessages.SERVER_SIDE_SCRIPT_MISSING_CALL;
+    }
+    return commandLine;
+  }
+
+  public int callScript(Form form, List<FileProperties> files) throws Message {
+    List<String> commandLine = form2CommandLine(form);
+    Integer timeoutMs = Integer.valueOf(form.getFirstValue("timeout", "-1"));
+    return callScript(commandLine, timeoutMs, files);
+  }
+
+  public int callScript(List<String> commandLine, Integer timeoutMs, List<FileProperties> files)
+      throws Message {
+    return callScript(commandLine, timeoutMs, files, generateAuthToken());
+  }
+
+  public Object generateAuthToken() {
+    return SessionToken.generate((Principal) getUser().getPrincipal(), null);
+  }
+
+  public int callScript(
+      List<String> commandLine, Integer timeoutMs, List<FileProperties> files, Object authToken)
+      throws Message {
+    caller =
+        new ServerSideScriptingCaller(
+            commandLine.toArray(new String[commandLine.size()]), timeoutMs, files, authToken);
+    return caller.invoke();
+  }
+
+  @Override
+  protected Representation httpGetInChildClass() throws Exception {
+    getResponse().setStatus(Status.CLIENT_ERROR_METHOD_NOT_ALLOWED);
+    return null;
+  }
+}

src/main/java/caosdb/server/scripting/CallerSerializer.java

+package caosdb.server.scripting;
+
+import caosdb.server.CaosDBException;
+import caosdb.server.utils.Serializer;
+import java.io.IOException;
+import org.jdom2.Element;
+
+public class CallerSerializer implements Serializer<ServerSideScriptingCaller, Element> {
+
+  @Override
+  public Element serialize(ServerSideScriptingCaller caller) {
+    Element command = new Element("call");
+    command.setText(String.join(" ", caller.getCommandLine()));
+
+    Element stdout = new Element("stdout");
+    try {
+      stdout.addContent(caller.getStdOut());
+    } catch (IOException e) {
+      throw new CaosDBException(e);
+    }
+
+    Element stderr = new Element("stderr");
+    try {
+      stderr.addContent(caller.getStdErr());
+    } catch (IOException e) {
+      throw new CaosDBException(e);
+    }
+
+    Element script = new Element("script");
+    script.setAttribute("code", caller.getCode().toString());
+    script.addContent(command);
+    script.addContent(stdout);
+    script.addContent(stderr);
+    return script;
+  }
+}

src/main/java/caosdb/server/scripting/ScriptingUtils.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.scripting;
+
+import caosdb.server.CaosDBServer;
+import caosdb.server.ServerProperties;
+import caosdb.server.entity.Message;
+import caosdb.server.utils.ConfigurationException;
+import caosdb.server.utils.ServerMessages;
+import caosdb.server.utils.Utils;
+import java.io.File;
+import java.nio.file.Path;
+
+public class ScriptingUtils {
+
+  private File bin;
+  private File working;
+
+  public ScriptingUtils() {
+    this.bin =
+        new File(
+            CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR));
+    this.working =
+        new File(
+            CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR));
+    if (!bin.exists()) {
+      bin.mkdirs();
+    }
+    if (!bin.isDirectory()) {
+      throw new ConfigurationException(
+          "The ServerProperty `"
+              + ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR
+              + "` must point to a directory");
+    }
+
+    if (!working.exists()) {
+      working.mkdirs();
+    }
+    if (!working.isDirectory()) {
+      throw new ConfigurationException(
+          "The ServerProperty `"
+              + ServerProperties.KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR
+              + "` must point to a directory");
+    }
+  }
+
+  public File getScriptFile(final String command) {
+    final Path script = bin.toPath().resolve(command);
+    return script.toFile();
+  }
+
+  public void checkScriptExists(final String command) throws Message {
+    if (!getScriptFile(command).exists()) {
+      throw ServerMessages.SERVER_SIDE_SCRIPT_DOES_NOT_EXIST;
+    }
+  }
+
+  public void checkScriptExecutable(final String command) throws Message {
+    if (!getScriptFile(command).canExecute()) {
+      throw ServerMessages.SERVER_SIDE_SCRIPT_NOT_EXECUTABLE;
+    }
+  }
+
+  public File getTmpWorkingDir() {
+    String uid = Utils.getUID();
+    return working.toPath().resolve(uid).toFile();
+  }
+
+  public File getStdOutFile(File workingDir) {
+    return workingDir.toPath().resolve(".STDOUT").toFile();
+  }
+
+  public File getStdErrFile(File workingDir) {
+    return workingDir.toPath().resolve(".STDERR").toFile();
+  }
+}

src/main/java/caosdb/server/scripting/ServerSideScriptingCaller.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.scripting;
+
+import caosdb.server.CaosDBException;
+import caosdb.server.entity.FileProperties;
+import caosdb.server.entity.Message;
+import caosdb.server.utils.ServerMessages;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.ProcessBuilder.Redirect;
+import java.util.Collection;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+
+public class ServerSideScriptingCaller {
+
+  public static final String UPLOAD_FILES_DIR = ".upload_files";
+  public static final Integer STARTED = -1;
+  private final String[] commandLine;
+  private final int timeoutMs;
+  private ScriptingUtils utils;
+  private List<FileProperties> files;
+  private File workingDir;
+  private Object authToken;
+  private File stdOutFile;
+  private File stdErrFile;
+  private String stdErr = null;
+  private String stdOut;
+  private Integer code = null;
+
+  public Integer getCode() {
+    return code;
+  }
+
+  File getUploadFilesDir() {
+    return getTmpWorkingDir().toPath().resolve(UPLOAD_FILES_DIR).toFile();
+  }
+
+  public ServerSideScriptingCaller(
+      String[] commandLine, Integer timeoutMs, List<FileProperties> files, Object authToken) {
+    this(commandLine, timeoutMs, files, authToken, new ScriptingUtils());
+  }
+
+  public ServerSideScriptingCaller(
+      String[] commandLine,
+      Integer timeoutMs,
+      List<FileProperties> files,
+      Object authToken,
+      ScriptingUtils utils) {
+    this(commandLine, timeoutMs, files, authToken, utils, utils.getTmpWorkingDir());
+  }
+
+  public ServerSideScriptingCaller(
+      String[] commandLine,
+      Integer timeoutMs,
+      List<FileProperties> files,
+      Object authToken,
+      ScriptingUtils utils,
+      File workingDir) {
+    this.utils = utils;
+    this.files = files;
+    this.commandLine = commandLine;
+    this.timeoutMs = timeoutMs;
+    this.authToken = authToken;
+    this.workingDir = workingDir;
+    this.stdOutFile = utils.getStdOutFile(workingDir);
+    this.stdErrFile = utils.getStdErrFile(workingDir);
+  }
+
+  public int invoke() throws Message {
+    try {
+      checkCommandLine(commandLine);
+      try {
+        createWorkingDir();
+        putFilesInWorkingDir(files);
+      } catch (final Exception e) {
+        e.printStackTrace();
+        throw ServerMessages.SERVER_SIDE_SCRIPT_SETUP_ERROR;
+      }
+
+      try {
+        return callScript();
+      } catch (TimeoutException e) {
+        throw ServerMessages.SERVER_SIDE_SCRIPT_TIMEOUT;
+      } catch (final Throwable e) {
+        e.printStackTrace();
+        throw ServerMessages.SERVER_SIDE_SCRIPT_ERROR;
+      }
+    } finally {
+      cleanup();
+    }
+  }
+
+  void checkCommandLine(String[] commandLine) throws Message {
+    utils.checkScriptExists(commandLine[0]);
+    utils.checkScriptExecutable(commandLine[0]);
+  }
+
+  void putFilesInWorkingDir(final Collection<FileProperties> files)
+      throws FileNotFoundException, CaosDBException {
+    if (files == null) {
+      return;
+    }
+
+    // create files dir
+    if (!getUploadFilesDir().mkdir()) {
+      throw new FileNotFoundException("Could not crete the FILE_UPLOAD_DIR.");
+    }
+    for (final FileProperties f : files) {
+      if (f.getPath() == null || f.getPath().isEmpty()) {
+        throw new CaosDBException("The path must not be null or empty!");
+      }
+      caosdb.server.utils.FileUtils.createSymlink(
+          getUploadFilesDir().toPath().resolve(f.getPath()).toFile(), f.getFile());
+    }
+  }
+
+  void createWorkingDir() throws Exception {
+    if (getTmpWorkingDir().exists()) {
+      throw new Exception("The working directory must be non-existing when the caller is invoked.");
+    }
+
+    // create working dir
+    getTmpWorkingDir().mkdirs();
+  }
+
+  void cleanup() {
+    // catch exception and throw afterwards,
+    IOException e1 = null;
+    try {
+      // cache script outputs
+      getStdErr();
+      getStdOut();
+    } catch (final IOException e2) {
+      e1 = e2;
+    }
+    try {
+      deleteWorkingDir(getTmpWorkingDir());
+    } catch (final IOException e2) {
+      if (e1 == null) {
+        e1 = e2;
+      } else {
+        e1.addSuppressed(e2);
+      }
+    }
+    if (e1 != null) throw new RuntimeException("Cleanup failed.", e1);
+  }
+
+  void deleteWorkingDir(final File pwd) throws IOException {
+    if (pwd.exists()) FileUtils.forceDelete(pwd);
+  }
+
+  String makeCallAbsolute(String call) {
+    return utils.getScriptFile(call).getAbsolutePath();
+  }
+
+  String[] injectAuthToken(String[] commandLine) {
+    String[] newCommandLine = new String[commandLine.length + 1];
+    newCommandLine[0] = commandLine[0];
+    newCommandLine[1] = "--auth-token=" + authToken.toString();
+    for (int i = 2; i < newCommandLine.length; i++) {
+      newCommandLine[i] = commandLine[i - 1];
+    }
+    return newCommandLine;
+  }
+
+  int callScript() throws IOException, InterruptedException, TimeoutException {
+    String[] effectiveCommandLine = injectAuthToken(getCommandLine());
+    effectiveCommandLine[0] = makeCallAbsolute(effectiveCommandLine[0]);
+    final ProcessBuilder pb = new ProcessBuilder(effectiveCommandLine);
+    pb.redirectOutput(Redirect.to(getStdOutFile()));
+    pb.redirectError(Redirect.to(getStdErrFile()));
+    pb.directory(getTmpWorkingDir());
+
+    code = STARTED;
+    final TimeoutProcess process = new TimeoutProcess(pb.start(), getTimeoutMs());
+
+    code = process.waitFor();
+    return code;
+  }
+
+  public File getStdOutFile() {
+    return stdOutFile;
+  }
+
+  public File getStdErrFile() {
+    return stdErrFile;
+  }
+
+  public String[] getCommandLine() {
+    return this.commandLine;
+  }
+
+  File getTmpWorkingDir() {
+    return this.workingDir;
+  }
+
+  int getTimeoutMs() {
+    return this.timeoutMs;
+  }
+
+  public String getStdErr() throws IOException {
+    if (stdErr == null && getStdErrFile().exists()) {
+      stdErr = FileUtils.readFileToString(getStdErrFile(), "UTF-8");
+    }
+    return stdErr;
+  }
+
+  public String getStdOut() throws IOException {
+    if (stdOut == null && getStdOutFile().exists()) {
+      stdOut = FileUtils.readFileToString(getStdOutFile(), "UTF-8");
+    }
+    return stdOut;
+  }
+}

src/main/java/caosdb/server/scripting/TimeoutException.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.scripting;
+
+public class TimeoutException extends Exception {
+
+  public TimeoutException(final String string) {
+    super(string);
+  }
+
+  private static final long serialVersionUID = 4871406372815136756L;
+}

src/main/java/caosdb/server/scripting/TimeoutProcess.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.scripting;
+
+import java.io.InputStream;
+
+public class TimeoutProcess {
+
+  private final Process process;
+  private boolean wasTimeouted = false;
+
+  public TimeoutProcess(final Process process, final int ms) {
+    this.process = process;
+    if (ms > 0) {
+      watchTimeout(this, ms).start();
+    }
+  }
+
+  public static Thread watchTimeout(final TimeoutProcess p, final int ms) {
+    return new Thread(
+        new Runnable() {
+
+          @Override
+          public void run() {
+            try {
+              Thread.sleep(ms);
+            } catch (final InterruptedException e) {
+              e.printStackTrace();
+            }
+            p.timeout();
+          }
+        },
+        "Timeout " + p.toString());
+  }
+
+  protected void timeout() {
+    if (this.process.isAlive()) {
+      this.wasTimeouted = true;
+      this.process.destroyForcibly();
+    }
+  }
+
+  public boolean wasTimeouted() {
+    return this.wasTimeouted;
+  }
+
+  public int waitFor() throws InterruptedException, TimeoutException {
+    int code = this.process.waitFor();
+    if (wasTimeouted) {
+      throw new TimeoutException("The process was terminated due to a timeout.");
+    } else {
+      return code;
+    }
+  }
+
+  public InputStream getErrorStream() {
+    return this.process.getErrorStream();
+  }
+
+  public int exitValue() {
+    return this.process.exitValue();
+  }
+}

src/main/java/caosdb/server/utils/ConfigurationException.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.utils;
+
+public class ConfigurationException extends RuntimeException {
+
+  private static final long serialVersionUID = -8445574584694720914L;
+
+  public ConfigurationException(String reason) {
+    super(reason);
+  }
+}

src/main/java/caosdb/server/utils/FileUtils.java

@@ -460,6 +460,9 @@ public class FileUtils {
   }
 
   public static File createSymlink(final File link, final File target) {
+    if (target == null) {
+      throw new NullPointerException("target was null.");
+    }
     if (!target.exists()) {
       throw new TransactionException("target does not exist.");
     }

src/main/java/caosdb/server/utils/Serializer.java

+package caosdb.server.utils;
+
+public interface Serializer<T, S> {
+
+  public S serialize(T object);
+}

src/main/java/caosdb/server/utils/ServerMessages.java

 /*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
+ * ** header v3.0 This file is a part of the CaosDB Project.
  *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2018 Research Group Biomedical Physics, Max-Planck-Institute for Dynamics and
+ * Self-Organization Göttingen
  *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
+ * This program is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+ * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Affero General Public License for more details.
  *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ * You should have received a copy of the GNU Affero General Public License along with this program.
+ * If not, see <https://www.gnu.org/licenses/>.
  *
  * ** end header
  */
@@ -399,4 +396,26 @@ public class ServerMessages {
 
   public static final Message CANNOT_PARSE_UNIT =
       new Message(MessageType.Error, 0, "This unit cannot be parsed.");
+
+  public static final Message SERVER_SIDE_SCRIPT_DOES_NOT_EXIST =
+      new Message(
+          MessageType.Error, 404, "This server-side script does not exist. Did you install it?");
+
+  public static final Message SERVER_SIDE_SCRIPT_NOT_EXECUTABLE =
+      new Message(MessageType.Error, 400, "This server-side script is not executable.");
+
+  public static final Message SERVER_SIDE_SCRIPT_ERROR =
+      new Message(MessageType.Error, 500, "The invocation of this server-side script failed.");
+
+  public static final Message SERVER_SIDE_SCRIPT_SETUP_ERROR =
+      new Message(
+          MessageType.Error,
+          500,
+          "The setup routine for the server-side script failed. This might indicate a misconfiguration of the server. Please contact the administrator.");
+
+  public static final Message SERVER_SIDE_SCRIPT_TIMEOUT =
+      new Message(MessageType.Error, 400, "This server-side script did not finish in time.");
+
+  public static final Message SERVER_SIDE_SCRIPT_MISSING_CALL =
+      new Message(MessageType.Error, 400, "You must specify the `call` field.");
 }

src/main/java/caosdb/server/utils/Utils.java

@@ -165,7 +165,11 @@ public class Utils {
   }
 
   public static String InputStream2String(final InputStream is) {
-    try (Scanner s = new Scanner(is)) {
+    return InputStream2String(is, "UTF-8");
+  }
+
+  public static String InputStream2String(final InputStream is, String charset) {
+    try (Scanner s = new Scanner(is, charset)) {
       s.useDelimiter("\\A");
       return s.hasNext() ? s.next() : "";
     }

src/main/java/caosdb/server/utils/fsm/ActionNotAllowedException.java

+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.utils.fsm;
+
+public class ActionNotAllowedException extends RuntimeException {
+
+  private final String action;
+
+  public ActionNotAllowedException(final String action) {
+    this.action = action;
+  }
+
+  private static final long serialVersionUID = -7723481489788838572L;
+
+  @Override
+  public String getMessage() {
+    return "The action `" + this.action + "` is not allowed in the current state.";
+  }
+}