From b881eeb6c67a6b13e96a07dabc56565a3745973c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Thu, 5 Nov 2020 13:25:14 +0000
Subject: [PATCH] F table bottomline

---
 .gitignore                                    |   1 +
 .gitlab-ci.yml                                |   8 +
 CHANGELOG.md                                  |   2 +
 README_SETUP.md                               |   4 +
 build.properties.d/00_default.properties      |   3 +-
 install-sss.sh                                |  13 ++
 makefile                                      |  28 +++-
 src/core/js/ext_bottom_line.js                |  30 +++-
 src/core/js/ext_table_preview.js              |  97 ++++++++++++
 src/core/xsl/main.xsl                         |   5 +
 .../ext_table_preview/pandas_table_preview.py | 145 ++++++++++++++++++
 test/core/index.html                          |   1 +
 test/core/js/modules/ext_bottom_line.js.js    |   2 +-
 test/docker/Dockerfile                        |   7 +-
 .../pandas_table_preview.cpython-37.pyc       | Bin 0 -> 3257 bytes
 ..._table_preview.cpython-37-pytest-6.0.2.pyc | Bin 0 -> 4060 bytes
 .../ext_table_preview/data/bad.csv            | Bin 0 -> 7 bytes
 .../ext_table_preview/data/bad.tsv            | Bin 0 -> 7 bytes
 .../ext_table_preview/data/bad.xls            | Bin 0 -> 66 bytes
 .../ext_table_preview/data/bad.xlsx           | Bin 0 -> 66 bytes
 .../ext_table_preview/data/server_error.csv   |   1 +
 .../ext_table_preview/data/test.csv           |  12 ++
 .../ext_table_preview/data/test.tsv           |  12 ++
 .../ext_table_preview/data/test.xls           | Bin 0 -> 5632 bytes
 .../ext_table_preview/data/test.xlsx          | Bin 0 -> 4801 bytes
 .../ext_table_preview/data/xss_attack.csv     |   8 +
 .../ext_table_preview/pandas_table_preview.py |   1 +
 .../ext_table_preview/requirements.txt        |   3 +
 .../test_pandas_table_preview.py              |  74 +++++++++
 29 files changed, 441 insertions(+), 16 deletions(-)
 create mode 100755 install-sss.sh
 create mode 100644 src/core/js/ext_table_preview.js
 create mode 100755 src/server_side_scripting/ext_table_preview/pandas_table_preview.py
 create mode 100644 test/server_side_scripting/ext_table_preview/__pycache__/pandas_table_preview.cpython-37.pyc
 create mode 100644 test/server_side_scripting/ext_table_preview/__pycache__/test_pandas_table_preview.cpython-37-pytest-6.0.2.pyc
 create mode 100644 test/server_side_scripting/ext_table_preview/data/bad.csv
 create mode 100644 test/server_side_scripting/ext_table_preview/data/bad.tsv
 create mode 100644 test/server_side_scripting/ext_table_preview/data/bad.xls
 create mode 100644 test/server_side_scripting/ext_table_preview/data/bad.xlsx
 create mode 100644 test/server_side_scripting/ext_table_preview/data/server_error.csv
 create mode 100644 test/server_side_scripting/ext_table_preview/data/test.csv
 create mode 100644 test/server_side_scripting/ext_table_preview/data/test.tsv
 create mode 100644 test/server_side_scripting/ext_table_preview/data/test.xls
 create mode 100644 test/server_side_scripting/ext_table_preview/data/test.xlsx
 create mode 100644 test/server_side_scripting/ext_table_preview/data/xss_attack.csv
 create mode 120000 test/server_side_scripting/ext_table_preview/pandas_table_preview.py
 create mode 100644 test/server_side_scripting/ext_table_preview/requirements.txt
 create mode 100644 test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py

diff --git a/.gitignore b/.gitignore
index cc9336b6..ddfb9ac0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
 
 # the build dir
 /public
+/sss_bin
 
 # screen logs
 screenlog.*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 548f8645..2a014f45 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -58,6 +58,14 @@ test:
   script:
     - make run-qunit
 
+test-server-side-scripting:
+  timeout: 10 minutes
+  tags: [ docker ]
+  stage: test
+  script:
+    - whereis pytest pytest3 py.test pytest-3 py.test-3
+    - make test-sss
+
 # Trigger building of server image and integration tests
 trigger_build:
   timeout: 15 minutes
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5ef6c2a1..a30e209b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added (for new features, dependecies etc.)
+- table previews in the bottom line module
+
 - added preview for tif images
 
 * new function `form_elements.make_alert` which generates a proceed/cancel
diff --git a/README_SETUP.md b/README_SETUP.md
index 4706efe2..f50ed540 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -53,6 +53,10 @@ information.
 
 * Run `make install` to compile/copy the webinterface to a newly created
   `public` folder.
+* Also, `make install` wil copy the scripts from `src/server_side_scripting/`
+  to `sss_bin/`. If you want to make the server-side scripts callable for the
+  server as server-side scripts you need to include the `sss_bin/` directory
+  into the server property `SERVER_SIDE_SCRIPTING_BIN_DIRS`.
 
 # Test
 
diff --git a/build.properties.d/00_default.properties b/build.properties.d/00_default.properties
index 33567a80..536a2925 100644
--- a/build.properties.d/00_default.properties
+++ b/build.properties.d/00_default.properties
@@ -45,7 +45,8 @@ BUILD_MODULE_EXT_PREVIEW=ENABLED
 BUILD_MODULE_EXT_RESOLVE_REFERENCES=ENABLED
 BUILD_MODULE_EXT_SSS_MARKDOWN=DISABLED
 BUILD_MODULE_EXT_TRIGGER_CRAWLER_FORM=DISABLED
-BUILD_MODULE_EXT_BOTTOM_LINE=DISABLED
+BUILD_MODULE_EXT_BOTTOM_LINE=ENABLED
+BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW=DISABLED
 BUILD_MODULE_EXT_BOTTOM_LINE_TIFF_PREVIEW=DISABLED
 
 ##############################################################################
diff --git a/install-sss.sh b/install-sss.sh
new file mode 100755
index 00000000..432e1ce9
--- /dev/null
+++ b/install-sss.sh
@@ -0,0 +1,13 @@
+SRC_DIR=$1
+INSTALL_DIR=$2
+
+mkdir -p $INSTALL_DIR
+
+# from here on do your module-wise installing
+
+# ext_table_preview
+if [ "${BUILD_MODULE_EXT_TABLE_PREVIEW}" == "ENABLED" ]; then
+    mkdir -p $INSTALL_DIR/ext_table_preview
+    cp $SRC_DIR/ext_table_preview/*.py $INSTALL_DIR/ext_table_preview/
+    echo "installed all server-side scripts for ext_table_preview"
+fi
diff --git a/makefile b/makefile
index 41c7cf51..f35bc3c3 100644
--- a/makefile
+++ b/makefile
@@ -33,13 +33,16 @@ SQ=\'
 ROOT_DIR = $(abspath .)
 MISC_DIR = $(abspath misc)
 PUBLIC_DIR = $(abspath public)
+SSS_BIN_DIR = $(abspath sss_bin)
 CONF_CORE_DIR = $(abspath conf/core)
 CONF_EXT_DIR = $(abspath conf/ext)
 SRC_CORE_DIR = $(abspath src/core)
 SRC_EXT_DIR = $(abspath src/ext)
+SRC_SSS_DIR = $(abspath src/server_side_scripting)
 LIBS_DIR = $(abspath libs)
 TEST_CORE_DIR = $(abspath test/core/)
 TEST_EXT_DIR = $(abspath test/ext)
+TEST_SSS_DIR =$(abspath test/server_side_scripting)
 LIBS = fonts css/bootstrap.css js/bootstrap.js js/state-machine.js js/jquery.js js/showdown.js js/dropzone.js css/dropzone.css js/loglevel.js js/leaflet.js css/leaflet.css css/images js/leaflet-latlng-graticule.js js/proj4.js js/proj4leaflet.js js/leaflet-coordinates.js css/leaflet-coordinates.css js/leaflet-graticule.js js/bootstrap-select.js css/bootstrap-select.css js/bootstrap-autocomplete.min.js js/plotly.js js/pako.js js/utif.js
 
 TEST_LIBS = $(LIBS) js/qunit.js css/qunit.css $(subst $(TEST_CORE_DIR)/,,$(shell find $(TEST_CORE_DIR)/))
@@ -49,9 +52,9 @@ LIBS_SUBDIRS = $(addprefix $(LIBS_DIR)/, js css fonts)
 
 ALL: install
 
-install: clean cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) build_properties merge_xsl
+install: clean install-sss cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) build_properties merge_xsl
 
-test: clean cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) build_properties merge_xsl
+test: clean install-sss cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) build_properties merge_xsl
 	@for f in $(shell find $(TEST_EXT_DIR) -type f -iname *.js) ; do \
 		sed -i "/EXTENSIONS/a \<script src=\"$${f#$(TEST_EXT_DIR)/}\" ></script>" $(PUBLIC_DIR)/index.html ; \
 		echo include $$f; \
@@ -129,6 +132,18 @@ run-qunit: test
 		exit 1; \
 	fi
 
+install-sss:
+	@set -a -e ; \
+	pushd build.properties.files ; \
+	for f in ../build.properties.d/* ; do source "$$f" ; done ; \
+	popd ; \
+	./install-sss.sh $(SRC_SSS_DIR) $(SSS_BIN_DIR)
+
+PYTEST ?= pytest-3
+test-sss: install-sss
+	$(PYTEST) -vv $(TEST_SSS_DIR)
+
+
 CMD_COPY_EXT_FILES = cp -i -r -L
 cp-ext:
 	# TODO FIXME Base path for not-XSL-expanded files
@@ -277,6 +292,7 @@ $(addprefix $(LIBS_DIR)/, js css):
 
 .PHONY: clean
 clean:
+	$(RM) -r $(SSS_BIN_DIR)
 	$(RM) -r $(PUBLIC_DIR)
 	for f in $(LIBS_SUBDIRS); do unlink $$f || $(RM) -r $$f || true; done
 	for f in $(patsubst %.zip,%/,$(LIBS_ZIP)); do $(RM) -r $$f; done
@@ -287,11 +303,7 @@ unzip:
 	for f in $(LIBS_ZIP); do unzip -q -o -d libs $$f; done
 
 
-PYLINT = pylint3 -d all -e E,F
+PYLINT ?= pylint3
 PYTHON_FILES = $(subst $(ROOT_DIR)/,,$(shell find $(ROOT_DIR)/ -iname "*.py"))
 pylint: $(PYTHON_FILES)
-	for f in $(PYTHON_FILES); do $(PYLINT) $$f || exit 1; done
-
-PYLINT_LOCAL = /usr/bin/pylint3 -d all -e E,F
-pylint-local: $(PYTHON_FILES)
-	for f in $(PYTHON_FILES); do $(PYLINT_LOCAL) $$f || exit 1; done
+	for f in $(PYTHON_FILES); do $(PYLINT) -d all -e E,F $$f || exit 1; done
diff --git a/src/core/js/ext_bottom_line.js b/src/core/js/ext_bottom_line.js
index 4f563f04..ff448f19 100644
--- a/src/core/js/ext_bottom_line.js
+++ b/src/core/js/ext_bottom_line.js
@@ -43,8 +43,9 @@
  * @requires getEntityPath (function from caosdb.js)
  * @requires connection (module from webcaosdb.js)
  * @requires UTIF (from utif.js library)
+ * @requires ext_table_preview (module from ext_table_preview.js)
  */
-var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntityPath, connection, UTIF) {
+var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntityPath, connection, UTIF, ext_table_preview) {
 
     /**
      * @property {string|function} create - a function with one parameter
@@ -131,6 +132,22 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit
         }
     }
 
+    const BottomLineWarning = function (arg) {
+        this._is_bottom_line_error = true;
+
+        if (arg.message) {
+            // arg is an Error object
+            this.message = arg.message;
+            this.stack = arg.stack;
+        } else {
+            this.message = arg;
+        }
+
+        this.to_html = function() {
+            return $(`<div>${this.message}<div>`)[0];
+        }
+    }
+
     /**
      * Create a preview for tiff files.
      *
@@ -210,11 +227,15 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit
             is_applicable: (entity) => _path_has_file_extension(
                 entity, ["mp4", "mov", "webm"]),
             create: _create_video_preview,
+        }, { // tables
+            id: "_default_creators.table_preview",
+            is_applicable: (e) => ext_table_preview.is_table(e),
+            create: (e) => ext_table_preview.get_preview(e),
         }, { // fallback
             id: "_default_creators.fallback",
             is_applicable: (entity) => true,
             create: (entity) => fallback_preview,
-        },
+        }
 
     ];
 
@@ -539,8 +560,11 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit
         _css_class_preview_container_button,
         _css_class_preview_container_resolvable,
         BottomLineError: BottomLineError,
+        BottomLineWarning: BottomLineWarning,
     }
-}($, log.getLogger("ext_bottom_line"), resolve_references.is_in_viewport_vertically, load_config, getEntityPath, connection, UTIF);
+}($, log.getLogger("ext_bottom_line"),
+  resolve_references.is_in_viewport_vertically, load_config, getEntityPath,
+  connection, UTIF, ext_table_preview);
 
 
 /**
diff --git a/src/core/js/ext_table_preview.js b/src/core/js/ext_table_preview.js
new file mode 100644
index 00000000..1d9da6fa
--- /dev/null
+++ b/src/core/js/ext_table_preview.js
@@ -0,0 +1,97 @@
+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+
+'use strict';
+
+/**
+ * The ext_table_preview module provides a very basic preview for table files.
+ *
+ * The preview is generated using a server side script.
+ *
+ * @module ext_table_preview 
+ * @version 0.1
+ *
+ * @requires jQuery
+ * @requires log
+ * @requires getEntityPath
+ * @requires getEntityID
+ * @requires markdown
+ */
+var ext_table_preview  = function ($, logger, connection, getEntityPath, getEntityID, markdown) {
+
+    const get_preview = async function (entity) {
+        try {
+            const script_result = await connection.runScript("ext_table_preview/pandas_table_preview.py",
+                {"-p0": getEntityID(entity)}
+            );
+
+            const code = script_result.getElementsByTagName("script")[0].getAttribute("code");
+            if (parseInt(code) > 1) {
+                return script_result.getElementsByTagName("stderr")[0]
+            } else if (parseInt(code) != 0) {
+                throw ("An error occurred during execution of the server-side "
+                    + "script:\n"
+                    + script_result.getElementsByTagName("stderr")[0]);
+            } else {
+                const tablecontent = script_result.getElementsByTagName("stdout")[0];
+                const unformatted = markdown.textToHtml(tablecontent.textContent)
+                const formatted = $('<div class="table-responsive"/>').append(unformatted);
+                formatted.find("table").addClass("table table-bordered table-condensed").removeAttr("border");
+                return formatted[0];
+            }
+        } catch (err) {
+            if (err.message && err.message.indexOf && err.message.indexOf("HTTP status 403") > -1) {
+                throw new ext_bottom_line.BottomLineWarning("You are not allowed to generate the table preview. Please log in.");
+            } else {
+                throw err;
+            }
+        }
+    };
+
+    const is_table = function (entity) {
+        const path = getEntityPath(entity);
+        return path && (path.toLowerCase().endsWith('.xls') 
+            || path.toLowerCase().endsWith('.xlsx') 
+            || path.toLowerCase().endsWith('.csv') 
+            || path.toLowerCase().endsWith('.tsv'));
+    };
+
+    const init = function () {
+        // only enable when init is being called
+        ext_table_preview.is_table = is_table;
+    };
+
+    return {
+        init: init,
+        get_preview: get_preview,
+        is_table: () => false,
+    };
+
+}($, log.getLogger("ext_table_preview"), connection, getEntityPath, getEntityID, markdown);
+
+// this will be replaced by require.js in the future.
+$(document).ready(function () {
+    if ("${BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW}" == "ENABLED") {
+        caosdb_modules.register(ext_table_preview);
+    }
+});
diff --git a/src/core/xsl/main.xsl b/src/core/xsl/main.xsl
index 91f75733..b915f958 100644
--- a/src/core/xsl/main.xsl
+++ b/src/core/xsl/main.xsl
@@ -180,6 +180,11 @@
         <xsl:value-of select="concat($basepath,'webinterface/${BUILD_NUMBER}/js/ext_references.js')"/>
       </xsl:attribute>
     </xsl:element>
+    <xsl:element name="script">
+      <xsl:attribute name="src">
+        <xsl:value-of select="concat($basepath,'webinterface/${BUILD_NUMBER}/js/ext_table_preview.js')"/>
+      </xsl:attribute>
+    </xsl:element>
     <xsl:element name="script">
       <xsl:attribute name="src">
         <xsl:value-of select="concat($basepath,'webinterface/${BUILD_NUMBER}/js/ext_xls_download.js')"/>
diff --git a/src/server_side_scripting/ext_table_preview/pandas_table_preview.py b/src/server_side_scripting/ext_table_preview/pandas_table_preview.py
new file mode 100755
index 00000000..c0659d9b
--- /dev/null
+++ b/src/server_side_scripting/ext_table_preview/pandas_table_preview.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+This script tries to read typical table data files (.csv etc.) with pandas and
+creates a html (partial) representation of the table.
+"""
+
+import logging
+import os
+import sys
+from datetime import datetime
+
+import caosdb as db
+import pandas as pd
+from caosadvancedtools.serverside.helper import get_argument_parser
+from caosadvancedtools.serverside.logging import configure_server_side_logging
+
+MAXIMUMFILESIZE = 1e8
+VALID_ENDINGS = [".csv", ".tsv", ".xls", ".xlsx"]
+
+
+def get_file(eid):
+    """ retrieves the file entity from caosdb """
+    try:
+        fi = db.File(id=eid)
+        fi.retrieve()
+    except db.exceptions.EntityDoesNotExistError:
+        print("Cannot create preview for Entity with ID={}, because it seems"
+              "not to exist.".format(eid), file=sys.stderr)
+        sys.exit(1)
+
+    return fi
+
+
+def size_is_ok(fi):
+    """ show previews only for files that are not too large """
+
+    return fi.size <= MAXIMUMFILESIZE
+
+
+def get_ending(fipath):
+    """ return which of the valid endings (tsv etc.) is the one present"""
+
+    for end in VALID_ENDINGS:
+        if fipath.lower().endswith(end):
+            return end
+
+    return None
+
+
+def ending_is_valid(fipath):
+    """ return whether the ending indicates a file type that can be treated"""
+
+    return get_ending(fipath) is not None
+
+
+def read_file(fipath, ftype):
+    """ tries to read the provided file """
+
+    try:
+        if ftype in [".xls", ".xlsx"]:
+            df = pd.read_excel(fipath)
+        elif ftype == ".tsv":
+            df = pd.read_csv(fipath, sep="\t", comment="#")
+        elif ftype == ".csv":
+            df = pd.read_csv(fipath, comment="#")
+        else:
+            print("File type unknown: {}".format(ftype))
+            raise RuntimeError("")
+    except Exception:
+        raise ValueError()
+
+    return df
+
+
+def create_table_preview(fi):
+    if not ending_is_valid(fi.path):
+        print("Cannot create preview for Entity with ID={}, because download"
+              "failed.".format(entity_id), file=sys.stderr)
+        sys.exit(5)
+
+    ending = get_ending(fi.path)
+
+    if not size_is_ok(fi):
+        print("Skipped creating a preview for Entity with ID={}, because the"
+              "file is large!".format(entity_id), file=sys.stderr)
+        sys.exit(2)
+
+    try:
+        tmpfile = fi.download()
+    except Exception:
+        print("Cannot create preview for Entity with ID={}, because download"
+              "failed.".format(entity_id), file=sys.stderr)
+
+        sys.exit(3)
+
+    try:
+        df = read_file(tmpfile, ending)
+    except ValueError:
+        print("Cannot read File Entity with ID={}.".format(entity_id),
+              file=sys.stderr)
+        sys.exit(4)
+
+    print(df.to_html(max_cols=10, max_rows=10))
+
+
+if __name__ == "__main__":
+    conlogger = logging.getLogger("connection")
+    conlogger.setLevel(level=logging.ERROR)
+
+    parser = get_argument_parser()
+    args = parser.parse_args()
+
+    debug_file = configure_server_side_logging()
+    logger = logging.getLogger("caosadvancedtools")
+
+    db.configure_connection(auth_token=args.auth_token)
+    entity_id = args.filename
+
+    fi = get_file(entity_id)
+
+    create_table_preview(fi)
diff --git a/test/core/index.html b/test/core/index.html
index c57f6c1f..fc8c21d6 100644
--- a/test/core/index.html
+++ b/test/core/index.html
@@ -66,6 +66,7 @@
   <script src="js/proj4.js"></script>
   <script src="js/proj4leaflet.js"></script>
   <script src="js/ext_map.js"></script>
+  <script src="js/ext_table_preview.js"></script>
   <script src="js/ext_bottom_line.js"></script>
   <script src="js/ext_revisions.js"></script>
   <script src="js/autocomplete.js"></script>
diff --git a/test/core/js/modules/ext_bottom_line.js.js b/test/core/js/modules/ext_bottom_line.js.js
index a26cc218..825c0d92 100644
--- a/test/core/js/modules/ext_bottom_line.js.js
+++ b/test/core/js/modules/ext_bottom_line.js.js
@@ -67,7 +67,7 @@ var ext_bottom_line_test_suite = function ($, ext_bottom_line, QUnit) {
     });
 
     QUnit.test("_creators", function (assert) {
-        assert.equal(ext_bottom_line._creators.length, 8, "eight creators");
+        assert.equal(ext_bottom_line._creators.length, 9, "nine creators, 5 default ones, 4 from these tests.");
     });
 
     QUnit.test("add_preview_container", function(assert) {
diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile
index a19843fb..bccd94fe 100644
--- a/test/docker/Dockerfile
+++ b/test/docker/Dockerfile
@@ -1,7 +1,8 @@
 FROM debian:latest
 RUN apt-get update && \
     apt-get install firefox-esr gettext-base pylint3 python3-pip \
-    python3-httpbin git curl x11-apps xvfb unzip -y
-RUN git clone -b dev https://gitlab.com/caosdb/caosdb-pylib.git && \
-    cd caosdb-pylib && pip3 install .
+    python3-httpbin git curl x11-apps xvfb unzip -y python3-pytest
+RUN pip3 install caosdb
+RUN pip3 install pandas xlrd
+RUN pip3 install git+https://gitlab.com/caosdb/caosdb-advanced-user-tools.git@dev
 
diff --git a/test/server_side_scripting/ext_table_preview/__pycache__/pandas_table_preview.cpython-37.pyc b/test/server_side_scripting/ext_table_preview/__pycache__/pandas_table_preview.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..613d9dce64b94c3b4c66891f22cd02a6c337dff6
GIT binary patch
literal 3257
zcmZ?b<>g{vU|{(3a(4W8eg=leAPx+(FfcGUFfcF_e`8=^NMVR#NMTH2%3+LR1k+4W
zOkkQhiaCWbg*k^MmnDjY5u%1QiVZBs9>tNun8K37nadT$1(svY;m+lW;$dV+VT<BT
zVNc;m;Y{I5XNlrV;ZEUcVTj^S;Z5OdVTck);ZG50VTck;5lj(kVTck+5l#_lVTck=
z5ls<mVTck*6-^P(W-9uW%D6x*MPeailz6ImmP9IJmSl=#FC!yEDpxZ@lvJwJ0_lYe
zQ8KAK%?wepV6{>U8KdM<*|OwQWwI1fq`_)svJ|6~Qu$Mrni->%Q$<phn;97yQka4n
zG-ayVxI!{AixrBKi!uvJ6iSLRQ;QW!@)e3w6H^pQDho1`6LS<w5|eUL6;cvQ5*5-i
zb5e^HH1v{-%M?;elJztd$}>wc6bcgaQWA?567y2Hl8aIkOHzv!5*0E^a&r_k3KEM-
zG81z&6^c>|ic*VH^GXs+GV}8k^3xPbGEyOC>v6pV#fqOM<1LPq#FEsK%-mE>##_Sa
zsU`7=Md_uvsd**w1&Kw)sYRNMw`7y^^U^ZYON&zDi&KlrQj6k?GgDIIbMn*EGxO5Z
zK_<0apYB-20&-;)i(W}_Srv<3MNV-QD~PN}W&|l=U|?WkU|`^2U|?_trGgL!28K$B
z5{4|s8io|cUdCF+8pZ`o3mF(0!Wr_+SQttevY4}2vbl=<7#TpYgf)d}0b32jLdFvI
zERGcBUM5C{67~h0DJ%;a85trO@|YqRY8ZnVG+C?E6^c?pAzKCt+l*9jAS$Hhm1LGw
zDx?+V=PD#8<`<_VDQGg@V#-XZ@^Vhh%gZlOfP|z%K~ZX1W@@=YT7Hp&D?}4G%spLf
zt7~->l2VfsON&z#GD{SSQ&V$`K{`wF6;dlQi%ax08E>(GqU<He%bHxbm{O8%vAAXC
zq~79yIq4QxYDIEt0Vqz2Z;3-JaLG?C_RBAE1)1SmRFq$Ii?yIAGq2<pTUvfmZeqzT
z=HklYTWrN8DXB$8w^&juGD|d>qL@=NQ*JS(Wv*l>5@ujv_*JZ*k)NBYUy`O@T$HR2
z@gtPdg+v^TSDu<wnyFusT3n(Jk$}V~BvEGOrR%3wl*EHmVLUW6^dTt@RZy>>@)idu
z&49eD3yM?$P)sv1G4e3-FbXlUG4e4AFcpb1FfhQ=04QKU=>VJtxEL51Y8bK@N*J?2
zF~?YCs!*JfUk;0cVuk#?oJw#+fNUsMD9K1HQAjLGRe%I~zCuo7QF^L^CetmJ;>@bl
zTl~I`5uU!GzHXjAuECyBu9}QRLJSO$-~oqGkq83=!!0h5;`q$s`21`ukgGxNVPGzj
zKz1ic3{+x)-KoOBz)&Gq!r;OXD^|-`!?1vHA;SWu8pef;MQornSI8F5kOJ}nQ<VcK
z?UfegDU@eqCTGBMU0GsIW{N^;UP@+Oda;5=2|PDvf;^R~ke`<d@-!&7`e`!X;tg~3
z@pOrI^>gv`a}U17nv-9iT6BveH7}(YloU0YZn33h79^Hr++t46OM&<dlmwvO14RkM
zE^v7Z%5n_MT#S5-985)W$eu)r3?2psh7yJvhGxbT#%#`D22G|a7ldb0OEOZ6z$FO8
zWQEMUl+0vk$peZVQ0bKliJ|1gJcXoGg%WVdl;Wq!2)0#|u}B6>Wbi}G14Rcol)R8)
zL>}Z=P)vZ56*F?_(Eutv5^6wBWh)XZVOYQjDj8~-@;qu7Q<$=uiVR8^vY2WZQkYX%
zQdoPLYnk%&Y8X@4vYCoJpfW5e?7d92OnG7@%qbiTSU|A}DyK_WQ#hL$n;63x@}yW8
zO4zcwimE{652&<hW@uuBmNcN!oU2L^y#&ty#bbV1W=d)bq}b79LMaO&#p5qV&R>km
znoPHti&G13u_x!}f=XOS0jkMZq|Cs;P$dc~HXwmonwOoIU!G^BP+fbAL6hSaQ$b1;
z7pTUF2NfkbQ5;})a&cJ^FDQWdKxv#Os5B2$tV4<uPFGl&bBil1F{d;YEUd{4j*(le
zX(g2fskfL?($JF<C)i|A^ag;U1(ZV=n3)(k7<m{u7=;))7zLR47<m}^7<rhAv_X*t
zN>QN50%aXgx&vpQ573mA#n{YPtWm<0!nlCBgk>RP32PQx3KO^>XI;Ra!VD_NOE_v6
zK<a84OE^Jku{ffHDTM{AdRCD7N}d{q1zZamYMH_r@{Cv*O1QJRil#v$43|x8Af1qi
ztmG@<sbQ*NY-XxuhMK^e%~f;*ZUQ@2yEzahlrYsWXYr+Qf|?$^OrRu`!Vt`$$?aF=
zLRQ6+l3$*elb@K9mYA87nxd!4Sj5J_!0?h4lqkG|voi|{Qd1yxRc2ngLL#W#Af~=5
z$w&poqe5meq|{V=$pkVMl&h+gp$-MbwgNcaLFy$)V*_0A>AeI+=}S<Ft;rO{k(*c%
zpPZjl3}F@JmltdD7lE?tEf!F@gp?3DAr)_YW=au1D0PCWh$2A{3se+=3*B2BFh3Wm
zgCsyHsYru?fgy^$BtIV1tkGlv7v#6tAce&(_LAHJkZ+2#K<dFoAD93a?;?<j9JQ`W
z2BlveP|{@N0#{62Ok7MHP!_1d;sddn1ell@`55^axfr=XRhJnkD{;if=O$+6#mC>`
zO3u&AOHBsV-I|QISaVX#QgfmNK{ah+N?BrFa%xIRetu4|CgUxx#L|+C_>%nW)I3d<
zTkP@iDf!9q@weEajiy^n`Nfdn07b$rHjoi1NkyO>6U7FpYKv4s0iX<$K`~qpQe%Uf
zV0sy;IR&XjMW8s0Qo*YdYHtxJ5{tw^fhGweKvgq13cw{aI0C`>xCrL)yu{qp`1o6#
z>8T|?`RVDYMNyo|`FSAIKowGPYKadh3~sTy1_k*C-C~2Z8E<icX;4$L_!diIQF?I{
zS4wJ9X?i>;&Tg?mwTmFym56XH0@cX3I6w+Pb{83eoQYP6BchtaCO1E&G$++g78Jc8
zN3$?-FoGZtnC4;RVPawAVdmiFVB_H8VB-+xkmq3JV&r3DW8`3BW8`AuV`O9EU;+RR
Ca&Ef-

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/__pycache__/test_pandas_table_preview.cpython-37-pytest-6.0.2.pyc b/test/server_side_scripting/ext_table_preview/__pycache__/test_pandas_table_preview.cpython-37-pytest-6.0.2.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c3563a411e0c836d2613ab7189dc6833be735e00
GIT binary patch
literal 4060
zcmZ?b<>ktFGdo_En}Ojm0}^0iU|?`yU|=Y&VPIfLVMt-jVTfV`(@eR{Obm<+xhzqv
zj1U>NC^oPRdlWmE=7{1<VMt-h;mYNX;s&c>&f&@Bjp7BfS#tPt`J?!A1)>CU1)~IW
zg`$LVg`<QS8Qd9CSX0<q7*g0$*|S8NnWIEg7=sx!*<XU3<)_JbOESJ7v8XsTzBn^4
zJtsB3A~z>KH77MUHLpaI@fK@pNl|L5ChIMJU&jbf-%wvSPaoG{&nVYhBFROmi6yD=
zC5cHnsqqCxsb!g|<+u1#^HMVN(&ICW<I56rGE;7G7NsVp#HVHEq~78x&a6rW$>e7z
zgDi((CRm`TFfcHrf&(OqDTOhGsf8hmIfXferG+7i1>%V)))e+&22GAz+yM{^LsE-N
zk{O{UfkKy!fq}sp6vA3e3=B043m8fmQy7~W7cw%!cuXifW(Y5ZrIxXTv4*jkv6iWZ
zskjAXT?x|y=7kK|Y{fw(EDKmm7#A`Y#gwotU<0x8m{Qn4>>9>}j9{8+A!7>r9A+_w
zTILe=5|#xVHOvbcn;DxJQy5qnY8h*pQaDg-c!pxb7Pt-jK&Fvt11GW#6BvvAknNbj
zSS$mxWddW78U^-nq1m$y&7Kw{d-^D_ha1HnreZlXd)^?~^Nj*~7=jrzdHkw`^-D4`
zi}f>$_4D%eD{_kUa`JPk*kKH=#B@CvTL`W)Q9ma?2P9sllUQ7wT2!K-s!>pBqFKyU
z4P_|Uf`tq;i#0$ZMw-Q%T(w1@WU9%0i@BiE07Mvp2$L#wbCZk9FvWr~#hin~s@Ope
zaSjgC<c#8mIK?e7r#KaC(=8r{g39=$(#)Ka%)H`TY&rSKi8;l$xEu;9OHzwV;uDK*
zNyHat<d^28#22L&6ve0K<R>NO#OEdErrzR=FHTHL1qt8ca|HV_Ge6I@s3^bamT-Jp
zeo<~>NqlNWK~7>`Vo7Fxo+isJmg3Z$v|HRDJ0UJHMCKSnI1rttD;bI;7#J9SRp@8r
z=cekHr0EwICF>_A<`<_VK`Gth)S|M~A{ehcHK{aHALMR*h(vsGW=d*&adJ^+K}lv_
zx_)X!32LbXRvcfDn3s}Rj4Gp7P<cxLWE8lBiiZ?ls-WV-2~?&q@_``>0|x^OgAD^8
z0|z5In~lK%E{@5rl0hn`^w2GW7hD_+3=AL)F1Rch85mL+YZ*%zN*EU~)i7o;r!b{3
z_cDTtlUk+{mKvsJ##-hY=HjjtmK4?!)&*?fLdvd$eE~-a%R<H?j}rC;oFG;n6F9@b
zR3v~@)G#e%EXo3@0I|Tu82bWFP+<ivzF11QO4t{0*RX)>0~KGj%m|m1@RabD@TG8w
zGc+?cGs0cPTEbGp3UL(=LoHhk+XTj9ofJ+a*WCfx4RTY_GmzaN7T9%MF!xVjEM@|!
zsbO8nIDxT<2c!zb&SOdexd+*W>`)i7!(7P1P|IA)mIBJg+<sNwHg1_YsS24Xwn~OZ
z#wJP%pj2$D6r7)%3X)J#C`c^HuvLO&VGvudq_|8;0aQ8JDpiB1S|vLheUMtaC{`l_
z0|Nt1##<~P36xB349n!kn#E|D+z^t<K~Z{(wKz91Cr6XHh>L-N0hG-RL4@&3P^DF6
z0LdQ;dHIlngP@tWSaTAK(o;40i$Jwz5i`h4uC${3+~Sg=%)InlY>;eK1hTq_mw|zy
zhz~^Yg9rf-AqXObK!h+zFLQB8(Jki8ypkeN<*CV0B*MVJ5XF_6SCUy$32F-z$$*q0
z<z*w}>}yh_fF^BHBn~nhl&8TNyGRnGfDe)MLE)$m%3(_29LB=L#3;f9%3om&d<-m%
zOpF|i5{w)SYz%A+Q4D+xRZ`e<5XcyK{sDOu6ujX469&#dpw>eQvkOBrV=dzZ#tN1i
zMi++IKebFH3^hzCEHz9E7*kjlGN!OfGKe#zu=O(4G9!znu!}R)vXn5@FgJr54$L(y
z*-W*pHLO|SbXLn=!&bwd!qLlE%TdDtPPr@#SW`G_Sko9&xN4YdSkjmlvZQdAuq|K*
z=gnEL20_u15*A2q%VSF6PT_&8*aKDp%Du<HDp(dW<}s!4r0~vR5@V?4OyR5H1dFqQ
zn;V=p>`<CxA!7>v92PN#TCNhV5|#zrHC#x!f|Z4#mW_p>mc5pvP%NAwMF1Qsg5Xf$
zuF$FBc43HBspY9*SOCh`3^hC{EHymHA+wN$kzoR3p;kBpICly8!Gj!>{MZ>77=l4j
z32L~dGt@A|%7BWkEXD;)B@9`N3z$LKJcXf#aUt_Wrb3or22egqNi0d!WWB|dUwn(D
zAh9Il7E4xsX5KCKl*}SfM!m%m4=Ib|HJNTP=^5N&OoOB&aEbyYre9n(Ihn;J$@#ej
zb~Yf}K~7;~tkT9vDEXx&1*Ii=HjwJhP7k3^lckCSlqB^)H7tZxQd|ZVtH>#?;snc9
z<P=xjV$^&2|NsC0nv7AbX(g2fskc}_tyxXRD3-*E%wkQ(Ure^Pnv7LKpxWI4R1`oe
zd?U@`Tdd&5@h#5c)Wo9XjO^4(O(t+LP{hr^z)<A_&HNyZh6b?G1jK?CCuT4)Gmx?1
zW`v1mv0h?cC0DHm$Z}hdRts=pqsdal1IlU41(jx?0><JNOF^Zf0j2R(q{+a*pvhXK
z1<KUgAVLR3=z<76kQ7r9C^HusfCRu%1SUZ7RAk7&z~G9AAWfDk_N2rVaFBx;pb!T$
zKmpDHk^+UdCSMUKUEN|WF38C&xy4#gl$lp@i#ai`vIta)-4cvXPRz*xH&Z}KAh9SF
zshq=7#6|Hyf;lKLvpBW*7FSqePH8H*`J%}SF70lyg5tCI7E@Ygkr~Jv<{&~86qrcG
zo++f*gP3I=g(PDM;Xn#rV@OjA!Z9?A0(lE;d=zt1S|(abzQqknW$}<?Y7Gi04RD#r
z#>m0M!zjfl#K-|I30WAp7}yvR7*rVe7(|#rvRsTDj8)<|61<-#8z>ina#MW#Ew1?Z
z-29Z%oYeUETRidcg{6r(5Sbz=kUv2-7J*_3$!&-z<YZuA0A=oCP<xewk%NnarwAmb
zDRzqk)|><fKd2W8ZcFMxLMSsoPp>GoyeP9I^%f^w=oULv3>=l<M0Ja!G%vFR6b`r8
zAg#VzOeslGA`rG-a(-@Zex6=#eoAUiF}Ti)V#%q<%>mbZQ6kvdbl{q-h!x~HkgLI|
y30xoE;sEs)ofC^w!G6Hz;aeOwkifD71x_(2IdL!uF>o+)FmW*QF!C_UumAwg0y>cZ

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/bad.csv b/test/server_side_scripting/ext_table_preview/data/bad.csv
new file mode 100644
index 0000000000000000000000000000000000000000..d29a9312a387186beb8bf4f77a8ec0e4b0ab80fa
GIT binary patch
literal 7
OcmbOzAY?kllK}t++5yG@

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/bad.tsv b/test/server_side_scripting/ext_table_preview/data/bad.tsv
new file mode 100644
index 0000000000000000000000000000000000000000..d29a9312a387186beb8bf4f77a8ec0e4b0ab80fa
GIT binary patch
literal 7
OcmbOzAY?kllK}t++5yG@

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/bad.xls b/test/server_side_scripting/ext_table_preview/data/bad.xls
new file mode 100644
index 0000000000000000000000000000000000000000..1f31bf2754258e3d07f88fd1e6bdee4d7b11bee1
GIT binary patch
literal 66
ncmZP;;o@UpU|?`$U;<GXI3zgC&C|yfB<>6%7(oOQ=9K~fV@?A5

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/bad.xlsx b/test/server_side_scripting/ext_table_preview/data/bad.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..1f31bf2754258e3d07f88fd1e6bdee4d7b11bee1
GIT binary patch
literal 66
ncmZP;;o@UpU|?`$U;<GXI3zgC&C|yfB<>6%7(oOQ=9K~fV@?A5

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/server_error.csv b/test/server_side_scripting/ext_table_preview/data/server_error.csv
new file mode 100644
index 00000000..3e770df0
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/data/server_error.csv
@@ -0,0 +1 @@
+Hi,  this line contains a unicode backspace. This causes a server error, when pandas_table_preview.py's output is serialized into XML.
\ No newline at end of file
diff --git a/test/server_side_scripting/ext_table_preview/data/test.csv b/test/server_side_scripting/ext_table_preview/data/test.csv
new file mode 100644
index 00000000..7c9bfd13
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/data/test.csv
@@ -0,0 +1,12 @@
+# test header
+# two lines
+A1,B1,C1,D1,E1,F1,G1,H1,I1,J1,K1,L1,M1,N1,O1,P1,Q1,R1,S1,T1
+A2,B2,C2,D2,E2,F2,G2,H2,I2,J2,K2,L2,M2,N2,O2,P2,Q2,R2,S2,T2
+A3,B3,csvfile,D3,E3,F3,G3,H3,I3,J3,K3,L3,M3,N3,O3,P3,Q3,R3,S3,T3
+A5,B5,C5,D5,E5,F5,G5,H5,I5,J5,K5,L5,M5,N5,O5,P5,Q5,R5,S5,T5
+A6,B6,C6,D6,E6,F6,G6,H6,I6,J6,K6,L6,M6,N6,O6,P6,Q6,R6,S6,T6
+A7,B7,csvfile,D7,E7,F7,G7,H7,I7,J7,K7,L7,M7,N7,O7,P7,Q7,R7,S7,T7
+A8,B8,C8,D8,E8,F8,G8,H8,I8,J8,K8,L8,M8,N8,O8,P8,Q8,R8,S8,T8
+A9,B9,C9,D9,E9,F9,G9,H9,I9,J9,K9,L9,M9,N9,O9,P9,Q9,R9,S9,T9
+A10,B10,C10,D10,E10,F10,G10,H10,I10,J10,K10,L10,M10,N10,O10,P10,Q10,R10,S10,T10
+A11,B11,C11,D11,E11,F11,G11,H11,I11,J11,K11,L11,M11,N11,O11,P11,Q11,R11,S11,T11
diff --git a/test/server_side_scripting/ext_table_preview/data/test.tsv b/test/server_side_scripting/ext_table_preview/data/test.tsv
new file mode 100644
index 00000000..863f692b
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/data/test.tsv
@@ -0,0 +1,12 @@
+# test header
+# two lines
+A1	B1	C1	D1	E1	F1	G1	H1	I1	J1	K1	L1	M1	N1	O1	P1	Q1	R1	S1	T1
+A2	B2	C2	D2	E2	F2	G2	H2	I2	J2	K2	L2	M2	N2	O2	P2	Q2	R2	S2	T2
+A3	B3	csvfile	D3	E3	F3	G3	H3	I3	J3	K3	L3	M3	N3	O3	P3	Q3	R3	S3	T3
+A5	B5	C5	D5	E5	F5	G5	H5	I5	J5	K5	L5	M5	N5	O5	P5	Q5	R5	S5	T5
+A6	B6	C6	D6	E6	F6	G6	H6	I6	J6	K6	L6	M6	N6	O6	P6	Q6	R6	S6	T6
+A7	B7	tsvfile	D7	E7	F7	G7	H7	I7	J7	K7	L7	M7	N7	O7	P7	Q7	R7	S7	T7
+A8	B8	C8	D8	E8	F8	G8	H8	I8	J8	K8	L8	M8	N8	O8	P8	Q8	R8	S8	T8
+A9	B9	C9	D9	E9	F9	G9	H9	I9	J9	K9	L9	M9	N9	O9	P9	Q9	R9	S9	T9
+A10	B10	C10	D10	E10	F10	G10	H10	I10	J10	K10	L10	M10	N10	O10	P10	Q10	R10	S10	T10
+A11	B11	C11	D11	E11	F11	G11	H11	I11	J11	K11	L11	M11	N11	O11	P11	Q11	R11	S11	T11
diff --git a/test/server_side_scripting/ext_table_preview/data/test.xls b/test/server_side_scripting/ext_table_preview/data/test.xls
new file mode 100644
index 0000000000000000000000000000000000000000..a355756b9ab72f9035246c5303800a2076d9bfc0
GIT binary patch
literal 5632
zcmca`Uhu)fjZzO8(10}qGsD0CoD6J8;*1Oo3>+}N00RRP0|Ns{?BD<Y|6#)aN5PN^
zfxl4Kg5r>cfq{XQfdL%n><kPHATdrT4OKQu4YLpkV#sI6XDDG%U~px~V<=%LVyJ{=
zWuj;?237{-AR$H*C|wbwj#xFL_9H@om@q>Qe^B`iD#w`_7#Ps=Kd7t+<$o>)1_o{h
z1_mAm1_oXR1_nL`1_pix1_l8J1_nU}1_mJp1_ogU1_luZ1_n_E1_m((1_p5k1_lWR
z1_nt61_mi;86pFvLH5ftFfb@U`ASe4;fqlgy+VMKLx6#Sjg?_H?-@{$4=(#3GB7c0
zU^xh`-ybnBFvKtvfYOF@VotIGy?_(g4T+4Pw0VG$fx(uMiGh(}4g(VdAH#aE=pHa>
z3r>9u3=9qo3KrnR&&cqF(E;rCUNETyCixgZ`nQAi+yaXjGDtC;U|?YQUq1nw30N5&
z8HyM(84?+C7|>)vH6xW|<yd$amN2k0Fu14Yr4}XTJOJmLB@AExgYpB|3k(bl2NXc!
zAcuj)zYr%53N38vafyRW#ioAviZ5Yc1eZG?AA&F}oV6k0{{bWh!Z2}79O61S#I<mU
zb7F{3W@2GrXkhpcq6He5!L(ol8<-YqU;)#@4eVfAq=EH+0yyoqGVn1}f(mK|HiqDg
z)YKBg9tIW$Lk2^JgNz&u2N@X{LV6h)UNc02O6U(kAg?g6eE^ljpezhZ2cR^CjExz%
zKY&W`{R|uo91M+&oD2*c2N)d~7&sgl<UWGRcwPnu4ju*u4siwsj_(W%0)H69z&QwN
znj<>{Lq$$;T4qk_e+DiF4h9DnNL`W4A<B>pszku9fYc2jCKISaffPIfC<PA>0~13E
z13x&c3WBW=VBlc*v-0C5-=qNhc(8Z_q6@;H3JO972F6APCWeD7T3{X8V6q9UR)j%?
zL6Lz$%~?$;Br`X)Sivu~Tp=hwH!)8qC^fw_C$UIL&Cp2AQItWGL5-lgfW-7v1+{=?
zumvq(QjLLw!59o)+N&e6H5fP;nssk9>E8HYuZhH7$e_f)$&kXp2!>4zI*dAuVC}{)
z?O`+{Lns3i0|!GbxV~jzY~u#iwG0dlOrUxf((8isyEvFYu>)eOf+CcGfsvm9BnILz
zG9bkMGJpacQfD(TfOsGk{NTRZ8-@T-fPe4+*XJA`K+y$~1*H~#2Idb2;JTjWg9f;-
z$N2$bBPbRTn2CW4tPi9I)PJyJ5@Of}?l?Li6fwv$@H2o~8DN7T;^`dX4Cx>nAzV<L
zf$BXF58`4F3qNLL`1hZgivipQ72p7+NCt)j5G`)f3=9munaM@@#rbI^3a%B&sW}Rk
z=DHz?NvSzGso<*4DKjk%Qf)x=kW4bdB7|g}5DowSgW?b3Zionk1jWIF*7=`)`ez8N
zp2)$Vt-kptC@yNC{WFj}$l;*&87P!N;pGEur$s>9YAMk6S^+5WFfcH@{L8=~2W1<9
zt4vS>Z4#*Y#Q-t`Vl${JWnlOi|M9igqHCaXfI-}Sl0@N>YalCOO5iju_k(=IboK1j
zIeO9plc#eqXzTo3=K`ul(Zph)?L?5_APh=hAiW^Bf=Vrr7ziVa!z~|WP#gl(?Eit|
z2`vv|b0DY`H~`J|;SBi<MGV;tN#K5HHa6vW#e^7`z$G!LN`#dicr`#J=YY#6P~LN9
z$Y;o9C}8ksNMgu>$`Vh3><9S=WH$&CqZ3pvc|sL3GWav(Fr-3xq*GiBOyIH)WH$&C
zV}dNSuLe@f$`H&@%8<*D%aF(b>ic^#<T0dy;{+s9!jQ?3&yWXFhy!at?FW_n#M%!k
zgVsRJV`Xq*$Y)3fyCD@k0zsO4ETHy->;_?|v3My(1_p)_yedeO1lbG2#OMU|e~D2?
PtQwfzAetCG$o2yOou4fe

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/test.xlsx b/test/server_side_scripting/ext_table_preview/data/test.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..bc291f1aa86cd6d550320f07a7ce69cf813b8116
GIT binary patch
literal 4801
zcmWIWW@Zs#;Nak3a0v+vWIzJk3=9nMMX5Q(`g$O8?WFTw%!UH(?`yjzRF`iPbxGO0
zx8uTWkJ^SAXZ|Ic&oq!Mt*vh6h+4Unz2CF$^I3D}S$BofzIM%V?{dpu6{X$N_eY|7
z@qE7Tuch~9ho4nG7v`nHpL}Z7l_NP{6ei^{Zjv^*5fgYQBF^ej!cv)??J0Lts#|hn
z<C*`by!h6!^4q<Pd%t_b+P}9<->~wvTYLGFI3_pCi;uz=P0;<W;ihlAN6`PgOK|7{
zDZkx0`sF{X{WvtPMQ?o2Q8;%;$mX6I`%7(hwinH{7k=Kd?q9Lx4i2~ee|skRzC4|A
z^}t2-410zEZ+4EfRZZI;FfcG!GchpW4j6t028N0p{qp>x?4<nsY`u!yoZjG*dAAJ&
z_I}r9xtFUOmnA)oIlONI!(D;pCudG_6U*)rIq`AYqW|{<Q<rMqa^^R)<i5XWW@Xg9
zIoU5|4@N8uJ##4WfK9+oy_Q=?-`q~NnqhceDV%kBl;at_gqvqy-Tk!Wtlj0R_ZO>E
zlfx9vxbJVTWHsIHe79kd$4cL`6MZ&+&P{F*-^O$x>Q|tE&)-?xN3B{~f6DX3PE%^{
zJ;!gV@y3D2qwmPQutghlw(7LDEMKSCZ+<pCPWWxh={Y}Fhu@0to&K?=J|pN<U*yT9
ztXW$wE!V$Ueqy54qadU84>%s$NPO3<cgR@cqvd?=+DDZKDfP#rFDu^5RF2!?q_=(N
z#h_z6XCFrOb}TyaU-ag-=Bg(%-_CuWzL2Z7OLFatPtWR#uBQhkxN^_wOM85;oINQ#
z@l4u7n<TkUZx+gioKa)6)o$JRqRT?WZ>~(^tF2<DPxik&@pg^9?Nhc_ac#PxR`0o2
zy(wuDvEtG;^6WY56mZ~<_(bP7i7W2Td8RMG^HlxUf)1w}MMeR2mslOwaK4qQk3Dzw
zUW1FDO1q=LwhaZx`o->6hEBRt^55GxD22`HuZ5=J`vjLWb$jN!T~0CUWxHr4!+YpM
znb8jeDSx(?xw9@knB;5ZUq5M$%&Vq4Pxt$a7Un-;JoUCUtoohGwz4TTr(XS&HQpb7
z<drxm&2(27zx~e0z>vp`FU{~m(oAtlWlm}_D8;;ui1oj1AX0mO{f0kwlcohFcr{A|
zFW>8vA;Ot?tJ!nTkwfwm-#Q&$f4_9j63$j%t;yHCYB#+<X?tA#`t?<_UnHqoR4ut>
zpqgG$B9i2ix+qBPv*f<mGxqZmQ*<V3KJ>aIxo*Wl@sI5%ZggFq-4xXA;5bWCZ;h@!
zYojNZR)=Mf>eLS^!BGO-3(qkJXL$uZ_Gn^%Y%qtd$)NMGhu+bch-RU!Q^ayg1U?<!
zE^^={i?V+C(x45Wrh2X7JnksN+Up>b>b0&^?FMsU%Djup4;6%Vm3Xpz{kSwOPEon(
z<EiFkb`{o(Z1>Jxe&xF7idfUba~ED8oZ4A<Z)#?bt$@4g#%WoW?F^=$T|=GLs^;vQ
zzVh?GNprJ<dXK5wI@uoB_1$avg`B^=-vqoQPfk_&vu)19o&x7W_S!)IO{|YQE}v=I
zkXHH7ZS|=Goe`^9e4_fIZ)ukcN7yZ#bAQLVy{E6`%QwByy8Xr^lD$A-&iAxeQ)e|y
zQC<J+RA<EACA=SY`bj!WpYt^6f_UfksK{A2em%HaAT!ZSP}FO4`h+s+8Rot3&aF)C
zZ2t2*Ddpv<e*x1fL_d64;<=MygSOJ-%)JIS#;s|e5*I#OQ_YjAWi8#Y`bp!mwI`dF
zI=oF~Tm8m$>7mlOZnv+k{P=TIK<$qBmD_l~9Y6CU;Tf-S^!&BWGhs0Ij89+r%jrc%
zw=~Ts&n+^(r)vKE|4p9@`}h4mUGraV|69|#skv6~e^)O5TD$x5{^dFU^A3MskmY!}
zVekH_vBrLd{I2i#fAQ3p-`A^HU+~@gnC@gdE;G(>FOEyk`u*5i9-DK%NdK`~*_VCE
z%hgMkUJFqRJ#HympjQ8g=bDxD2fz5ovB7f>$dyd_%y_ikAVOTY>1+D3ALgJ0?c}4|
zHI0dZVYLvx1T6tc(4a!FI3qQ+q*xzJ8G_RG-mu$ww=D!}@2?m5a$)&e)#}&Jf^ND#
zShJn=l10a5-3|F$EH-rn$V^-EMlF87^Q)-zjU3++d5?&tpD=Du*Yo=J(Oq>hvxIhl
z?xU9~y~+WWMGL=m|9E})*V{^&$m1gELZUj4)Hi%Q{#5^_;Mv8xj4>_@r}Q3OvaFq{
zVy3?^sjNv#q>{@-w0Mij?;}~vJ?+j@ebO?2*{z%rP}p3M8Q0wCs#vt+*q(I`JK2sj
zr1ZVG9h7l<;ok@A4?S*^eYr|@zTnZ<$1Xqiy}$RsYL~nVNe|U5!*sV!-@eB~uWncR
z)ctJ{N6#d$y)U$3+3B|}BJVT{dZtG*>^BZnT*CaB)$-}mgiB06O>Q3+yAv+^=N;qw
z6Bl+)PMx-;eY&K$%0mC@lm!t-!Y`j*J@dYx*&odZ8SdvN>VDjzR<|`}TQm1D`<@5p
zPK=RrS4Ka!l?&IfpC}L#nS4Cnp{tZl`J1ZYp2kC8C-lADrl44U;P`X1t&>Gkeuumb
z_{Jc^73Te1Gr+=R?q7!<@r|E~yqa9zI<Il?sy}OS>hHbtE&u<;%GfH0-JatqlsRkJ
z;;*?Dby;7F?rl(C)2y@hy7^(gj~8d#<h`0KlK1fG)idUPr<M97BKZwANM1X8@kCv|
zu+fv#%>DCsUs>aPG+<S<sz+;n)9y(XzUrSVtIF)=o~@p%p*{aVw%@y(EmFT{Zn_v5
z^>?G-FE&1f%z6crZGkP%>Q4HzMA=<GvoSU8gG8X-o=aOxBTQ%i5^(+#RKT~Y++_Fv
z)_?@(o0EN_<K<&7KDV25{ZgB3$&E>0H~4IR;^0&KOWDtM<IVp*Ibt2Y?njoXADJ|<
z>K4aSGb^7hE0%~Y(lps}r!HfAF!$El>kO(=AA3$2?p$QZnfBmG`d;Cm-tL#q^y!~@
z!^%2sq2}e2H-FyT^Y%)j=T@0}qMvU@{#U$O_QGRX{ii93LOcwti=@|1uFJbR>y~Tt
z&93xUGr5KGuFPsRxb~B6&)U_pCZb|#G2x%zZqol`@o9s!a8j|&s&=6(Z6XWaNIEjL
zd_SgiEn`k%)2w3k$bS_}e{2uhQJ3TPWtv3qiaAlGtyAk6BkfZ~Zf0@64lY~1a>unK
z!T-<nFFj4z@MHyRi;|+Q)YfZ`i#xg(EwIrP|G@4wxAO4M_w5VArb`yx<GJ@#I_xF?
z>JNXPRrkI-%-mZeefN^Z{^MOM<KHLxf4*aKZbQBmw|#h|(SeF1ZD;D8zsz!Z_*3t|
zwunfhed@Zu>oup<GlTMX=B9&5vzQqerg7lQ-%^nL4QT*^ayhK218EFS^vynOz|;2r
zuc+Jiqn_-^Jr;X=9M}K-!FVk2(S;*sldf(3U#BM>&6WD*!sH9rKkxJS`pTsDRmTx$
z9=27BSXvJrE#uhR`1jr0hmv7u7u|c1q1wMeUTF0m;TsL&p88RnKU+lVB^{bz9lPYD
zm*2|kCwDc!4b(sV$@B6Y#<QZqYonRB<Y@=&E<W^sL-KWAzWBw}^9@e$a-J=cn`nG7
zs6Xj}w!RVjm)xIEZJ*jJ$=*;twZ3_NkXbqB)%Y~I_f_2aoVV8o)V=wm^7+&DnXYTs
z-@9AG1PU4d4?n9eF)%Qw;%)wkKtiTCBe5toCAg$0GcO%nqu2KNavd@dX#GA<=}cU3
zY=_4rr6%<Y%q=Ci*d}?NT@sX^FLGq=>Eple^=D0&IKpiF+eK>IY&Moj-gB2b9)4RD
zDeq9Q@yg8!9HurC7n*7xjdAG}%9ik+*%!1)+A!+g63&%YwWlsl)B3DXc`CzNPSBgD
z=(<U@<XUwBy|W=EZye4%d2c*#|1FJgZnbKCHj&}ev)PmD6lP4GbX4P~+P?tr2TQ**
z)^Av|{46)ftAbY18Y>wX7(OxJO9X-p3=Apx$pJ<A1;zTw`9-Oq(!Dq6wBKO^fjyr^
zC%#`fdCQa})s-wu)wL7@Sstam$<6AJJzW-G-#^(Tvr%qB&5gN|=igP$_4~ewXIsLW
z6(Mbp99x(I);w|&m8^<9cbJoF&El@hn}Q}RWR5U7yFO?c=V~6CnO@C>OgRsFr>Od*
zXS|)GCK0`j&GYR6?J4TvQzu^ec|{~5_?s*D0Z)<MjV$h#r6Skc7QU0ccyZF^dqMd=
zHzf+%7P4@jij?6`s<~uka4Bxj1%q@i%{z{VT{GRkz1l0fw_y8i-m=4PuYUJ`y#46o
zPDxXdRdrfy8vgyg0%;9a4;sxCa(DJs%HBIat$2!hpit{Z`^Bj?ir15c)Zc6VE8TCF
z^Z3*g(`v7pGY5^7w|utFIVs<8-g{M7Z~cck&KA`fDz+i(H!ZoQ`{z<P(~r9ct78uA
z>U$#i`@DzwX$!@}f}MHi-Cps`I>oka(ZA}S8#He%FTGy&`6K6<t;%0J`{x#@ZYdD|
z_p|>exGuh;|6V<Vk%8eh-V`Z-98HM@1)x}(JHa26Lyx}yEpqV7tQ(xl+D8j-N?40-
zb2cmEUbl9O+&}j<Q@-3je|y2(N^@J=kCVUrC@b7%ad3*y2icX{Zd@O<G9L5GR=$rv
zWvVIUwSFnDoK;bq%eF^-`+XG;wXg+E<g_b&(jj3!MXi)YJ$P#F{JDlceo50VH?|d;
zaLsF;C0ntCH(9rp^<ege)IAI*<qL}s{dX!2u>Zca|DTMp{|C94Ekce3Rl2_$E`2-J
zxT5&C(vRexYZ{%Ff*Xy0bH>$&Uuf?NExo#??AZC!A9m)of7`c1QsNVL{DKKa+p1>u
zUQIpxQn%-FcGe?lP;x#cVt(}%0|P@9BfjJ;%)r19?VO)ilA2c%A5vM6S{w@ss=XoB
z{)Y_&?)}zQTCY)dx<SC}R^xGwsT#&cA5{%|<u~2E^soAyW=JTfW=F@#P4D+j_b)TM
zy+SRr`SYb0b9`RRP+7pOykuR`r!70Y^-tzlof3=YwmTU2wm7IgVZFp>SKhAbES-!s
zQ;uoM6};9;tK9fRbi?P5Kbl0{*e@vZPHgtsEUoc#73ayfZ2jfCIMh#mU8nE$v?coX
zi6(ZP%@L_{E!cbCGp^feQ8c~IVsS@Kb&c*?``>W~U%m2hST#duUb%9dZi(|PrX;x!
zyXJ~aIJ1-cdrTs8b)Tn%tcCydOUn}TkLkwuL|*!r{XW1rRg+s_S%hXwK)!6j!|a9y
zQn%KH6t47t@q0t=0sgi3CjC>W<o>WGR*iY~jnDHxY?S`~Q08CemwnIpjwBcS*vCHq
zfMWT4{v+q-Zj`HMW8ZK7ai;a6;BJ=(O>gA?#tE+cKS%0Ow79~)tbU_wbyF^vgJP!t
z+f%bhMh1p(R#40^GKnxCMjMd3+uTSa4-h7#>H$|b0p6$@k^5Yr-Z(@9h?Hc+=$oT!
zMsDANx@9135bVng){N++p=(91IzbI!h&B)@#|G94swlzjVC3d7x+%!j45*p~>4)Hd
zoM2NRJ_nhFQt6_bgItD#DkX>+AX0)GYz{^hg>C|J@d7IAL3$yWnFmYBkFFg#*MZ6)
vh*l7}iWf;cv_wMJjhr$-ITNH0f-mu5&8Gq0tZX0!oD7@{_Dl>6Ibb^hkcs=%

literal 0
HcmV?d00001

diff --git a/test/server_side_scripting/ext_table_preview/data/xss_attack.csv b/test/server_side_scripting/ext_table_preview/data/xss_attack.csv
new file mode 100644
index 00000000..e7d43505
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/data/xss_attack.csv
@@ -0,0 +1,8 @@
+# as it seems all these characters are escaped correctly.
+"![Alt text](url/to/image)","%3C","&lt","&lt;","&LT","&LT;","&#60","&#060","&#0060","&#00060"
+"&#000060","&#0000060","&#60;","&#060;","&#0060;","&#00060;","&#000060;","&#0000060;","&#x3c","&#x03c"
+"&#x003c","&#x0003c","&#x00003c","&#x000003c","&#x3c;","&#x03c;","&#x003c;","&#x0003c;","&#x00003c;","&#x000003c;"
+"&#X3c","&#X03c","&#X003c","&#X0003c","&#X00003c","&#X000003c","&#X3c;","&#X03c;","&#X003c;","&#X0003c;"
+"&#X00003c;","&#X000003c;","&#x3C","&#x03C","&#x003C","&#x0003C","&#x00003C","&#x000003C","&#x3C;","&#x03C;"
+"&#x003C;","&#x0003C;","&#x00003C;","&#x000003C;","&#X3C","&#X03C","&#X003C","&#X0003C","&#X00003C","&#X000003C"
+"&#X3C;","&#X03C;","&#X003C;","&#X0003C;","&#X00003C;","&#X000003C;","\x3c","\x3C","\u003c","\u003C"
diff --git a/test/server_side_scripting/ext_table_preview/pandas_table_preview.py b/test/server_side_scripting/ext_table_preview/pandas_table_preview.py
new file mode 120000
index 00000000..f24b3901
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/pandas_table_preview.py
@@ -0,0 +1 @@
+../../../src/server_side_scripting/ext_table_preview/pandas_table_preview.py
\ No newline at end of file
diff --git a/test/server_side_scripting/ext_table_preview/requirements.txt b/test/server_side_scripting/ext_table_preview/requirements.txt
new file mode 100644
index 00000000..4628529b
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/requirements.txt
@@ -0,0 +1,3 @@
+caosdb
+caosadvancedtools
+pandas
diff --git a/test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py b/test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py
new file mode 100644
index 00000000..00d1c7f3
--- /dev/null
+++ b/test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+import os
+import unittest
+
+import caosdb as db
+from caosdb.common.models import _parse_single_xml_element
+from lxml import etree
+from pandas_table_preview import (MAXIMUMFILESIZE, create_table_preview,
+                                  ending_is_valid, read_file, size_is_ok)
+
+
+class PreviewTest(unittest.TestCase):
+    def test_file_ending(self):
+        self.assertFalse(ending_is_valid("/this/is/no/xls.lol"))
+        self.assertFalse(ending_is_valid("xls.lol"))
+        self.assertFalse(ending_is_valid("ag.xls.lol"))
+        assert ending_is_valid("/this/is/a/lol.xls")
+        assert ending_is_valid("/this/is/a/lol.csv")
+        assert ending_is_valid("/this/is/a/lol.cSv")
+        assert ending_is_valid("/this/is/a/lol.CSV")
+        assert ending_is_valid("lol.CSV")
+
+    def test_file_size(self):
+        entity_xml = ('<File id="1234" name="SomeFile" '
+                      'path="/this/path.tsv" size="{size}"></File>')
+        small = _parse_single_xml_element(
+            etree.fromstring(entity_xml.format(size="20000")))
+
+        assert size_is_ok(small)
+        large = _parse_single_xml_element(
+            etree.fromstring(entity_xml.format(
+                size=str(int(MAXIMUMFILESIZE+1)))))
+        assert not size_is_ok(large)
+
+    def test_output(self):
+        files = [os.path.join(os.path.dirname(__file__), "data", f)
+                 for f in ["test.csv", "test.tsv", "test.xls", "test.xlsx"]]
+
+        for fi in files:
+            table = read_file(fi, ftype="."+fi.split(".")[-1])
+            searchkey = fi.split(".")[-1]+"file"
+            print(table)
+            assert (table == searchkey).any(axis=None)
+
+        badfiles = [os.path.join(os.path.dirname(__file__), "data", f)
+                    for f in ["bad.csv", "bad.tsv", "bad.xls", "bad.xlsx"]]
+
+        for bfi in badfiles:
+            self.assertRaises(ValueError, read_file,
+                              bfi, "."+bfi.split(".")[-1])
-- 
GitLab