diff --git a/.gitignore b/.gitignore index cc9336b6256771f79eb104a8b1325a55352657e1..ddfb9ac071b823c0b1f9b2495c1e44c49290ec1b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ # the build dir /public +/sss_bin # screen logs screenlog.* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 548f86457c3216f5eed7b75b4b81d7f048351248..2a014f451fb7b6de3dd5a8715b3aeae701d806e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,6 +58,14 @@ test: script: - make run-qunit +test-server-side-scripting: + timeout: 10 minutes + tags: [ docker ] + stage: test + script: + - whereis pytest pytest3 py.test pytest-3 py.test-3 + - make test-sss + # Trigger building of server image and integration tests trigger_build: timeout: 15 minutes diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef6c2a114c1ca5c8dd4440e0e7e122444d6d8e0..a30e209b9fddb3d11b36f5b8f52bd7017a858774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added (for new features, dependecies etc.) +- table previews in the bottom line module + - added preview for tif images * new function `form_elements.make_alert` which generates a proceed/cancel diff --git a/README_SETUP.md b/README_SETUP.md index 4706efe28f798a0260b0bc46374491dca7c0d667..f50ed540942587756747d5b90d401b9a7101bd01 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -53,6 +53,10 @@ information. * Run `make install` to compile/copy the webinterface to a newly created `public` folder. +* Also, `make install` wil copy the scripts from `src/server_side_scripting/` + to `sss_bin/`. If you want to make the server-side scripts callable for the + server as server-side scripts you need to include the `sss_bin/` directory + into the server property `SERVER_SIDE_SCRIPTING_BIN_DIRS`. # Test diff --git a/build.properties.d/00_default.properties b/build.properties.d/00_default.properties index 33567a800f7f18449228c21e3a65d723093d7538..536a2925f94f8ee5b3a8d89631e454fb5fcd4563 100644 --- a/build.properties.d/00_default.properties +++ b/build.properties.d/00_default.properties @@ -45,7 +45,8 @@ BUILD_MODULE_EXT_PREVIEW=ENABLED BUILD_MODULE_EXT_RESOLVE_REFERENCES=ENABLED BUILD_MODULE_EXT_SSS_MARKDOWN=DISABLED BUILD_MODULE_EXT_TRIGGER_CRAWLER_FORM=DISABLED -BUILD_MODULE_EXT_BOTTOM_LINE=DISABLED +BUILD_MODULE_EXT_BOTTOM_LINE=ENABLED +BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW=DISABLED BUILD_MODULE_EXT_BOTTOM_LINE_TIFF_PREVIEW=DISABLED ############################################################################## diff --git a/install-sss.sh b/install-sss.sh new file mode 100755 index 0000000000000000000000000000000000000000..432e1ce9bf6532c0536becf0eaa34ec59884e3f4 --- /dev/null +++ b/install-sss.sh @@ -0,0 +1,13 @@ +SRC_DIR=$1 +INSTALL_DIR=$2 + +mkdir -p $INSTALL_DIR + +# from here on do your module-wise installing + +# ext_table_preview +if [ "${BUILD_MODULE_EXT_TABLE_PREVIEW}" == "ENABLED" ]; then + mkdir -p $INSTALL_DIR/ext_table_preview + cp $SRC_DIR/ext_table_preview/*.py $INSTALL_DIR/ext_table_preview/ + echo "installed all server-side scripts for ext_table_preview" +fi diff --git a/makefile b/makefile index 41c7cf51acaf47caa38a5306af924633801c130e..f35bc3c3f678144b006f08c72df85a99f30cd5d0 100644 --- a/makefile +++ b/makefile @@ -33,13 +33,16 @@ SQ=\' ROOT_DIR = $(abspath .) MISC_DIR = $(abspath misc) PUBLIC_DIR = $(abspath public) +SSS_BIN_DIR = $(abspath sss_bin) CONF_CORE_DIR = $(abspath conf/core) CONF_EXT_DIR = $(abspath conf/ext) SRC_CORE_DIR = $(abspath src/core) SRC_EXT_DIR = $(abspath src/ext) +SRC_SSS_DIR = $(abspath src/server_side_scripting) LIBS_DIR = $(abspath libs) TEST_CORE_DIR = $(abspath test/core/) TEST_EXT_DIR = $(abspath test/ext) +TEST_SSS_DIR =$(abspath test/server_side_scripting) LIBS = fonts css/bootstrap.css js/bootstrap.js js/state-machine.js js/jquery.js js/showdown.js js/dropzone.js css/dropzone.css js/loglevel.js js/leaflet.js css/leaflet.css css/images js/leaflet-latlng-graticule.js js/proj4.js js/proj4leaflet.js js/leaflet-coordinates.js css/leaflet-coordinates.css js/leaflet-graticule.js js/bootstrap-select.js css/bootstrap-select.css js/bootstrap-autocomplete.min.js js/plotly.js js/pako.js js/utif.js TEST_LIBS = $(LIBS) js/qunit.js css/qunit.css $(subst $(TEST_CORE_DIR)/,,$(shell find $(TEST_CORE_DIR)/)) @@ -49,9 +52,9 @@ LIBS_SUBDIRS = $(addprefix $(LIBS_DIR)/, js css fonts) ALL: install -install: clean cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) build_properties merge_xsl +install: clean install-sss cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) build_properties merge_xsl -test: clean cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) build_properties merge_xsl +test: clean install-sss cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) build_properties merge_xsl @for f in $(shell find $(TEST_EXT_DIR) -type f -iname *.js) ; do \ sed -i "/EXTENSIONS/a \<script src=\"$${f#$(TEST_EXT_DIR)/}\" ></script>" $(PUBLIC_DIR)/index.html ; \ echo include $$f; \ @@ -129,6 +132,18 @@ run-qunit: test exit 1; \ fi +install-sss: + @set -a -e ; \ + pushd build.properties.files ; \ + for f in ../build.properties.d/* ; do source "$$f" ; done ; \ + popd ; \ + ./install-sss.sh $(SRC_SSS_DIR) $(SSS_BIN_DIR) + +PYTEST ?= pytest-3 +test-sss: install-sss + $(PYTEST) -vv $(TEST_SSS_DIR) + + CMD_COPY_EXT_FILES = cp -i -r -L cp-ext: # TODO FIXME Base path for not-XSL-expanded files @@ -277,6 +292,7 @@ $(addprefix $(LIBS_DIR)/, js css): .PHONY: clean clean: + $(RM) -r $(SSS_BIN_DIR) $(RM) -r $(PUBLIC_DIR) for f in $(LIBS_SUBDIRS); do unlink $$f || $(RM) -r $$f || true; done for f in $(patsubst %.zip,%/,$(LIBS_ZIP)); do $(RM) -r $$f; done @@ -287,11 +303,7 @@ unzip: for f in $(LIBS_ZIP); do unzip -q -o -d libs $$f; done -PYLINT = pylint3 -d all -e E,F +PYLINT ?= pylint3 PYTHON_FILES = $(subst $(ROOT_DIR)/,,$(shell find $(ROOT_DIR)/ -iname "*.py")) pylint: $(PYTHON_FILES) - for f in $(PYTHON_FILES); do $(PYLINT) $$f || exit 1; done - -PYLINT_LOCAL = /usr/bin/pylint3 -d all -e E,F -pylint-local: $(PYTHON_FILES) - for f in $(PYTHON_FILES); do $(PYLINT_LOCAL) $$f || exit 1; done + for f in $(PYTHON_FILES); do $(PYLINT) -d all -e E,F $$f || exit 1; done diff --git a/src/core/js/ext_bottom_line.js b/src/core/js/ext_bottom_line.js index 4f563f049da55a532c663350cc2b4ca50e09104a..ff448f19e707bfcc4b28e2c1803c0b21defab32d 100644 --- a/src/core/js/ext_bottom_line.js +++ b/src/core/js/ext_bottom_line.js @@ -43,8 +43,9 @@ * @requires getEntityPath (function from caosdb.js) * @requires connection (module from webcaosdb.js) * @requires UTIF (from utif.js library) + * @requires ext_table_preview (module from ext_table_preview.js) */ -var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntityPath, connection, UTIF) { +var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntityPath, connection, UTIF, ext_table_preview) { /** * @property {string|function} create - a function with one parameter @@ -131,6 +132,22 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit } } + const BottomLineWarning = function (arg) { + this._is_bottom_line_error = true; + + if (arg.message) { + // arg is an Error object + this.message = arg.message; + this.stack = arg.stack; + } else { + this.message = arg; + } + + this.to_html = function() { + return $(`<div>${this.message}<div>`)[0]; + } + } + /** * Create a preview for tiff files. * @@ -210,11 +227,15 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit is_applicable: (entity) => _path_has_file_extension( entity, ["mp4", "mov", "webm"]), create: _create_video_preview, + }, { // tables + id: "_default_creators.table_preview", + is_applicable: (e) => ext_table_preview.is_table(e), + create: (e) => ext_table_preview.get_preview(e), }, { // fallback id: "_default_creators.fallback", is_applicable: (entity) => true, create: (entity) => fallback_preview, - }, + } ]; @@ -539,8 +560,11 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit _css_class_preview_container_button, _css_class_preview_container_resolvable, BottomLineError: BottomLineError, + BottomLineWarning: BottomLineWarning, } -}($, log.getLogger("ext_bottom_line"), resolve_references.is_in_viewport_vertically, load_config, getEntityPath, connection, UTIF); +}($, log.getLogger("ext_bottom_line"), + resolve_references.is_in_viewport_vertically, load_config, getEntityPath, + connection, UTIF, ext_table_preview); /** diff --git a/src/core/js/ext_table_preview.js b/src/core/js/ext_table_preview.js new file mode 100644 index 0000000000000000000000000000000000000000..1d9da6fa9334a52de5eb39a66b585c7eb67cd120 --- /dev/null +++ b/src/core/js/ext_table_preview.js @@ -0,0 +1,97 @@ +/* + * ** header v3.0 + * This file is a part of the CaosDB Project. + * + * Copyright (C) 2020 IndiScale GmbH <info@indiscale.com> + * Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * ** end header + */ + +'use strict'; + +/** + * The ext_table_preview module provides a very basic preview for table files. + * + * The preview is generated using a server side script. + * + * @module ext_table_preview + * @version 0.1 + * + * @requires jQuery + * @requires log + * @requires getEntityPath + * @requires getEntityID + * @requires markdown + */ +var ext_table_preview = function ($, logger, connection, getEntityPath, getEntityID, markdown) { + + const get_preview = async function (entity) { + try { + const script_result = await connection.runScript("ext_table_preview/pandas_table_preview.py", + {"-p0": getEntityID(entity)} + ); + + const code = script_result.getElementsByTagName("script")[0].getAttribute("code"); + if (parseInt(code) > 1) { + return script_result.getElementsByTagName("stderr")[0] + } else if (parseInt(code) != 0) { + throw ("An error occurred during execution of the server-side " + + "script:\n" + + script_result.getElementsByTagName("stderr")[0]); + } else { + const tablecontent = script_result.getElementsByTagName("stdout")[0]; + const unformatted = markdown.textToHtml(tablecontent.textContent) + const formatted = $('<div class="table-responsive"/>').append(unformatted); + formatted.find("table").addClass("table table-bordered table-condensed").removeAttr("border"); + return formatted[0]; + } + } catch (err) { + if (err.message && err.message.indexOf && err.message.indexOf("HTTP status 403") > -1) { + throw new ext_bottom_line.BottomLineWarning("You are not allowed to generate the table preview. Please log in."); + } else { + throw err; + } + } + }; + + const is_table = function (entity) { + const path = getEntityPath(entity); + return path && (path.toLowerCase().endsWith('.xls') + || path.toLowerCase().endsWith('.xlsx') + || path.toLowerCase().endsWith('.csv') + || path.toLowerCase().endsWith('.tsv')); + }; + + const init = function () { + // only enable when init is being called + ext_table_preview.is_table = is_table; + }; + + return { + init: init, + get_preview: get_preview, + is_table: () => false, + }; + +}($, log.getLogger("ext_table_preview"), connection, getEntityPath, getEntityID, markdown); + +// this will be replaced by require.js in the future. +$(document).ready(function () { + if ("${BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW}" == "ENABLED") { + caosdb_modules.register(ext_table_preview); + } +}); diff --git a/src/core/xsl/main.xsl b/src/core/xsl/main.xsl index 91f75733d7746c9e99bf5501a3cbb77155cddea3..b915f9585a71ca7c8c7183b2c7564440718bbe10 100644 --- a/src/core/xsl/main.xsl +++ b/src/core/xsl/main.xsl @@ -180,6 +180,11 @@ <xsl:value-of select="concat($basepath,'webinterface/${BUILD_NUMBER}/js/ext_references.js')"/> </xsl:attribute> </xsl:element> + <xsl:element name="script"> + <xsl:attribute name="src"> + <xsl:value-of select="concat($basepath,'webinterface/${BUILD_NUMBER}/js/ext_table_preview.js')"/> + </xsl:attribute> + </xsl:element> <xsl:element name="script"> <xsl:attribute name="src"> <xsl:value-of select="concat($basepath,'webinterface/${BUILD_NUMBER}/js/ext_xls_download.js')"/> diff --git a/src/server_side_scripting/ext_table_preview/pandas_table_preview.py b/src/server_side_scripting/ext_table_preview/pandas_table_preview.py new file mode 100755 index 0000000000000000000000000000000000000000..c0659d9b1839c43e0629a878d792c414577ea344 --- /dev/null +++ b/src/server_side_scripting/ext_table_preview/pandas_table_preview.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +""" +This script tries to read typical table data files (.csv etc.) with pandas and +creates a html (partial) representation of the table. +""" + +import logging +import os +import sys +from datetime import datetime + +import caosdb as db +import pandas as pd +from caosadvancedtools.serverside.helper import get_argument_parser +from caosadvancedtools.serverside.logging import configure_server_side_logging + +MAXIMUMFILESIZE = 1e8 +VALID_ENDINGS = [".csv", ".tsv", ".xls", ".xlsx"] + + +def get_file(eid): + """ retrieves the file entity from caosdb """ + try: + fi = db.File(id=eid) + fi.retrieve() + except db.exceptions.EntityDoesNotExistError: + print("Cannot create preview for Entity with ID={}, because it seems" + "not to exist.".format(eid), file=sys.stderr) + sys.exit(1) + + return fi + + +def size_is_ok(fi): + """ show previews only for files that are not too large """ + + return fi.size <= MAXIMUMFILESIZE + + +def get_ending(fipath): + """ return which of the valid endings (tsv etc.) is the one present""" + + for end in VALID_ENDINGS: + if fipath.lower().endswith(end): + return end + + return None + + +def ending_is_valid(fipath): + """ return whether the ending indicates a file type that can be treated""" + + return get_ending(fipath) is not None + + +def read_file(fipath, ftype): + """ tries to read the provided file """ + + try: + if ftype in [".xls", ".xlsx"]: + df = pd.read_excel(fipath) + elif ftype == ".tsv": + df = pd.read_csv(fipath, sep="\t", comment="#") + elif ftype == ".csv": + df = pd.read_csv(fipath, comment="#") + else: + print("File type unknown: {}".format(ftype)) + raise RuntimeError("") + except Exception: + raise ValueError() + + return df + + +def create_table_preview(fi): + if not ending_is_valid(fi.path): + print("Cannot create preview for Entity with ID={}, because download" + "failed.".format(entity_id), file=sys.stderr) + sys.exit(5) + + ending = get_ending(fi.path) + + if not size_is_ok(fi): + print("Skipped creating a preview for Entity with ID={}, because the" + "file is large!".format(entity_id), file=sys.stderr) + sys.exit(2) + + try: + tmpfile = fi.download() + except Exception: + print("Cannot create preview for Entity with ID={}, because download" + "failed.".format(entity_id), file=sys.stderr) + + sys.exit(3) + + try: + df = read_file(tmpfile, ending) + except ValueError: + print("Cannot read File Entity with ID={}.".format(entity_id), + file=sys.stderr) + sys.exit(4) + + print(df.to_html(max_cols=10, max_rows=10)) + + +if __name__ == "__main__": + conlogger = logging.getLogger("connection") + conlogger.setLevel(level=logging.ERROR) + + parser = get_argument_parser() + args = parser.parse_args() + + debug_file = configure_server_side_logging() + logger = logging.getLogger("caosadvancedtools") + + db.configure_connection(auth_token=args.auth_token) + entity_id = args.filename + + fi = get_file(entity_id) + + create_table_preview(fi) diff --git a/test/core/index.html b/test/core/index.html index c57f6c1f46dd5ff18c6d3508d135c96b10d7b198..fc8c21d6f7230024a4fa9f4edce11b7bb797ffc2 100644 --- a/test/core/index.html +++ b/test/core/index.html @@ -66,6 +66,7 @@ <script src="js/proj4.js"></script> <script src="js/proj4leaflet.js"></script> <script src="js/ext_map.js"></script> + <script src="js/ext_table_preview.js"></script> <script src="js/ext_bottom_line.js"></script> <script src="js/ext_revisions.js"></script> <script src="js/autocomplete.js"></script> diff --git a/test/core/js/modules/ext_bottom_line.js.js b/test/core/js/modules/ext_bottom_line.js.js index a26cc2188eacef210d1acc9eb7de59c53f06b3e7..825c0d921414cb583f7a8e3f8a957db6abb06cff 100644 --- a/test/core/js/modules/ext_bottom_line.js.js +++ b/test/core/js/modules/ext_bottom_line.js.js @@ -67,7 +67,7 @@ var ext_bottom_line_test_suite = function ($, ext_bottom_line, QUnit) { }); QUnit.test("_creators", function (assert) { - assert.equal(ext_bottom_line._creators.length, 8, "eight creators"); + assert.equal(ext_bottom_line._creators.length, 9, "nine creators, 5 default ones, 4 from these tests."); }); QUnit.test("add_preview_container", function(assert) { diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index a19843fb3de11bbedf1c01f71619470bcc99f75c..bccd94fe14f49a79d8a43b559b3b857ed1d7d07d 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -1,7 +1,8 @@ FROM debian:latest RUN apt-get update && \ apt-get install firefox-esr gettext-base pylint3 python3-pip \ - python3-httpbin git curl x11-apps xvfb unzip -y -RUN git clone -b dev https://gitlab.com/caosdb/caosdb-pylib.git && \ - cd caosdb-pylib && pip3 install . + python3-httpbin git curl x11-apps xvfb unzip -y python3-pytest +RUN pip3 install caosdb +RUN pip3 install pandas xlrd +RUN pip3 install git+https://gitlab.com/caosdb/caosdb-advanced-user-tools.git@dev diff --git a/test/server_side_scripting/ext_table_preview/__pycache__/pandas_table_preview.cpython-37.pyc b/test/server_side_scripting/ext_table_preview/__pycache__/pandas_table_preview.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..613d9dce64b94c3b4c66891f22cd02a6c337dff6 Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/__pycache__/pandas_table_preview.cpython-37.pyc differ diff --git a/test/server_side_scripting/ext_table_preview/__pycache__/test_pandas_table_preview.cpython-37-pytest-6.0.2.pyc b/test/server_side_scripting/ext_table_preview/__pycache__/test_pandas_table_preview.cpython-37-pytest-6.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3563a411e0c836d2613ab7189dc6833be735e00 Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/__pycache__/test_pandas_table_preview.cpython-37-pytest-6.0.2.pyc differ diff --git a/test/server_side_scripting/ext_table_preview/data/bad.csv b/test/server_side_scripting/ext_table_preview/data/bad.csv new file mode 100644 index 0000000000000000000000000000000000000000..d29a9312a387186beb8bf4f77a8ec0e4b0ab80fa Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/data/bad.csv differ diff --git a/test/server_side_scripting/ext_table_preview/data/bad.tsv b/test/server_side_scripting/ext_table_preview/data/bad.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d29a9312a387186beb8bf4f77a8ec0e4b0ab80fa Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/data/bad.tsv differ diff --git a/test/server_side_scripting/ext_table_preview/data/bad.xls b/test/server_side_scripting/ext_table_preview/data/bad.xls new file mode 100644 index 0000000000000000000000000000000000000000..1f31bf2754258e3d07f88fd1e6bdee4d7b11bee1 Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/data/bad.xls differ diff --git a/test/server_side_scripting/ext_table_preview/data/bad.xlsx b/test/server_side_scripting/ext_table_preview/data/bad.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1f31bf2754258e3d07f88fd1e6bdee4d7b11bee1 Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/data/bad.xlsx differ diff --git a/test/server_side_scripting/ext_table_preview/data/server_error.csv b/test/server_side_scripting/ext_table_preview/data/server_error.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e770df012f65d73ce4721a5f65d7e3f39959519 --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/data/server_error.csv @@ -0,0 +1 @@ +Hi, this line contains a unicode backspace. This causes a server error, when pandas_table_preview.py's output is serialized into XML. \ No newline at end of file diff --git a/test/server_side_scripting/ext_table_preview/data/test.csv b/test/server_side_scripting/ext_table_preview/data/test.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c9bfd1354393439f551021cfe340577433ce2aa --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/data/test.csv @@ -0,0 +1,12 @@ +# test header +# two lines +A1,B1,C1,D1,E1,F1,G1,H1,I1,J1,K1,L1,M1,N1,O1,P1,Q1,R1,S1,T1 +A2,B2,C2,D2,E2,F2,G2,H2,I2,J2,K2,L2,M2,N2,O2,P2,Q2,R2,S2,T2 +A3,B3,csvfile,D3,E3,F3,G3,H3,I3,J3,K3,L3,M3,N3,O3,P3,Q3,R3,S3,T3 +A5,B5,C5,D5,E5,F5,G5,H5,I5,J5,K5,L5,M5,N5,O5,P5,Q5,R5,S5,T5 +A6,B6,C6,D6,E6,F6,G6,H6,I6,J6,K6,L6,M6,N6,O6,P6,Q6,R6,S6,T6 +A7,B7,csvfile,D7,E7,F7,G7,H7,I7,J7,K7,L7,M7,N7,O7,P7,Q7,R7,S7,T7 +A8,B8,C8,D8,E8,F8,G8,H8,I8,J8,K8,L8,M8,N8,O8,P8,Q8,R8,S8,T8 +A9,B9,C9,D9,E9,F9,G9,H9,I9,J9,K9,L9,M9,N9,O9,P9,Q9,R9,S9,T9 +A10,B10,C10,D10,E10,F10,G10,H10,I10,J10,K10,L10,M10,N10,O10,P10,Q10,R10,S10,T10 +A11,B11,C11,D11,E11,F11,G11,H11,I11,J11,K11,L11,M11,N11,O11,P11,Q11,R11,S11,T11 diff --git a/test/server_side_scripting/ext_table_preview/data/test.tsv b/test/server_side_scripting/ext_table_preview/data/test.tsv new file mode 100644 index 0000000000000000000000000000000000000000..863f692bf64e7dcabf74703587a93e37adf27e67 --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/data/test.tsv @@ -0,0 +1,12 @@ +# test header +# two lines +A1 B1 C1 D1 E1 F1 G1 H1 I1 J1 K1 L1 M1 N1 O1 P1 Q1 R1 S1 T1 +A2 B2 C2 D2 E2 F2 G2 H2 I2 J2 K2 L2 M2 N2 O2 P2 Q2 R2 S2 T2 +A3 B3 csvfile D3 E3 F3 G3 H3 I3 J3 K3 L3 M3 N3 O3 P3 Q3 R3 S3 T3 +A5 B5 C5 D5 E5 F5 G5 H5 I5 J5 K5 L5 M5 N5 O5 P5 Q5 R5 S5 T5 +A6 B6 C6 D6 E6 F6 G6 H6 I6 J6 K6 L6 M6 N6 O6 P6 Q6 R6 S6 T6 +A7 B7 tsvfile D7 E7 F7 G7 H7 I7 J7 K7 L7 M7 N7 O7 P7 Q7 R7 S7 T7 +A8 B8 C8 D8 E8 F8 G8 H8 I8 J8 K8 L8 M8 N8 O8 P8 Q8 R8 S8 T8 +A9 B9 C9 D9 E9 F9 G9 H9 I9 J9 K9 L9 M9 N9 O9 P9 Q9 R9 S9 T9 +A10 B10 C10 D10 E10 F10 G10 H10 I10 J10 K10 L10 M10 N10 O10 P10 Q10 R10 S10 T10 +A11 B11 C11 D11 E11 F11 G11 H11 I11 J11 K11 L11 M11 N11 O11 P11 Q11 R11 S11 T11 diff --git a/test/server_side_scripting/ext_table_preview/data/test.xls b/test/server_side_scripting/ext_table_preview/data/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..a355756b9ab72f9035246c5303800a2076d9bfc0 Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/data/test.xls differ diff --git a/test/server_side_scripting/ext_table_preview/data/test.xlsx b/test/server_side_scripting/ext_table_preview/data/test.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..bc291f1aa86cd6d550320f07a7ce69cf813b8116 Binary files /dev/null and b/test/server_side_scripting/ext_table_preview/data/test.xlsx differ diff --git a/test/server_side_scripting/ext_table_preview/data/xss_attack.csv b/test/server_side_scripting/ext_table_preview/data/xss_attack.csv new file mode 100644 index 0000000000000000000000000000000000000000..e7d43505aef42c397f1859805bc87aab8b6da1a2 --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/data/xss_attack.csv @@ -0,0 +1,8 @@ +# as it seems all these characters are escaped correctly. +"","%3C","<","<","<","<","<","<","<","<" +"<","<","<","<","<","<","<","<","<","<" +"<","<","<","<","<","<","<","<","<","<" +"<","<","<","<","<","<","<","<","<","<" +"<","<","<","<","<","<","<","<","<","<" +"<","<","<","<","<","<","<","<","<","<" +"<","<","<","<","<","<","\x3c","\x3C","\u003c","\u003C" diff --git a/test/server_side_scripting/ext_table_preview/pandas_table_preview.py b/test/server_side_scripting/ext_table_preview/pandas_table_preview.py new file mode 120000 index 0000000000000000000000000000000000000000..f24b3901ce8610fd02fc28468b747b7171834307 --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/pandas_table_preview.py @@ -0,0 +1 @@ +../../../src/server_side_scripting/ext_table_preview/pandas_table_preview.py \ No newline at end of file diff --git a/test/server_side_scripting/ext_table_preview/requirements.txt b/test/server_side_scripting/ext_table_preview/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4628529ba9dce50a08d574e21d3b4a71b50af2b1 --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/requirements.txt @@ -0,0 +1,3 @@ +caosdb +caosadvancedtools +pandas diff --git a/test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py b/test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py new file mode 100644 index 0000000000000000000000000000000000000000..00d1c7f38746abe437abc76cd51b29600adcd049 --- /dev/null +++ b/test/server_side_scripting/ext_table_preview/test_pandas_table_preview.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +import os +import unittest + +import caosdb as db +from caosdb.common.models import _parse_single_xml_element +from lxml import etree +from pandas_table_preview import (MAXIMUMFILESIZE, create_table_preview, + ending_is_valid, read_file, size_is_ok) + + +class PreviewTest(unittest.TestCase): + def test_file_ending(self): + self.assertFalse(ending_is_valid("/this/is/no/xls.lol")) + self.assertFalse(ending_is_valid("xls.lol")) + self.assertFalse(ending_is_valid("ag.xls.lol")) + assert ending_is_valid("/this/is/a/lol.xls") + assert ending_is_valid("/this/is/a/lol.csv") + assert ending_is_valid("/this/is/a/lol.cSv") + assert ending_is_valid("/this/is/a/lol.CSV") + assert ending_is_valid("lol.CSV") + + def test_file_size(self): + entity_xml = ('<File id="1234" name="SomeFile" ' + 'path="/this/path.tsv" size="{size}"></File>') + small = _parse_single_xml_element( + etree.fromstring(entity_xml.format(size="20000"))) + + assert size_is_ok(small) + large = _parse_single_xml_element( + etree.fromstring(entity_xml.format( + size=str(int(MAXIMUMFILESIZE+1))))) + assert not size_is_ok(large) + + def test_output(self): + files = [os.path.join(os.path.dirname(__file__), "data", f) + for f in ["test.csv", "test.tsv", "test.xls", "test.xlsx"]] + + for fi in files: + table = read_file(fi, ftype="."+fi.split(".")[-1]) + searchkey = fi.split(".")[-1]+"file" + print(table) + assert (table == searchkey).any(axis=None) + + badfiles = [os.path.join(os.path.dirname(__file__), "data", f) + for f in ["bad.csv", "bad.tsv", "bad.xls", "bad.xlsx"]] + + for bfi in badfiles: + self.assertRaises(ValueError, read_file, + bfi, "."+bfi.split(".")[-1])