Merge branch 'integrationtest' into 'master'

Integrationtest

See merge request bmp-caosdb/caosdb-advanced-user-tools!4

.docker/Dockerfile

+FROM debian:latest
+RUN apt-get update && \
+	 apt-get install curl python3 python3-pip git python-autopep8 \
+     python3-pytest -y
+COPY .docker/wait-for-it.sh /wait-for-it.sh
+RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-pylib.git && \
+   cd caosdb-pylib && pip3 install .
+RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-models.git && \
+   cd caosdb-models && pip3 install .
+ADD https://gitlab.com/api/v4/projects/13601752/repository/branches \
+	scifolder_version.txt
+RUN git clone https://gitlab.com/henrik_indiscale/scifolder.git && \
+   cd scifolder && pip3 install .
+COPY . /git
+RUN rm -r /git/.git && mv /git/.docker/pycaosdb.ini /git/integrationtests 
+RUN cd /git && pip3 install .
+WORKDIR /git/integrationtests/
+CMD /wait-for-it.sh caosdb-server:10443 -t 120 -- ./test.sh

.docker/cert/caosdb.cert.pem

+-----BEGIN CERTIFICATE-----
+MIIDETCCAfmgAwIBAgIUM1S4TR6nEsstWuXDjZGBv6sb0LAwDQYJKoZIhvcNAQEL
+BQAwGDEWMBQGA1UEAwwNY2Fvc2RiLXNlcnZlcjAeFw0xOTA3MTgxNDM3MzNaFw0x
+OTA4MTcxNDM3MzNaMBgxFjAUBgNVBAMMDWNhb3NkYi1zZXJ2ZXIwggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMFzou21Jugj9Z3YUkhXANt2oGNvuCyt9S
+IfBZO5ZpgwSTcRs8UAuKDYAMmGMeQtqLfqEYoQFEeybSd9NsrFb0IgrUaKrSNAzs
+phjsGvXU1UbMRw0vva3l6r94ceM8ecw+x2RIsTwfJkfVL60XMm9DvTaQFLzjcKLf
+lOQbXG6jgCHtvO5MThk/C+TvsAZlRDErp7HgDqDFg+jnJC5YsEFQQyAYCLJXOotQ
+A5FnRbT8dtoDHy03mHyed5Ji3IgxKxE5mF0ygG8/FRfgzqImg4FMnCb+KBwom2xh
+y2hvtFnm3+kCbB+waDTUaPZwvNUONUZMfcl3PggX0IwipqWL8DBJAgMBAAGjUzBR
+MB0GA1UdDgQWBBSglY56eKgLeyW1o5UdZgqlzDJJEjAfBgNVHSMEGDAWgBSglY56
+eKgLeyW1o5UdZgqlzDJJEjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
+A4IBAQAF4sn/C7wx9fq/9/v6UYb9DbOXaonS+1jOzf1lAOEwHtJh0M9SDTqvaeQz
+bshJ33ZFbwH3qIq7+OzGj9Yon9LN1GGEl4CcKKQRQQkjo7jt7X47C6PkrW6l23+i
+OrDRk7Oke9BJS/ZRKLmxvlaLYULpj02lOsbddfE4igXxRPCiouosc+HQ3kd6Mqhv
+5Gt0Yki3z4fvi+CWrtUjxYeMUJkraFAyKDKwFVZgDnWwyipGo9yh3a6iXD7duQcx
+FRk7zm1oaF3ia9hFfU1AVDfkgIlPLWW30CT4Jgi25/wpiGZiT1uXAgfoJCEDoYXp
+bQIM5KQqw0Kgjqc/KuLCRe1KeCzq
+-----END CERTIFICATE-----

.docker/cert/caosdb.key.pem

+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-256-CBC,0B42601A73984CD9F1DF2F786F894EA0
+
+5ZY5CR8ho3a9449XIUmSpIsCZZWsVQ1pWX+SJPSUlk2A9N3BHdY0AdvCx+sTF7p4
+bSoAulnAEFlIeXqimtK+InH0d3HLbvn41gvEbFTPjZdejxro6m9DvSHCpyDZK9TH
+m5A2aRsJioeibETUA7uvld/d9Szdz//GPN2YzWqJpzXZtTWHH/tR2J2L6KE+wBDa
+IViryBCv0+9e6l2WeEjXqqYZhznR/gUsxc2xEs7x8wEpWBVlmVzYWuSt7OA5oOm/
+r1OAR9LEH0PIISUv2h+P7ccOZJD/JE2YDM3IBdClxLv2MRyj/88ougcV5gv688+f
+6XxT2+LfCLd+viqGcFZ5a6tT1wpUthVWWvUWPna6SlVVrsj/7bCc6y2Jptff+Kqv
+eL8Gl1dWZXwe0JXr+Ss5KfIddbY0pJv8FHjCsN+Nkdozo7Wdufm1Xkw+s1f2td6u
+bHwbo1Y4mX5nm9I2OkAMhGubSPalGZRUvr9cfl1/1V/QgX0jPmJmNmH5yFUdc5MQ
+QIdcpZmz8xIH1XAdJWTbzoYKzKnXTHpZFaZL1acMrK2CERMxDStfe9W/DSRy6/qn
+AsQbh6tDk+Fe05EQwpZBZQ1YlrfAru7QDaDE9+uO+mxkf1UHynfPHNp2jZAe6XBt
+p97GXUruLR8rBOAGX19EuoG2sSek44dLW5IsLGbR1MpTlq48YZv2z3TrESNJHKAM
+VAxg5kwqvYocykdFrKmggbdBv3oo6g59RxiufKparlkI7YZFhks6V1gujKW+JChc
+Eoy8e2EJJP0pY0Pb3ZE9QkK7ZX4otXNRgqqnf2f7kTiGMRBSDSD5xGUw0syXAS3G
+Ji/4pMFh8ALsYUOOOYUISWAUB6O8/438SIq+g+TNHYVbrli/WK9ip0+RGPmJ+Ulm
+iWQMQOcsSYTBqDytvr0241JDBuq9RjMBJE4FO2ltr7L4gmkN8+d/WGnvu4onJBoE
+8ApriArJjuHBP1iex3uygJKD3sdiuAuukrPlEla1xjywH3QSzj/SWABIu4QJAr3a
+70jEFGL04eblvBPGN2hnXEQCxHqRJLpFH1huWMlO3PuoRnGxIuxymkEOOEWfswLO
+r9qLbkUWmFFFqJBNB3T4kfzuZDqiCQnS8+QNvhTiKCWLcUswXTirSac8B5cWDaqd
+/Nf119n3ezZkpzgKr0kw+kLxlU2Tbl7UBZ7TRadxklXaAmEo7z+Wh7WKmbcUMqit
+xh6P7P0ZWcLYIiXCC5gYr6jzU+Rjx4ZGF5+Q0LcXC3Ai6Ujz4zdrVRLBiw27zKgB
+RsYBCYRmQ2qO0AzHFBWixl7IVGYi+QFKU0SFrPLzme+yOJjjLAv2Xzu24KFiwHLz
+8MWCd0yM8UjXAql+jCk54O6e4QwER2Rfr6vxlH6x90diy8eJQXejBEVNIfyOpyST
+Mqn2jc5vWanVxRk/SgrZ0pOJ8TBeGSzolJF9MBPjZ9PJTbu3WzJpuguQ4CeAAV7E
+0+rbPGALnuDKmrsnyIb9WX+fEpT7oBPEKV84XyG1+UkBrfDaQvK3SujeRvnOaZLG
+uSNc8yX9CZmt2OJTRWK5fKnR5fGzAnMR7cRUYNvcWJ0T6B4wj8cwF4wY9laGzMO7
+-----END RSA PRIVATE KEY-----

.docker/docker-compose.yml

+version: '3.7'
+services:
+  sqldb:
+    image: mariadb:10.4
+    environment:
+      MYSQL_ROOT_PASSWORD: caosdb1234
+    networks:
+      - caosnet
+  caosdb-server:
+    image: "$CI_REGISTRY/caosdb:$CAOSDB_TAG"
+    user: 999:999
+    depends_on:
+      - sqldb
+    networks:
+      - caosnet
+    volumes:
+      - type: bind
+        source: "$EXEPATH/.docker/cert"
+        target: /opt/caosdb/cert
+      - type: bind
+        source: "$EXEPATH/integrationtests/extroot"
+        target: /opt/caosdb/mnt/extroot
+        read_only: true
+    ports:
+      # - "from_outside:from_inside"
+      - "10443:10443"
+      - "10080:10080"
+    environment:
+      DEBUG: 1
+
+networks:
+  caosnet:
+    driver: bridge

.docker/pycaosdb.ini

+[IntegrationTests]
+test_server_side_scripting.bin_dir=../caosdb-server/test_scripting/bin/
+
+[Connection]
+url=https://caosdb-server:10443
+username=admin
+cacert=../.docker/cert/caosdb.cert.pem
+#cacert=/etc/ssl/cert.pem
+debug=0
+
+passwordmethod=plain
+password=caosdb
+
+ssl_insecure=True
+timeout=500
+[Container]
+debug=0

.docker/tester.yml

+version: '3.6'
+services:
+  tester:
+    image: "$CI_REGISTRY_IMAGE:latest"
+    networks:
+      - docker_caosnet
+networks:
+  docker_caosnet:
+    external: true

.docker/wait-for-it.sh

+#!/usr/bin/env bash
+# License: 
+# From https://github.com/vishnubob/wait-for-it
+# The MIT License (MIT)
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+        WAITFORIT_ISBUSY=1
+        WAITFORIT_BUSYTIMEFLAG="-t"
+
+else
+        WAITFORIT_ISBUSY=0
+        WAITFORIT_BUSYTIMEFLAG=""
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi
+

.gitignore

+__pycache__
+.tox
 .coverage
+cache.db
 *.egg-info
-.tox
-__pycache__
-

.gitlab-ci.yml

+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019 Henrik tom Wörden 
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+variables:
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-advanced-testenv
+  # With dind the docker deamon is available on the network
+   DOCKER_HOST: tcp://docker:2375/
+   # When using dind, it's wise to use the overlayfs driver for
+   # improved performance.
+   DOCKER_DRIVER: overlay2
+
+services:
+  - docker:dind
+
+stages: 
+  - setup
+  - style
+  - test
+
+test:
+  tags: [docker]
+  stage: test
+  image: 
+    name: docker/compose:1.24.1
+    entrypoint: ["/bin/sh", "-c"]
+  script:
+      - if [[ "$CAOSDB_TAG" == "" ]]; then
+            CAOSDB_TAG=master-latest;
+        fi
+      - echo $CAOSDB_TAG
+      - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+      - docker pull $CI_REGISTRY_IMAGE:latest
+      - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose 
+        -f .docker/docker-compose.yml up -d
+      - docker-compose -f .docker/tester.yml run tester 
+      - rc=$?
+      - docker-compose -f .docker/docker-compose.yml down
+      - exit $rc
+
+build-testenv:
+  tags: [docker]
+  image: docker:latest
+  stage: setup
+  script: 
+      - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+        # use here general latest or specific branch latest...
+      - docker pull $CI_REGISTRY_IMAGE:latest || true
+      - docker build 
+        --pull
+        --file .docker/Dockerfile
+        --cache-from $CI_REGISTRY_IMAGE:latest 
+        -t $CI_REGISTRY_IMAGE:latest .
+      - docker push $CI_REGISTRY_IMAGE:latest
+
+
+style:
+  tags: [docker]
+  stage: style
+  image: $CI_REGISTRY_IMAGE:latest
+  script:
+      - autopep8 -r --diff --exit-code .
+  allow_failure: true

integrationtests/crawl.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+import argparse
+from argparse import RawTextHelpFormatter
+
+import caosdb as db
+
+from caosadvancedtools.crawler import Crawler
+from scifolder import (AnalysisCFood, ExperimentCFood, PublicationCFood,
+                       SimulationCFood)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=RawTextHelpFormatter)
+
+    parser.add_argument("path")
+
+    return parser
+
+
+def access(path):
+    return "extroot" + path
+
+
+if __name__ == "__main__":
+    parser = get_parser()
+    args = parser.parse_args()
+
+    print("Starting query...")
+    files = Crawler.query_files(args.path)
+    print("Query done...")
+    config = db.configuration.get_config()
+    c = Crawler(food=[
+        AnalysisCFood(use_cache=True, access=access),
+        ExperimentCFood(use_cache=True, access=access),
+        PublicationCFood(use_cache=True, access=access),
+        SimulationCFood(use_cache=True, access=access),
+    ])
+    c.crawl(files)

integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md

+---
+responsible:	
+- Only Responsible
+description: 	A description of another example analysis.
+
+data:
+- filename:	"/ExperimentalData/TestProject/2019-02-03/*.dat"
+  description:  an example reference to a results file
+
+scripts:
+- filename: plot.py
+  description: a plotting script
+results:
+- filename: results.pdf
+...

integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md

+---
+responsible:	
+- First Person
+description: 	A description of an example analysis.
+
+results:
+- filename:	"images/*.png"
+  description:  an example reference to a results file
+
+data:
+- filename: /ExperimentalData/TestProject/2019-02-03_something/use*.xlsx
+  description:  an example reference to data files
+
+scripts:
+- analyse.py
+
+tags:
+- collagen
+- time sweep
+- frequency sweep
+...