diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..2188dae2ed495eb2dab789cf8865f795380a2d6f --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,18 @@ +FROM debian:latest +RUN apt-get update && \ + apt-get install curl python3 python3-pip git python-autopep8 \ + python3-pytest -y +COPY .docker/wait-for-it.sh /wait-for-it.sh +RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-pylib.git && \ + cd caosdb-pylib && pip3 install . +RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-models.git && \ + cd caosdb-models && pip3 install . +ADD https://gitlab.com/api/v4/projects/13601752/repository/branches \ + scifolder_version.txt +RUN git clone https://gitlab.com/henrik_indiscale/scifolder.git && \ + cd scifolder && pip3 install . +COPY . /git +RUN rm -r /git/.git && mv /git/.docker/pycaosdb.ini /git/integrationtests +RUN cd /git && pip3 install . +WORKDIR /git/integrationtests/ +CMD /wait-for-it.sh caosdb-server:10443 -t 120 -- ./test.sh diff --git a/.docker/cert/all-certs.pkcs12 b/.docker/cert/all-certs.pkcs12 new file mode 100644 index 0000000000000000000000000000000000000000..382b9cfe7dd1cec7c9f98f96c1fea63a971585ab Binary files /dev/null and b/.docker/cert/all-certs.pkcs12 differ diff --git a/.docker/cert/caosdb.cert.pem b/.docker/cert/caosdb.cert.pem new file mode 100644 index 0000000000000000000000000000000000000000..af44eed0f636bb87b2672219fe14d825b4b5e9b7 --- /dev/null +++ b/.docker/cert/caosdb.cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfmgAwIBAgIUM1S4TR6nEsstWuXDjZGBv6sb0LAwDQYJKoZIhvcNAQEL +BQAwGDEWMBQGA1UEAwwNY2Fvc2RiLXNlcnZlcjAeFw0xOTA3MTgxNDM3MzNaFw0x +OTA4MTcxNDM3MzNaMBgxFjAUBgNVBAMMDWNhb3NkYi1zZXJ2ZXIwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMFzou21Jugj9Z3YUkhXANt2oGNvuCyt9S +IfBZO5ZpgwSTcRs8UAuKDYAMmGMeQtqLfqEYoQFEeybSd9NsrFb0IgrUaKrSNAzs +phjsGvXU1UbMRw0vva3l6r94ceM8ecw+x2RIsTwfJkfVL60XMm9DvTaQFLzjcKLf +lOQbXG6jgCHtvO5MThk/C+TvsAZlRDErp7HgDqDFg+jnJC5YsEFQQyAYCLJXOotQ +A5FnRbT8dtoDHy03mHyed5Ji3IgxKxE5mF0ygG8/FRfgzqImg4FMnCb+KBwom2xh +y2hvtFnm3+kCbB+waDTUaPZwvNUONUZMfcl3PggX0IwipqWL8DBJAgMBAAGjUzBR +MB0GA1UdDgQWBBSglY56eKgLeyW1o5UdZgqlzDJJEjAfBgNVHSMEGDAWgBSglY56 +eKgLeyW1o5UdZgqlzDJJEjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQAF4sn/C7wx9fq/9/v6UYb9DbOXaonS+1jOzf1lAOEwHtJh0M9SDTqvaeQz +bshJ33ZFbwH3qIq7+OzGj9Yon9LN1GGEl4CcKKQRQQkjo7jt7X47C6PkrW6l23+i +OrDRk7Oke9BJS/ZRKLmxvlaLYULpj02lOsbddfE4igXxRPCiouosc+HQ3kd6Mqhv +5Gt0Yki3z4fvi+CWrtUjxYeMUJkraFAyKDKwFVZgDnWwyipGo9yh3a6iXD7duQcx +FRk7zm1oaF3ia9hFfU1AVDfkgIlPLWW30CT4Jgi25/wpiGZiT1uXAgfoJCEDoYXp +bQIM5KQqw0Kgjqc/KuLCRe1KeCzq +-----END CERTIFICATE----- diff --git a/.docker/cert/caosdb.jks b/.docker/cert/caosdb.jks new file mode 100644 index 0000000000000000000000000000000000000000..355b78fd3b60ebdb92b111ee0d81bc6f1adb8add Binary files /dev/null and b/.docker/cert/caosdb.jks differ diff --git a/.docker/cert/caosdb.key.pem b/.docker/cert/caosdb.key.pem new file mode 100644 index 0000000000000000000000000000000000000000..3459474b70fbddb9b2a9080ef402c35db1713c0b --- /dev/null +++ b/.docker/cert/caosdb.key.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,0B42601A73984CD9F1DF2F786F894EA0 + +5ZY5CR8ho3a9449XIUmSpIsCZZWsVQ1pWX+SJPSUlk2A9N3BHdY0AdvCx+sTF7p4 +bSoAulnAEFlIeXqimtK+InH0d3HLbvn41gvEbFTPjZdejxro6m9DvSHCpyDZK9TH +m5A2aRsJioeibETUA7uvld/d9Szdz//GPN2YzWqJpzXZtTWHH/tR2J2L6KE+wBDa +IViryBCv0+9e6l2WeEjXqqYZhznR/gUsxc2xEs7x8wEpWBVlmVzYWuSt7OA5oOm/ +r1OAR9LEH0PIISUv2h+P7ccOZJD/JE2YDM3IBdClxLv2MRyj/88ougcV5gv688+f +6XxT2+LfCLd+viqGcFZ5a6tT1wpUthVWWvUWPna6SlVVrsj/7bCc6y2Jptff+Kqv +eL8Gl1dWZXwe0JXr+Ss5KfIddbY0pJv8FHjCsN+Nkdozo7Wdufm1Xkw+s1f2td6u +bHwbo1Y4mX5nm9I2OkAMhGubSPalGZRUvr9cfl1/1V/QgX0jPmJmNmH5yFUdc5MQ +QIdcpZmz8xIH1XAdJWTbzoYKzKnXTHpZFaZL1acMrK2CERMxDStfe9W/DSRy6/qn +AsQbh6tDk+Fe05EQwpZBZQ1YlrfAru7QDaDE9+uO+mxkf1UHynfPHNp2jZAe6XBt +p97GXUruLR8rBOAGX19EuoG2sSek44dLW5IsLGbR1MpTlq48YZv2z3TrESNJHKAM +VAxg5kwqvYocykdFrKmggbdBv3oo6g59RxiufKparlkI7YZFhks6V1gujKW+JChc +Eoy8e2EJJP0pY0Pb3ZE9QkK7ZX4otXNRgqqnf2f7kTiGMRBSDSD5xGUw0syXAS3G +Ji/4pMFh8ALsYUOOOYUISWAUB6O8/438SIq+g+TNHYVbrli/WK9ip0+RGPmJ+Ulm +iWQMQOcsSYTBqDytvr0241JDBuq9RjMBJE4FO2ltr7L4gmkN8+d/WGnvu4onJBoE +8ApriArJjuHBP1iex3uygJKD3sdiuAuukrPlEla1xjywH3QSzj/SWABIu4QJAr3a +70jEFGL04eblvBPGN2hnXEQCxHqRJLpFH1huWMlO3PuoRnGxIuxymkEOOEWfswLO +r9qLbkUWmFFFqJBNB3T4kfzuZDqiCQnS8+QNvhTiKCWLcUswXTirSac8B5cWDaqd +/Nf119n3ezZkpzgKr0kw+kLxlU2Tbl7UBZ7TRadxklXaAmEo7z+Wh7WKmbcUMqit +xh6P7P0ZWcLYIiXCC5gYr6jzU+Rjx4ZGF5+Q0LcXC3Ai6Ujz4zdrVRLBiw27zKgB +RsYBCYRmQ2qO0AzHFBWixl7IVGYi+QFKU0SFrPLzme+yOJjjLAv2Xzu24KFiwHLz +8MWCd0yM8UjXAql+jCk54O6e4QwER2Rfr6vxlH6x90diy8eJQXejBEVNIfyOpyST +Mqn2jc5vWanVxRk/SgrZ0pOJ8TBeGSzolJF9MBPjZ9PJTbu3WzJpuguQ4CeAAV7E +0+rbPGALnuDKmrsnyIb9WX+fEpT7oBPEKV84XyG1+UkBrfDaQvK3SujeRvnOaZLG +uSNc8yX9CZmt2OJTRWK5fKnR5fGzAnMR7cRUYNvcWJ0T6B4wj8cwF4wY9laGzMO7 +-----END RSA PRIVATE KEY----- diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..97fe0c6429a7f65de7ba5a56d456ffa61abf09cf --- /dev/null +++ b/.docker/docker-compose.yml @@ -0,0 +1,33 @@ +version: '3.7' +services: + sqldb: + image: mariadb:10.4 + environment: + MYSQL_ROOT_PASSWORD: caosdb1234 + networks: + - caosnet + caosdb-server: + image: "$CI_REGISTRY/caosdb:$CAOSDB_TAG" + user: 999:999 + depends_on: + - sqldb + networks: + - caosnet + volumes: + - type: bind + source: "$EXEPATH/.docker/cert" + target: /opt/caosdb/cert + - type: bind + source: "$EXEPATH/integrationtests/extroot" + target: /opt/caosdb/mnt/extroot + read_only: true + ports: + # - "from_outside:from_inside" + - "10443:10443" + - "10080:10080" + environment: + DEBUG: 1 + +networks: + caosnet: + driver: bridge diff --git a/.docker/pycaosdb.ini b/.docker/pycaosdb.ini new file mode 100644 index 0000000000000000000000000000000000000000..b57f2ebd13a37b75a5da3386ff8ded5d1bc9d2f8 --- /dev/null +++ b/.docker/pycaosdb.ini @@ -0,0 +1,17 @@ +[IntegrationTests] +test_server_side_scripting.bin_dir=../caosdb-server/test_scripting/bin/ + +[Connection] +url=https://caosdb-server:10443 +username=admin +cacert=../.docker/cert/caosdb.cert.pem +#cacert=/etc/ssl/cert.pem +debug=0 + +passwordmethod=plain +password=caosdb + +ssl_insecure=True +timeout=500 +[Container] +debug=0 diff --git a/.docker/tester.yml b/.docker/tester.yml new file mode 100644 index 0000000000000000000000000000000000000000..7eb8d4d6e63a540d16dfec128fa53512b10fc5ff --- /dev/null +++ b/.docker/tester.yml @@ -0,0 +1,9 @@ +version: '3.6' +services: + tester: + image: "$CI_REGISTRY_IMAGE:latest" + networks: + - docker_caosnet +networks: + docker_caosnet: + external: true diff --git a/.docker/wait-for-it.sh b/.docker/wait-for-it.sh new file mode 100755 index 0000000000000000000000000000000000000000..d69e99f1f13257b559dce2433de0515379663efa --- /dev/null +++ b/.docker/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# License: +# From https://github.com/vishnubob/wait-for-it +# The MIT License (MIT) +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi + diff --git a/.gitignore b/.gitignore index 046cdde9c7b69eb3456f65c10b731a68e2b3c0e4..51c99625da09cb13797014cd255e40caf2aa85bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ +__pycache__ +.tox .coverage +cache.db *.egg-info -.tox -__pycache__ - diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..fccefd2551dca726861cbadc35db66889fe1ded6 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,83 @@ +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Copyright (C) 2019 Henrik tom Wörden +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +variables: + CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-advanced-testenv + # With dind the docker deamon is available on the network + DOCKER_HOST: tcp://docker:2375/ + # When using dind, it's wise to use the overlayfs driver for + # improved performance. + DOCKER_DRIVER: overlay2 + +services: + - docker:dind + +stages: + - setup + - style + - test + +test: + tags: [docker] + stage: test + image: + name: docker/compose:1.24.1 + entrypoint: ["/bin/sh", "-c"] + script: + - if [[ "$CAOSDB_TAG" == "" ]]; then + CAOSDB_TAG=master-latest; + fi + - echo $CAOSDB_TAG + - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker pull $CI_REGISTRY_IMAGE:latest + - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose + -f .docker/docker-compose.yml up -d + - docker-compose -f .docker/tester.yml run tester + - rc=$? + - docker-compose -f .docker/docker-compose.yml down + - exit $rc + +build-testenv: + tags: [docker] + image: docker:latest + stage: setup + script: + - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + # use here general latest or specific branch latest... + - docker pull $CI_REGISTRY_IMAGE:latest || true + - docker build + --pull + --file .docker/Dockerfile + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:latest . + - docker push $CI_REGISTRY_IMAGE:latest + + +style: + tags: [docker] + stage: style + image: $CI_REGISTRY_IMAGE:latest + script: + - autopep8 -r --diff --exit-code . + allow_failure: true diff --git a/integrationtests/crawl.py b/integrationtests/crawl.py new file mode 100755 index 0000000000000000000000000000000000000000..df76f6cf5d19001f2f5bcb120a6db9f7bf2f16cd --- /dev/null +++ b/integrationtests/crawl.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +import argparse +from argparse import RawTextHelpFormatter + +import caosdb as db + +from caosadvancedtools.crawler import Crawler +from scifolder import (AnalysisCFood, ExperimentCFood, PublicationCFood, + SimulationCFood) + + +def get_parser(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=RawTextHelpFormatter) + + parser.add_argument("path") + + return parser + + +def access(path): + return "extroot" + path + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + + print("Starting query...") + files = Crawler.query_files(args.path) + print("Query done...") + config = db.configuration.get_config() + c = Crawler(food=[ + AnalysisCFood(use_cache=True, access=access), + ExperimentCFood(use_cache=True, access=access), + PublicationCFood(use_cache=True, access=access), + SimulationCFood(use_cache=True, access=access), + ]) + c.crawl(files) diff --git a/integrationtests/extroot/.cerate_dir b/integrationtests/extroot/.cerate_dir new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e557f2035e34f7f96975454223b15ebe0dd6f79c --- /dev/null +++ b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md @@ -0,0 +1,15 @@ +--- +responsible: +- Only Responsible +description: A description of another example analysis. + +data: +- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat" + description: an example reference to a results file + +scripts: +- filename: plot.py + description: a plotting script +results: +- filename: results.pdf +... diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/plot.py b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/plot.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/results.pdf b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/results.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c342053aaa94a474f59a714209b3066c084c706e --- /dev/null +++ b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md @@ -0,0 +1,21 @@ +--- +responsible: +- First Person +description: A description of an example analysis. + +results: +- filename: "images/*.png" + description: an example reference to a results file + +data: +- filename: /ExperimentalData/TestProject/2019-02-03_something/use*.xlsx + description: an example reference to data files + +scripts: +- analyse.py + +tags: +- collagen +- time sweep +- frequency sweep +... diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/analyse.py b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/analyse.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol1.png b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol1.png new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol2.png b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol2.png new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md new file mode 100644 index 0000000000000000000000000000000000000000..348a6fec3c00ad19a28a9e6b86e668aa36cd73eb --- /dev/null +++ b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md @@ -0,0 +1,9 @@ +--- +responsible: +- Only Responsible +description: A description of another example experiment. + +results: +- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat" + description: an example reference to a results file +... diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/datafile.dat b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/datafile.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e63c07cf8a943637c62d63f61faba489aa8c4f0 --- /dev/null +++ b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md @@ -0,0 +1,15 @@ +--- +responsible: +- First Person +- Second Person +description: A description of an example experiment. + +results: +- filename: "*.xlsx" + description: an example reference to a results file + +tags: +- collagen +- time sweep +- frequency sweep +... diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/usefull.xlsx b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/usefull.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/useless.xlsx b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/useless.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md new file mode 100644 index 0000000000000000000000000000000000000000..348a6fec3c00ad19a28a9e6b86e668aa36cd73eb --- /dev/null +++ b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md @@ -0,0 +1,9 @@ +--- +responsible: +- Only Responsible +description: A description of another example experiment. + +results: +- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat" + description: an example reference to a results file +... diff --git a/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/datafile.dat b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/datafile.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e63c07cf8a943637c62d63f61faba489aa8c4f0 --- /dev/null +++ b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md @@ -0,0 +1,15 @@ +--- +responsible: +- First Person +- Second Person +description: A description of an example experiment. + +results: +- filename: "*.xlsx" + description: an example reference to a results file + +tags: +- collagen +- time sweep +- frequency sweep +... diff --git a/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/usefull.xlsx b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/usefull.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/useless.xlsx b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/useless.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md new file mode 100644 index 0000000000000000000000000000000000000000..348a6fec3c00ad19a28a9e6b86e668aa36cd73eb --- /dev/null +++ b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md @@ -0,0 +1,9 @@ +--- +responsible: +- Only Responsible +description: A description of another example experiment. + +results: +- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat" + description: an example reference to a results file +... diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03/sim.py b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/sim.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03/snapshots.dat b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/snapshots.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e63c07cf8a943637c62d63f61faba489aa8c4f0 --- /dev/null +++ b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md @@ -0,0 +1,15 @@ +--- +responsible: +- First Person +- Second Person +description: A description of an example experiment. + +results: +- filename: "*.xlsx" + description: an example reference to a results file + +tags: +- collagen +- time sweep +- frequency sweep +... diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/large_sim.py b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/large_sim.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/parameters.p b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/parameters.p new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/timeseries.npy b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/timeseries.npy new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/integrationtests/filldb.sh b/integrationtests/filldb.sh new file mode 100755 index 0000000000000000000000000000000000000000..906e499c33f2607135713500b3e36cb0d2930dcf --- /dev/null +++ b/integrationtests/filldb.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# fill database +python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/ExperimentalData +python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/DataAnalysis +python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData +python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Publications +python3 insert_model.py +python3 insert_record.py +python3 crawl.py / diff --git a/integrationtests/insert_model.py b/integrationtests/insert_model.py new file mode 100755 index 0000000000000000000000000000000000000000..1dcf5bdb888988089e554dc41bb20fbbe5e60e23 --- /dev/null +++ b/integrationtests/insert_model.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from caosmodels.parser import parse_model_from_yaml + +model = parse_model_from_yaml("model.yml") +model.sync_data_model(noquestion=True) diff --git a/integrationtests/insert_record.py b/integrationtests/insert_record.py new file mode 100644 index 0000000000000000000000000000000000000000..5927b05813ffa85d2dff3045f98e7624acfb0f94 --- /dev/null +++ b/integrationtests/insert_record.py @@ -0,0 +1,5 @@ +import caosdb as db + +r = db.Record(name="TestProject") +r.add_parent(name="Project") +r.insert() diff --git a/integrationtests/model.yml b/integrationtests/model.yml new file mode 100644 index 0000000000000000000000000000000000000000..6fd888dc2f599639ce52af7ff9be17f6a03354fd --- /dev/null +++ b/integrationtests/model.yml @@ -0,0 +1,56 @@ +Experiment: + obligatory_properties: + date: + datatype: DATETIME + description: 'date of the experiment' + identifier: + datatype: TEXT + description: 'identifier of the experiment' + # TODO empty recommended_properties is a problem + #recommended_properties: + responsible: +Project: +Person: + obligatory_properties: + firstName: + datatype: TEXT + description: 'First name of a Person.' + lastName: + datatype: TEXT + description: 'LastName of a Person.' + responsible: +responsible: + datatype: REFERENCE +results: + datatype: REFERENCE +data: + datatype: REFERENCE +scripts: + datatype: REFERENCE +Source: +Simulation: + obligatory_properties: + date: + identifier: + responsible: +Analysis: + obligatory_properties: + date: + identifier: + responsible: +Publication: +Thesis: + inherit_from_suggested: + - Publication +Article: + inherit_from_suggested: + - Publication +Poster: + inherit_from_suggested: + - Publication +Presentation: + inherit_from_suggested: + - Publication +Report: + inherit_from_suggested: + - Publication diff --git a/integrationtests/test.sh b/integrationtests/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..8774500a2301930d2a05a3b7301eb27d99e4e88f --- /dev/null +++ b/integrationtests/test.sh @@ -0,0 +1,2 @@ +./filldb.sh +py.test-3 test_crawler.py diff --git a/integrationtests/test_crawler.py b/integrationtests/test_crawler.py new file mode 100755 index 0000000000000000000000000000000000000000..5797ba42cb795b362ba7271a2bfced8109b402aa --- /dev/null +++ b/integrationtests/test_crawler.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +import os +import unittest + +import caosdb as db + + +def get_entity_with_id(eid): + return db.execute_query("FIND "+str(eid), unique=True) + + +class CrawlerTest(unittest.TestCase): + def test_experiment(self): + ######################## + # # first experiment # # + ######################## + # TODO saving an empty string as value in a text property leads to a + # vanishing of the property + # thus an x is used here. Needs to be fixed. + exp = db.execute_query( + "FIND Experiment with date=2019-02-03 and identifier='x'", + unique=True) + + # There should be a Project with name TestProject which is referenced + project_id = exp.get_property("Project").value + project = get_entity_with_id(project_id) + assert project.name == "TestProject" + assert "Project" in [p.name for p in project.get_parents()] + + # There should be a datafile as result attached with path datafile.dat + datfile_id = exp.get_property("results").value[0] + datfile = get_entity_with_id(datfile_id) + assert os.path.basename(datfile.path) == "datafile.dat" + + ######################### + # # second experiment # # + ######################### + exp = db.execute_query( + "FIND Experiment with date=2019-02-03 and identifier='something'", + unique=True) + + # Should be the same project + assert project_id == exp.get_property("Project").value + + # Should have two data files + datfile_ids = exp.get_property("results").value + datafile_names = [] + + for did in datfile_ids: + datfile = get_entity_with_id(did) + datafile_names.append(os.path.basename(datfile.path)) + assert "usefull.xlsx" in datafile_names + assert "useless.xlsx" in datafile_names + + def test_analysis(self): + ###################### + # # first analysis # # + ###################### + ana = db.execute_query( + "FIND Analysis with date=2019-02-03 and identifier='x'", + unique=True) + + # There should be a Project with name TestProject which is referenced + project_id = ana.get_property("Project").value + project = get_entity_with_id(project_id) + assert "Project" in [p.name for p in project.get_parents()] + + # There should be a file as result attached with path results.pdf + datfile_id = ana.get_property("results").value[0] + datfile = get_entity_with_id(datfile_id) + assert os.path.basename(datfile.path) == "results.pdf" + + # There should be a file as script attached with path plot.py + datfile_id = ana.get_property("scripts").value[0] + datfile = get_entity_with_id(datfile_id) + assert os.path.basename(datfile.path) == "plot.py" + + ####################### + # # second analysis # # + ####################### + ana = db.execute_query( + "FIND Analysis with date=2019-02-03 and identifier='something'", + unique=True) + + # Should be the same project + assert project_id == ana.get_property("Project").value + + # Should have two data files + datfile_ids = ana.get_property("results").value + datafile_names = [] + + for did in datfile_ids: + datfile = get_entity_with_id(did) + datafile_names.append(os.path.basename(datfile.path)) + assert "lol1.png" in datafile_names + assert "lol2.png" in datafile_names + + # There should be a file as script attached with path plot.py + datfile_id = ana.get_property("scripts").value[0] + datfile = get_entity_with_id(datfile_id) + assert os.path.basename(datfile.path) == "analyse.py" + + def test_publication(self): + ######################### + # # first publication # # + ######################### + pub = db.execute_query("FIND really_cool_finding", unique=True) + + # There should be a Project with name TestProject which is referenced + + ########################## + # # second publication # # + ########################## + pub = db.execute_query("FIND paper_on_exciting_stuff ", unique=True) diff --git a/src/caosadvancedtools/cfood.py b/src/caosadvancedtools/cfood.py index 23e60d8e7d75d62ed41a4d482abb2c1b2e252eca..1f5602a01c32587d59f78a37cbc22e463bebb8c5 100644 --- a/src/caosadvancedtools/cfood.py +++ b/src/caosadvancedtools/cfood.py @@ -31,9 +31,10 @@ from copy import deepcopy from datetime import datetime import caosdb as db -from caosadvancedtools.cache import Cache from caosdb.exceptions import TransactionError +from caosadvancedtools.cache import Cache + ENTITIES = {} @@ -107,8 +108,9 @@ class AbstractCFood(object): for key, identifiable in entities.items(): if identifiable is None: - print("THIS IS STRANGE. No identifiables found in {}.".format( - crawled_file}) + print("THIS IS STRANGE. No identifiables found in {}.".format( + crawled_file)) + continue existing = None @@ -209,7 +211,7 @@ class AbstractCFood(object): # is using the unique keyword try: r = q.execute(unique=True) - except TransactionError as er: + except TransactionError: r = None if r is not None: diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py index 79c4ee071e129173dbca9289dbea522eb1c7b3de..d908b8285f184f6207e185f2d96594ce482c92b7 100644 --- a/src/caosadvancedtools/crawler.py +++ b/src/caosadvancedtools/crawler.py @@ -57,5 +57,3 @@ class Crawler(object): print("{} FILES TO BE PROCESSED.".format(len(files))) return files - - diff --git a/src/caosadvancedtools/read_md_header.py b/src/caosadvancedtools/read_md_header.py index 95096ed32d21ec4b58031fad3fc29f94fb0b2756..c4c278cca9059a73069b306f4afd04645773768d 100644 --- a/src/caosadvancedtools/read_md_header.py +++ b/src/caosadvancedtools/read_md_header.py @@ -57,6 +57,7 @@ def _clean_header(header): class NoValidHeader(Exception): pass + def get_header(filename): """Open an md file identified by filename and read out the yaml header. diff --git a/src/programs/crawl.py b/src/programs/crawl.py index a21d44349e54852bdb5f319c92f85a6124adf951..47d28d6cfbe3565f0a290ae3c3c9e149ff3b47d1 100755 --- a/src/programs/crawl.py +++ b/src/programs/crawl.py @@ -29,6 +29,7 @@ import argparse import caosdb as db from argparse import RawTextHelpFormatter + def get_parser(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) diff --git a/src/programs/example_cfood.py b/src/programs/example_cfood.py index 4b06345775d5f8758303bf4ffab92a1f9b138364..32d1114ae14cc5c431bdc3d17e475af4dc6fc4e0 100644 --- a/src/programs/example_cfood.py +++ b/src/programs/example_cfood.py @@ -27,6 +27,7 @@ import caosdb as db from caosadvancedtools.cfood import AbstractCFood + class ExampleCFood(AbstractCFood): def create_identifiables(self, crawled_file, match): print("create_identifiables") diff --git a/unittests/create_filetree.py b/unittests/create_filetree.py index c606da9e96cd58631100665842fad1df571863f4..6f95618dbc834c3bc140163efdc90aa51c8d5248 100644 --- a/unittests/create_filetree.py +++ b/unittests/create_filetree.py @@ -36,14 +36,14 @@ def main(folder, dry=True): if not os.path.exists(base_path) and not dry: os.system("mkdir -p "+base_path) - for ii, ser in enumerate(["Series_{}".format(i) for i in range(3)]): + for ii, ser in enumerate(["Series_{}".format(i) for i in range(3)]): series_path = os.path.join(base_path, ser) - print("Series:\n"+ser +"\n") + print("Series:\n"+ser + "\n") if not dry: os.mkdir(series_path) for date in [datetime.today()-timedelta(days=i)-timedelta(weeks=50*ii) for i in range(10)]: #import IPython - #IPython.embed() + # IPython.embed() exp_path = os.path.join(series_path, "Exp_"+str(date.date())) print("Exp: "+os.path.basename(exp_path)) if not dry: @@ -62,11 +62,6 @@ def main(folder, dry=True): os.system("touch "+d_path) - - - - - def get_parser(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) diff --git a/unittests/test_cfood.py b/unittests/test_cfood.py index 1fcf418ef03bafbe1d8a7b907d39c425b71bcd80..8894e1514e28dbed9bdafe148dbaff39d07d1fd6 100644 --- a/unittests/test_cfood.py +++ b/unittests/test_cfood.py @@ -39,6 +39,7 @@ class CFoodTest(unittest.TestCase): def test_check_existence(self): assert AbstractCFood.find_existing(self.exp) is None + class CFoodTestExist(CFoodTest): def setUp(self): super().setUp() @@ -46,7 +47,7 @@ class CFoodTestExist(CFoodTest): def test_check_existence(self): res = AbstractCFood.find_existing(self.exp) - assert res.id == self.exp.id + assert res.id == self.exp.id def tearDown(self): self.exp.delete() diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py index ca90f00297b59a1ae914cc202e6b6f0e5f1e7e99..9f0d618c1d7eba9dd688fe8022d34ec5282acf6f 100644 --- a/unittests/test_table_converter.py +++ b/unittests/test_table_converter.py @@ -82,6 +82,7 @@ class ToTsvTest(unittest.TestCase): class IntegrationTest(unittest.TestCase): """ converts tsv to a container and back and compares origin with result """ + def test_backandforth(self): cont = from_tsv(TEST_TABLE, "Measurement") tempfile = NamedTemporaryFile(delete=False)