Skip to content
Snippets Groups Projects
Commit fe5c7c70 authored by Henrik tom Woerden's avatar Henrik tom Woerden
Browse files

Merge branch 'integrationtest' into 'master'

Integrationtest

See merge request bmp-caosdb/caosdb-advanced-user-tools!4
parents 21ea9609 4df3821f
Branches
Tags
No related merge requests found
Showing
with 493 additions and 3 deletions
FROM debian:latest
RUN apt-get update && \
apt-get install curl python3 python3-pip git python-autopep8 \
python3-pytest -y
COPY .docker/wait-for-it.sh /wait-for-it.sh
RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-pylib.git && \
cd caosdb-pylib && pip3 install .
RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-models.git && \
cd caosdb-models && pip3 install .
ADD https://gitlab.com/api/v4/projects/13601752/repository/branches \
scifolder_version.txt
RUN git clone https://gitlab.com/henrik_indiscale/scifolder.git && \
cd scifolder && pip3 install .
COPY . /git
RUN rm -r /git/.git && mv /git/.docker/pycaosdb.ini /git/integrationtests
RUN cd /git && pip3 install .
WORKDIR /git/integrationtests/
CMD /wait-for-it.sh caosdb-server:10443 -t 120 -- ./test.sh
File added
-----BEGIN CERTIFICATE-----
MIIDETCCAfmgAwIBAgIUM1S4TR6nEsstWuXDjZGBv6sb0LAwDQYJKoZIhvcNAQEL
BQAwGDEWMBQGA1UEAwwNY2Fvc2RiLXNlcnZlcjAeFw0xOTA3MTgxNDM3MzNaFw0x
OTA4MTcxNDM3MzNaMBgxFjAUBgNVBAMMDWNhb3NkYi1zZXJ2ZXIwggEiMA0GCSqG
SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMFzou21Jugj9Z3YUkhXANt2oGNvuCyt9S
IfBZO5ZpgwSTcRs8UAuKDYAMmGMeQtqLfqEYoQFEeybSd9NsrFb0IgrUaKrSNAzs
phjsGvXU1UbMRw0vva3l6r94ceM8ecw+x2RIsTwfJkfVL60XMm9DvTaQFLzjcKLf
lOQbXG6jgCHtvO5MThk/C+TvsAZlRDErp7HgDqDFg+jnJC5YsEFQQyAYCLJXOotQ
A5FnRbT8dtoDHy03mHyed5Ji3IgxKxE5mF0ygG8/FRfgzqImg4FMnCb+KBwom2xh
y2hvtFnm3+kCbB+waDTUaPZwvNUONUZMfcl3PggX0IwipqWL8DBJAgMBAAGjUzBR
MB0GA1UdDgQWBBSglY56eKgLeyW1o5UdZgqlzDJJEjAfBgNVHSMEGDAWgBSglY56
eKgLeyW1o5UdZgqlzDJJEjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
A4IBAQAF4sn/C7wx9fq/9/v6UYb9DbOXaonS+1jOzf1lAOEwHtJh0M9SDTqvaeQz
bshJ33ZFbwH3qIq7+OzGj9Yon9LN1GGEl4CcKKQRQQkjo7jt7X47C6PkrW6l23+i
OrDRk7Oke9BJS/ZRKLmxvlaLYULpj02lOsbddfE4igXxRPCiouosc+HQ3kd6Mqhv
5Gt0Yki3z4fvi+CWrtUjxYeMUJkraFAyKDKwFVZgDnWwyipGo9yh3a6iXD7duQcx
FRk7zm1oaF3ia9hFfU1AVDfkgIlPLWW30CT4Jgi25/wpiGZiT1uXAgfoJCEDoYXp
bQIM5KQqw0Kgjqc/KuLCRe1KeCzq
-----END CERTIFICATE-----
File added
-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-256-CBC,0B42601A73984CD9F1DF2F786F894EA0
5ZY5CR8ho3a9449XIUmSpIsCZZWsVQ1pWX+SJPSUlk2A9N3BHdY0AdvCx+sTF7p4
bSoAulnAEFlIeXqimtK+InH0d3HLbvn41gvEbFTPjZdejxro6m9DvSHCpyDZK9TH
m5A2aRsJioeibETUA7uvld/d9Szdz//GPN2YzWqJpzXZtTWHH/tR2J2L6KE+wBDa
IViryBCv0+9e6l2WeEjXqqYZhznR/gUsxc2xEs7x8wEpWBVlmVzYWuSt7OA5oOm/
r1OAR9LEH0PIISUv2h+P7ccOZJD/JE2YDM3IBdClxLv2MRyj/88ougcV5gv688+f
6XxT2+LfCLd+viqGcFZ5a6tT1wpUthVWWvUWPna6SlVVrsj/7bCc6y2Jptff+Kqv
eL8Gl1dWZXwe0JXr+Ss5KfIddbY0pJv8FHjCsN+Nkdozo7Wdufm1Xkw+s1f2td6u
bHwbo1Y4mX5nm9I2OkAMhGubSPalGZRUvr9cfl1/1V/QgX0jPmJmNmH5yFUdc5MQ
QIdcpZmz8xIH1XAdJWTbzoYKzKnXTHpZFaZL1acMrK2CERMxDStfe9W/DSRy6/qn
AsQbh6tDk+Fe05EQwpZBZQ1YlrfAru7QDaDE9+uO+mxkf1UHynfPHNp2jZAe6XBt
p97GXUruLR8rBOAGX19EuoG2sSek44dLW5IsLGbR1MpTlq48YZv2z3TrESNJHKAM
VAxg5kwqvYocykdFrKmggbdBv3oo6g59RxiufKparlkI7YZFhks6V1gujKW+JChc
Eoy8e2EJJP0pY0Pb3ZE9QkK7ZX4otXNRgqqnf2f7kTiGMRBSDSD5xGUw0syXAS3G
Ji/4pMFh8ALsYUOOOYUISWAUB6O8/438SIq+g+TNHYVbrli/WK9ip0+RGPmJ+Ulm
iWQMQOcsSYTBqDytvr0241JDBuq9RjMBJE4FO2ltr7L4gmkN8+d/WGnvu4onJBoE
8ApriArJjuHBP1iex3uygJKD3sdiuAuukrPlEla1xjywH3QSzj/SWABIu4QJAr3a
70jEFGL04eblvBPGN2hnXEQCxHqRJLpFH1huWMlO3PuoRnGxIuxymkEOOEWfswLO
r9qLbkUWmFFFqJBNB3T4kfzuZDqiCQnS8+QNvhTiKCWLcUswXTirSac8B5cWDaqd
/Nf119n3ezZkpzgKr0kw+kLxlU2Tbl7UBZ7TRadxklXaAmEo7z+Wh7WKmbcUMqit
xh6P7P0ZWcLYIiXCC5gYr6jzU+Rjx4ZGF5+Q0LcXC3Ai6Ujz4zdrVRLBiw27zKgB
RsYBCYRmQ2qO0AzHFBWixl7IVGYi+QFKU0SFrPLzme+yOJjjLAv2Xzu24KFiwHLz
8MWCd0yM8UjXAql+jCk54O6e4QwER2Rfr6vxlH6x90diy8eJQXejBEVNIfyOpyST
Mqn2jc5vWanVxRk/SgrZ0pOJ8TBeGSzolJF9MBPjZ9PJTbu3WzJpuguQ4CeAAV7E
0+rbPGALnuDKmrsnyIb9WX+fEpT7oBPEKV84XyG1+UkBrfDaQvK3SujeRvnOaZLG
uSNc8yX9CZmt2OJTRWK5fKnR5fGzAnMR7cRUYNvcWJ0T6B4wj8cwF4wY9laGzMO7
-----END RSA PRIVATE KEY-----
version: '3.7'
services:
sqldb:
image: mariadb:10.4
environment:
MYSQL_ROOT_PASSWORD: caosdb1234
networks:
- caosnet
caosdb-server:
image: "$CI_REGISTRY/caosdb:$CAOSDB_TAG"
user: 999:999
depends_on:
- sqldb
networks:
- caosnet
volumes:
- type: bind
source: "$EXEPATH/.docker/cert"
target: /opt/caosdb/cert
- type: bind
source: "$EXEPATH/integrationtests/extroot"
target: /opt/caosdb/mnt/extroot
read_only: true
ports:
# - "from_outside:from_inside"
- "10443:10443"
- "10080:10080"
environment:
DEBUG: 1
networks:
caosnet:
driver: bridge
[IntegrationTests]
test_server_side_scripting.bin_dir=../caosdb-server/test_scripting/bin/
[Connection]
url=https://caosdb-server:10443
username=admin
cacert=../.docker/cert/caosdb.cert.pem
#cacert=/etc/ssl/cert.pem
debug=0
passwordmethod=plain
password=caosdb
ssl_insecure=True
timeout=500
[Container]
debug=0
version: '3.6'
services:
tester:
image: "$CI_REGISTRY_IMAGE:latest"
networks:
- docker_caosnet
networks:
docker_caosnet:
external: true
#!/usr/bin/env bash
# License:
# From https://github.com/vishnubob/wait-for-it
# The MIT License (MIT)
# Use this script to test if a given TCP host/port are available
WAITFORIT_cmdname=${0##*/}
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
usage()
{
cat << USAGE >&2
Usage:
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST Host or IP under test
-p PORT | --port=PORT TCP port under test
Alternatively, you specify the host and port as host:port
-s | --strict Only execute subcommand if the test succeeds
-q | --quiet Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit 1
}
wait_for()
{
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
else
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
fi
WAITFORIT_start_ts=$(date +%s)
while :
do
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
WAITFORIT_result=$?
else
(echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
WAITFORIT_result=$?
fi
if [[ $WAITFORIT_result -eq 0 ]]; then
WAITFORIT_end_ts=$(date +%s)
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
break
fi
sleep 1
done
return $WAITFORIT_result
}
wait_for_wrapper()
{
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
else
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
fi
WAITFORIT_PID=$!
trap "kill -INT -$WAITFORIT_PID" INT
wait $WAITFORIT_PID
WAITFORIT_RESULT=$?
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
fi
return $WAITFORIT_RESULT
}
# process arguments
while [[ $# -gt 0 ]]
do
case "$1" in
*:* )
WAITFORIT_hostport=(${1//:/ })
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
shift 1
;;
--child)
WAITFORIT_CHILD=1
shift 1
;;
-q | --quiet)
WAITFORIT_QUIET=1
shift 1
;;
-s | --strict)
WAITFORIT_STRICT=1
shift 1
;;
-h)
WAITFORIT_HOST="$2"
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
shift 2
;;
--host=*)
WAITFORIT_HOST="${1#*=}"
shift 1
;;
-p)
WAITFORIT_PORT="$2"
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
shift 2
;;
--port=*)
WAITFORIT_PORT="${1#*=}"
shift 1
;;
-t)
WAITFORIT_TIMEOUT="$2"
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
shift 2
;;
--timeout=*)
WAITFORIT_TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
WAITFORIT_CLI=("$@")
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
usage
;;
esac
done
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
echoerr "Error: you need to provide a host and port to test."
usage
fi
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
# check to see if timeout is from busybox?
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
WAITFORIT_ISBUSY=1
WAITFORIT_BUSYTIMEFLAG="-t"
else
WAITFORIT_ISBUSY=0
WAITFORIT_BUSYTIMEFLAG=""
fi
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
wait_for
WAITFORIT_RESULT=$?
exit $WAITFORIT_RESULT
else
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
wait_for_wrapper
WAITFORIT_RESULT=$?
else
wait_for
WAITFORIT_RESULT=$?
fi
fi
if [[ $WAITFORIT_CLI != "" ]]; then
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
exit $WAITFORIT_RESULT
fi
exec "${WAITFORIT_CLI[@]}"
else
exit $WAITFORIT_RESULT
fi
__pycache__
.tox
.coverage
cache.db
*.egg-info
.tox
__pycache__
#
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019 Henrik tom Wörden
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
#
variables:
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-advanced-testenv
# With dind the docker deamon is available on the network
DOCKER_HOST: tcp://docker:2375/
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
DOCKER_DRIVER: overlay2
services:
- docker:dind
stages:
- setup
- style
- test
test:
tags: [docker]
stage: test
image:
name: docker/compose:1.24.1
entrypoint: ["/bin/sh", "-c"]
script:
- if [[ "$CAOSDB_TAG" == "" ]]; then
CAOSDB_TAG=master-latest;
fi
- echo $CAOSDB_TAG
- docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $CI_REGISTRY_IMAGE:latest
- EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
-f .docker/docker-compose.yml up -d
- docker-compose -f .docker/tester.yml run tester
- rc=$?
- docker-compose -f .docker/docker-compose.yml down
- exit $rc
build-testenv:
tags: [docker]
image: docker:latest
stage: setup
script:
- docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
# use here general latest or specific branch latest...
- docker pull $CI_REGISTRY_IMAGE:latest || true
- docker build
--pull
--file .docker/Dockerfile
--cache-from $CI_REGISTRY_IMAGE:latest
-t $CI_REGISTRY_IMAGE:latest .
- docker push $CI_REGISTRY_IMAGE:latest
style:
tags: [docker]
stage: style
image: $CI_REGISTRY_IMAGE:latest
script:
- autopep8 -r --diff --exit-code .
allow_failure: true
#!/usr/bin/env python3
# encoding: utf-8
#
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
#
import argparse
from argparse import RawTextHelpFormatter
import caosdb as db
from caosadvancedtools.crawler import Crawler
from scifolder import (AnalysisCFood, ExperimentCFood, PublicationCFood,
SimulationCFood)
def get_parser():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=RawTextHelpFormatter)
parser.add_argument("path")
return parser
def access(path):
return "extroot" + path
if __name__ == "__main__":
parser = get_parser()
args = parser.parse_args()
print("Starting query...")
files = Crawler.query_files(args.path)
print("Query done...")
config = db.configuration.get_config()
c = Crawler(food=[
AnalysisCFood(use_cache=True, access=access),
ExperimentCFood(use_cache=True, access=access),
PublicationCFood(use_cache=True, access=access),
SimulationCFood(use_cache=True, access=access),
])
c.crawl(files)
---
responsible:
- Only Responsible
description: A description of another example analysis.
data:
- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat"
description: an example reference to a results file
scripts:
- filename: plot.py
description: a plotting script
results:
- filename: results.pdf
...
---
responsible:
- First Person
description: A description of an example analysis.
results:
- filename: "images/*.png"
description: an example reference to a results file
data:
- filename: /ExperimentalData/TestProject/2019-02-03_something/use*.xlsx
description: an example reference to data files
scripts:
- analyse.py
tags:
- collagen
- time sweep
- frequency sweep
...
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment