diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index efaa0b27ddc0827cb5db9e01552de53eaaf81246..8ac36de4373e060ffaff29c917f7784e2eeba91a 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -7,7 +7,7 @@ services: networks: - caosnet caosdb-server: - image: "$CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG" + image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG" user: 999:999 depends_on: - sqldb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20260afc156cba26e68d792740b4e5153555d2a5..fefc400e875b804056587cc521a879c7d37c46f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,8 +19,8 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. variables: - CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-pyinttest/testenv:latest - CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/caosdb-pyinttest/base:latest + CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/testenv:latest + CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest DEPLOY_REF: dev stages: @@ -78,11 +78,16 @@ stages: # containers. These volumes are intended to be used for testing server-side scripting and # file-system features. # -services: - - docker:20.10.3-dind test: tags: [docker] + services: + - docker:20.10.3-dind + variables: + # This is a workaround for the gitlab-runner health check mechanism when + # using docker-dind service. The runner will otherwise guess the port + # wrong and the health check will timeout. + SERVICE_PORT_2376_TCP_PORT: 2375 stage: test image: $CI_REGISTRY_IMAGE_BASE needs: ["cert"] @@ -103,23 +108,21 @@ test: - echo $CI_COMMIT_REF_NAME - echo $CI_REGISTRY_IMAGE - - docker login -u gitlab+deploy-token-ci-pull -p $TOKEN_CI_PULL $CI_REGISTRY_INDISCALE + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - time docker load < /image-cache/caosdb-${F_BRANCH}.tar || true - if [[ "$CAOSDB_TAG" == "" ]]; then if echo "$F_BRANCH" | grep -c "^f-" ; then CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH}-latest; - docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEPLOY_REF}-latest ; + docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEPLOY_REF}-latest ; else CAOSDB_TAG=${DEPLOY_REF}-latest; fi; fi - - docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev-latest ; + - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev-latest ; - echo $CAOSDB_TAG - time docker load < /image-cache/caosdb-pyint-testenv-${CI_COMMIT_REF_NAME}.tar || true - - time docker load < /image-cache/mariadb-${F_BRANCH}.tar || true - - time docker load < /image-cache/caosdb-${F_BRANCH}.tar || true - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - time docker load < /image-cache/mariadb.tar || true - docker pull $CI_REGISTRY_IMAGE - cd .docker # here the server and the mysql backend docker are being started @@ -137,6 +140,8 @@ test: - docker logs docker_sqldb_1 &> ../mariadb_log.txt - docker-compose -f .docker/docker-compose.yml down - rc=`cat .docker/result` + - echo mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') + - docker save mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') > /image-cache/mariadb.tar || true - exit $rc dependencies: [cert] timeout: 3h @@ -151,6 +156,10 @@ build-testenv: tags: [cached-dind] image: docker:19.03 stage: setup + timeout: 2h + only: + - shedules + - web needs: [] script: - df -h @@ -183,7 +192,6 @@ cert: tags: [docker] stage: cert image: $CI_REGISTRY_IMAGE - needs: ["build-testenv"] artifacts: paths: - .docker/cert/ @@ -194,7 +202,6 @@ cert: style: tags: [docker] stage: style - needs: ["build-testenv"] image: $CI_REGISTRY_IMAGE script: - autopep8 -r --diff --exit-code . @@ -217,7 +224,5 @@ store-version: only: - dev - master - # TODO remove - - f-versions - web diff --git a/README.md b/README.md index 2a11f69baaa0c7004aa35a84bff7146cd05cedb4..d5641554f251a85f3d89da4d6bfc2b1f76ebfc44 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,34 @@ CaosDB project. # Getting started # -- To run tests, start up a CaosDB server with (documentation for this can be found elsewhere). +- To run tests, start up a CaosDB server with the following properties (documentation for this can + be found elsewhere): - The CaosDB server must have debugging enabled. - - Certificates must be valid and be listed in `pycaosdb.ini`. -- Modify `pycaosdb.ini.template` accordingly and save it as `pycaosdb.ini`. + - The database should be empty. +- Modify `pycaosdb.ini.template` and save it as `pycaosdb.ini`, taking care of the following points: + - Certificates must be valid and be specified in `pycaosdb.ini`. + - Server-side scripting paths must be given, otherwise server-side scripting will be omitted. - Run the tests with `pytest` or `pytest-3` (depending on your system). - If you want to run just a single test, you can also select a single test file: `pytest-3 tests/test_issues.py` +## Requirements ## + +- Running the tests needs `pytest`, obviously, and also the `nose` library: + - `pip3 install pytest nose` + +## Run tests agains a remote caosdb server + +* There is a special mark `local_server`, which marks all test which cannot + pass when the server is not on the host where the test suite is running. +* Start tests with `pytest -m "not local_server"` or + `tox -- -m "not local_server"`. +* The remote caosdb server must have a TLS certificate which maches the name given in + `pycaosdb.ini`. If you use the *LinkAhead* distribution of CaosDB, this can be achieved by: + * `linkahead cert_gen -- --hostname <your hostname>` + * `linkahead certs -d <your profile path>/custom/other/cert` + * `linkahead start` + # Further Reading Please refer to the [official gitlab repository of the CaosDB @@ -24,7 +44,8 @@ project](https://gitlab.com/caosdb/caosdb) for more information. Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for Dynamics and Self-Organization Göttingen. +Copyright (C) 2021 Indiscale GmbH <info@indiscale.com> -All files in this repository are licensed under a [GNU Affero General Public +All files in this repository are licensed under the [GNU Affero General Public License](LICENCE.md) (version 3 or later). diff --git a/pycaosdb.ini.template b/pycaosdb.ini.template index 369b33f6f63d15b737d6a558422ab1bfe6c30f15..c94af17ae01408f9d7f24473ce36ce738c75331d 100644 --- a/pycaosdb.ini.template +++ b/pycaosdb.ini.template @@ -1,29 +1,51 @@ # -*- mode:conf; -*- -## This sections needs to exist in addition to the usual section + +## This sections must exist in addition to the usual section. [IntegrationTests] -test_server_side_scripting.bin_dir.local=/path/to/scripting/bin -test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scritping/bin +########## Server-side scripting ################## +## These are used by tests of server side scripting. Both paths have +## to point to existing directories in which the CaosDB server has the +## permissions to create and execute scripts. + +# location of the scripting bin dir which is used for the test scripts from the +# pyinttest's perspective. +#test_server_side_scripting.bin_dir.local=/path/to/scripting/bin + +# location of the scripting bin dir which is used for the test scripts from the +# server's perspective. +#test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin + +########## Files ################## +## Used by tests of file handling. Specify the path to an existing +## directory in which file tests are performed, once as seen by the +## host and once as seen by the server. + +# location of the files from the pyinttest (i.e. host) perspective +#test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/ + +# location of the files from the caosdb server's perspective +#test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/ -# location of the files from the pyinttest perspective -test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/ -# location of the files from the caosdb_servers perspective -test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/ +########## Authentication tokens ################## +# # location of the one-time tokens from the pyinttest's perspective +# test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt +# test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt +# test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt -# location of the one-time tokens from the pyinttest's perspective -test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt -test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt -test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt ## Insert your usual settings here [Connection] #url=https://caosdb-server:10443/ -#username=admin -#cacert=/path/to/cert/caosdb.cert.pem -#cacert=/etc/ssl/cert.pem -#debug=0 +## Some integration tests will fail without valid credentials. +#username=admin #password_method=plain #password=caosdb +## Provide SSL certificate used by the CaosDB server for all tests to +## run. Some integration tests will fail without a valid certificate here. +#cacert=/path/to/cert.pem + +#debug=0 #ssl_insecure=True #timeout=500