Merge branch 'dev' into f-fsm

.docker/docker-compose.yml

@@ -7,7 +7,7 @@ services:
     networks:
       - caosnet
   caosdb-server:
-    image: "$CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+    image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
     user: 999:999
     depends_on:
       - sqldb

.gitlab-ci.yml

@@ -19,8 +19,8 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-pyinttest/testenv:latest
-   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/caosdb-pyinttest/base:latest
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/testenv:latest
+   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
    DEPLOY_REF: dev
 
 stages:
@@ -78,11 +78,16 @@ stages:
 # containers.  These volumes are intended to be used for testing server-side scripting and
 # file-system features.
 #
-services:
-  - docker:20.10.3-dind
 
 test:
   tags: [docker]
+  services:
+    - docker:20.10.3-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
   stage: test
   image: $CI_REGISTRY_IMAGE_BASE
   needs: ["cert"]
@@ -103,23 +108,21 @@ test:
       - echo $CI_COMMIT_REF_NAME
       - echo $CI_REGISTRY_IMAGE
 
-      - docker login -u gitlab+deploy-token-ci-pull -p $TOKEN_CI_PULL $CI_REGISTRY_INDISCALE
+      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       - time docker load < /image-cache/caosdb-${F_BRANCH}.tar || true
       - if [[ "$CAOSDB_TAG" == "" ]]; then
           if echo "$F_BRANCH" | grep -c "^f-" ; then
             CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH}-latest;
-            docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEPLOY_REF}-latest ;
+            docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEPLOY_REF}-latest ;
           else
             CAOSDB_TAG=${DEPLOY_REF}-latest;
           fi;
         fi
-      - docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev-latest ;
+      - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev-latest ;
       - echo $CAOSDB_TAG
 
       - time docker load < /image-cache/caosdb-pyint-testenv-${CI_COMMIT_REF_NAME}.tar || true
-      - time docker load < /image-cache/mariadb-${F_BRANCH}.tar || true
-      - time docker load < /image-cache/caosdb-${F_BRANCH}.tar || true
-      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+      - time docker load < /image-cache/mariadb.tar || true
       - docker pull $CI_REGISTRY_IMAGE
       - cd .docker
         # here the server and the mysql backend docker are being started
@@ -137,6 +140,8 @@ test:
       - docker logs docker_sqldb_1 &> ../mariadb_log.txt
       - docker-compose -f .docker/docker-compose.yml down
       - rc=`cat .docker/result`
+      - echo mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}')
+      - docker save mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') > /image-cache/mariadb.tar || true
       - exit $rc
   dependencies: [cert]
   timeout: 3h
@@ -151,6 +156,10 @@ build-testenv:
   tags: [cached-dind]
   image: docker:19.03
   stage: setup
+  timeout: 2h
+  only:
+    - shedules
+    - web
   needs: []
   script:
       - df -h
@@ -183,7 +192,6 @@ cert:
   tags: [docker]
   stage: cert
   image: $CI_REGISTRY_IMAGE
-  needs: ["build-testenv"]
   artifacts:
     paths:
       - .docker/cert/
@@ -194,7 +202,6 @@ cert:
 style:
   tags: [docker]
   stage: style
-  needs: ["build-testenv"]
   image: $CI_REGISTRY_IMAGE
   script:
       - autopep8 -r --diff --exit-code .
@@ -217,7 +224,5 @@ store-version:
   only:
     - dev
     - master
-    # TODO remove
-    - f-versions
     - web
 

README.md

@@ -7,14 +7,34 @@ CaosDB project.
 
 # Getting started #
 
-- To run tests, start up a CaosDB server with (documentation for this can be found elsewhere).
+- To run tests, start up a CaosDB server with the following properties (documentation for this can
+  be found elsewhere):
   - The CaosDB server must have debugging enabled.
-  - Certificates must be valid and be listed in `pycaosdb.ini`.
-- Modify `pycaosdb.ini.template` accordingly and save it as `pycaosdb.ini`.
+  - The database should be empty.
+- Modify `pycaosdb.ini.template` and save it as `pycaosdb.ini`, taking care of the following points:
+  - Certificates must be valid and be specified in `pycaosdb.ini`.
+  - Server-side scripting paths must be given, otherwise server-side scripting will be omitted.
 - Run the tests with `pytest` or `pytest-3` (depending on your system).
 - If you want to run just a single test, you can also select a single test file:  
   `pytest-3 tests/test_issues.py`
 
+## Requirements ##
+
+- Running the tests needs `pytest`, obviously, and also the `nose` library:
+  - `pip3 install pytest nose`
+
+## Run tests agains a remote caosdb server
+
+* There is a special mark `local_server`, which marks all test which cannot
+  pass when the server is not on the host where the test suite is running.
+* Start tests with `pytest -m "not local_server"` or
+  `tox -- -m "not local_server"`.
+* The remote caosdb server must have a TLS certificate which maches the name given in
+  `pycaosdb.ini`.  If you use the *LinkAhead* distribution of CaosDB, this can be achieved by:
+    * `linkahead cert_gen -- --hostname <your hostname>`
+    * `linkahead certs -d <your profile path>/custom/other/cert`
+    * `linkahead start`
+
 # Further Reading
 
 Please refer to the [official gitlab repository of the CaosDB
@@ -24,7 +44,8 @@ project](https://gitlab.com/caosdb/caosdb) for more information.
 
 Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for
 Dynamics and Self-Organization Göttingen.
+Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
 
-All files in this repository are licensed under a [GNU Affero General Public
+All files in this repository are licensed under the [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).
 

pycaosdb.ini.template

 # -*- mode:conf; -*-
-## This sections needs to exist in addition to the usual section
+
+## This sections must exist in addition to the usual section.
 [IntegrationTests]
-test_server_side_scripting.bin_dir.local=/path/to/scripting/bin
-test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scritping/bin
+########## Server-side scripting ##################
+## These are used by tests of server side scripting. Both paths have
+## to point to existing directories in which the CaosDB server has the
+## permissions to create and execute scripts.
+
+# location of the scripting bin dir which is used for the test scripts from the
+# pyinttest's perspective.
+#test_server_side_scripting.bin_dir.local=/path/to/scripting/bin
+
+# location of the scripting bin dir which is used for the test scripts from the
+# server's perspective.
+#test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin
+
+########## Files ##################
+## Used by tests of file handling. Specify the path to an existing
+## directory in which file tests are performed, once as seen by the
+## host and once as seen by the server.
+
+# location of the files from the pyinttest (i.e. host) perspective
+#test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+
+# location of the files from the caosdb server's perspective
+#test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
 
-# location of the files from the pyinttest perspective
-test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
-# location of the files from the caosdb_servers perspective
-test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+########## Authentication tokens ##################
+# # location of the one-time tokens from the pyinttest's perspective
+# test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
+# test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
+# test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
 
-# location of the one-time tokens from the pyinttest's perspective
-test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
-test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
-test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
 
 ## Insert your usual settings here
 [Connection]
 #url=https://caosdb-server:10443/
-#username=admin
-#cacert=/path/to/cert/caosdb.cert.pem
-#cacert=/etc/ssl/cert.pem
-#debug=0
 
+## Some integration tests will fail without valid credentials.
+#username=admin
 #password_method=plain
 #password=caosdb
 
+## Provide SSL certificate used by the CaosDB server for all tests to
+## run. Some integration tests will fail without a valid certificate here.
+#cacert=/path/to/cert.pem
+
+#debug=0
 #ssl_insecure=True
 #timeout=500