Merge branch 'f-local-inttests' into 'dev'

F local inttests

See merge request !81

.docker/Dockerfile

@@ -11,11 +11,12 @@ RUN apt-get update && \
 ARG PYLIB
 RUN echo "PYLIB=${PYLIB}"
 COPY .docker/wait-for-it.sh /wait-for-it.sh
+COPY . /git
+RUN pip install -r /git/requirements.txt
 ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \
     pylib_version.json
 RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
     cd caosdb-pylib && git checkout ${PYLIB} && pip3 install .
-COPY . /git
 
 # Delete .git because it is huge.
 RUN rm -r /git/.git
@@ -27,7 +28,11 @@ RUN mv /git/.docker/tester_pylinkahead.ini /git/pylinkahead.ini
 WORKDIR /git
 # wait for server,
 CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \
-    # ... install pylinkahead.ini the server-side scripts
+    # ... install pylinkahead.ini and the server-side scripts
     cp /git/.docker/sss_pylinkahead.ini /scripting/home/.pylinkahead.ini && \
+    cp -r /git/resources /scripting/bin-debug && \
+    # ... put out general version information
+    python3 --version && \
+    python3 -c "import linkahead; print(linkahead.version.version)" && \
     # ... and run tests
-    tox
+    pytest --cov=linkahead -vv tests

.docker/tester_pylinkahead.ini

@@ -4,9 +4,6 @@
 ; location of the scripting bin dir which is used for the test scripts from the
 ; server's perspective.
 test_server_side_scripting.bin_dir.server = scripting/bin-debug/
-; location of the scripting bin dir which is used for the test scripts from the
-; pyinttest's perspective.
-test_server_side_scripting.bin_dir.local = /scripting/bin-debug/
 
 ; location of the files from the pyinttest perspective
 test_files.test_insert_files_in_dir.local = /extroot/test_insert_files_in_dir/

.gitignore

 # dot files
 .*
 !/.gitignore
+!.add_dir_to_git
+!/test_profile/custom/caosdb-server/scripting/home/.pylinkahead.ini
 
 # python stuff
 *.pyc
@@ -16,3 +18,6 @@ PyCaosDB*.egg
 
 # user config
 pycaosdb.ini
+pylinkahead.ini
+*~
+/test_profile/paths/authtoken/admin_token*

CHANGELOG.md

@@ -53,9 +53,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Test for [caosdb-server#263](https://gitlab.com/linkahead/linkahead-server/-/issues/263): Unexpected server error when attempting to retrieve internal ROLE
   entities.
 * Test for [caosdb-server#268](https://gitlab.com/linkahead/linkahead-server/-/issues/268): Unexpected Server Error when non-existent file shall be inserted.
+* test_profile for running the tests locally.
 
 ### Changed (for changes in existing functionality)
 
+* `setup.py` has been replaced by a `requirements.txt`.
 * Tests comply with the new entity error handling (see
   [#32](https://gitlab.com/caosdb/caosdb-pylib/-/issues/32) in
   caosdb-pylib).
@@ -69,8 +71,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed (for now removed features)
 
+- Support for tox
 - Some redundant tests from `test_tickets.py` that checked
   functionality that was already tested in `test_error_stuff.py`.
+- `test_server_side_scripting.bin_dir.local` option in pylinkahead.ini
 
 ### Fixed (for any bug fixes)
 

README.md

-[![pipeline status](https://gitlab.com/caosdb/caosdb-pyinttest/badges/dev/pipeline.svg?job=codestyle)](https://gitlab.com/caosdb/caosdb-pyinttest/commits/dev)
-
 # README
 
 ## Welcome
 
-This is the **CaosDB Python Integration Testing** repository and a part of the
-CaosDB project.
+This is the **Linkahead Python Integration Testing** repository and a part of the
+Linkahead project.
 
 ## Getting started #
 
-- To run tests, start up a CaosDB server with the following properties (documentation for this can
-  be found elsewhere):
-  - The CaosDB server must have debugging enabled.
+### Setup
+
+Dependencies can be installed using  `pip install -r requirements.txt`.  
+
+The tests must be run against a correctly configured linkahead server. The suggested
+way to achieve this is to start linkahead using the [test_profile](/test_profile/profile.yml)
+provided with this repository. Instructions can be found in the official LinkAhead
+[documentation](https://docs.indiscale.com//caosdb-deploy/README_TEST.html#integration-tests).
+
+After a linkahead server is set up, copy the content of 
+[`pylinkahead.ini.template`](pylinkahead.ini.template) to a new file named `pylinkahead.ini`.
+
+### Run the tests
+
+The tests can then be run from the base directory using `pytest` or `pytest-3` (depending on
+your system). You can also select a single test file with `pytest-3 tests/test_issues.py`,
+or a test matching a regular expression, f.e. using `pytest-3 tests/test_issues.py -k issue_123`.
+
+## Configuration for LinkAhead servers built from source
+
+The following steps describe how to set up the tests for running against
+a plain LinkAhead server built from the sources. 
+
+- To run tests, start up a Linkahead server with the following properties
+  (documentation for this can be found in the [official documentation](https://docs.indiscale.com/)):
+  - The Linkahead server must have debugging enabled.
   - The database should be empty.
-  - There should be a symlink named `debug-scripting-bin` inside the `custom` profile directory,
-    which points to the `resources` directory here in the pyinttest repository.  You can create a
-    symlink like that with the following command (adapt paths to match your setting):  
-    `custom$ ln -s ../../../../caosdb-pyinttest/resources debug-scripting-bin`
-- Modify `pylinkahead.ini.template` and save it as `pylinkahead.ini`, taking care of the following points:
+  - The `resources` directory should be made visible, e.g., by
+    symlinking or copying its contents to the server in
+    `scripting/bin-debug`.
+- Modify `pylinkahead.ini.template` and save it as `pylinkahead.ini`,
+  taking care of the following points:
   - Certificates must be valid and be specified in `pylinkahead.ini`.
-  - Server-side scripting paths must be given, otherwise server-side scripting will be omitted.
-    - The *local* path `test_server_side_scripting.bin_dir.local` should point to the
-      `linkahead-server/scripting/bin` directory in linkahead-server sources.
-    - The *remote* path `test_server_side_scripting.bin_dir.server` should probably be something
-      like `/opt/caosdb/git/caosdb-server/scripting/bin-debug`.  This is the path as seen by the
-      server (in most cases inside the docker container).
-  - Paths for the file tests must exist, or be creatable by the testing script and the server.
-- Run the tests with `pytest` or `pytest-3` (depending on your system).
-- If you want to run just a single test, you can also select a single test file:  
-  `pytest-3 tests/test_issues.py`  
-  or a test matching a regular expression:  
-  `pytest-3 tests/test_issues.py -k issue_123`
-
-## Requirements ##
-
-- Running the tests needs `pytest`, obviously, and also the `nose` library:
-  - `pip3 install pytest nose`
-
-## Run tests agains a remote caosdb server
+  - Server-side scripting paths must be given, otherwise server-side
+	scripting will be omitted. The remote path
+	`test_server_side_scripting.bin_dir.server` should probably be
+	something like
+	`/opt/caosdb/git/caosdb-server/scripting/bin-debug`.
+  - Paths for the file tests must exist, or be creatable by the
+    testing script and the server.
+
+## Run tests against a remote linkahead server
 
 * There is a special mark `local_server`, which marks all test which cannot
   pass when the server is not on the host where the test suite is running.
 * Start tests with `pytest -m "not local_server"` or
   `tox -- -m "not local_server"`.
-* The remote caosdb server must have a TLS certificate which maches the name given in
-  `pycaosdb.ini`.  If you use the *LinkAhead* distribution of CaosDB, this can be achieved by:
-    * `linkahead cert_gen -- --hostname <your hostname>`
-    * `linkahead certs -d <your profile path>/custom/other/cert`
-    * `linkahead start`
+* The remote linkahead server must have a TLS certificate which matches the name given
+  in `pylinkahead.ini`.  If you use LinkAhead Control, this can be achieved by:
+	* `linkahead cert_gen -- --hostname <your hostname>`
+	* `linkahead certs -d <your profile path>/custom/other/cert`
+	* `linkahead start`
 
 ## Further Reading
 
-Please refer to the [official documentation](https://caosdb.gitlab.io/caosdb-pylib/) for more information.
+Please refer to the [official documentation](https://docs.indiscale.com/) for more information.
 
 ## Contributing
 
-Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
+Thank you very much to all contributers—[past, present](https://gitlab.com/linkahead/linkahead/-/blob/dev/HUMANS.md), and prospective ones.
 
 ### Code of Conduct
 
-By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/linkahead/linkahead/-/blob/dev/CODE_OF_CONDUCT.md).
 
 ### How to Contribute
 
 * You found a bug, have a question, or want to request a feature? Please
-[create an issue](https://gitlab.com/caosdb/caosdb-pyinttest/-/issues).
+[create an issue](https://gitlab.com/linkahead/linkahead-pyinttest/-/issues).
 * You want to contribute code? Please fork the repository and create a merge
 request in GitLab and choose this repository as target. Make sure to select
 "Allow commits from members who can merge the target branch" under Contribution
 when creating the merge request. This allows our team to work with you on your request.
-- You can also contact us at **info (AT) caosdb.org**.
+- You can also contact us at **info (AT) indiscale.com**.
 
 ## License
 
 * Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
   for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2024 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under the [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).

pylinkahead.ini.template

@@ -7,13 +7,9 @@
 ## to point to existing directories in which the CaosDB server has the
 ## permissions to create and execute scripts.
 
-# Location of the scripting bin dir which is used for the test scripts from the
-# pyinttest's perspective.  Probably the scripting/bin dir in the caosdb-server sources.
-#test_server_side_scripting.bin_dir.local=/path/to/scripting/bin
-
-# Location of the scripting bin dir which is used for the test scripts from the
-# server's perspective.
-#test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin
+# Location of the scripting bin debug dir which is used for the test
+# scripts from the server's perspective.
+test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin-debug
 
 ########## Files paths ##################
 ## Used by tests of file handling. Specify the path to an existing
@@ -22,31 +18,35 @@
 
 # Location of the files from the pyinttest (i.e. host) perspective.  Probably the local extroot
 # path plus `/test_insert_files_in_dir/`.
-#test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+test_files.test_insert_files_in_dir.local=./test_profile/paths/extroot/test_insert_files_in_dir/
 
 # Location of the files from the caosdb server's perspective.  Probably with the same last
 # component(s) as the local variant (above).
-#test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
 
 ########## Authentication tokens ##################
-# # location of the one-time tokens from the pyinttest's perspective
-# test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
-# test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
-# test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
+# location of the one-time tokens from the pyinttest's
+# perspective. When using the test_profile, the parent directory has
+# to be the same as the value of `debug_token_auth_dir`
+test_authentication.admin_token_crud = /tmp/linkahead-pyinttest/authtoken/admin_token_crud.txt
+test_authentication.admin_token_expired = /tmp/linkahead-pyinttest/authtoken/admin_token_expired.txt
+test_authentication.admin_token_3_attempts = /tmp/linkahead-pyinttest/authtoken/admin_token_3_attempts.txt
 
+# Needed for one time-zone test
+test_misc.test_time_zone.time_zone = Cuba
 
 ## Insert your usual settings here
 [Connection]
-#url=https://caosdb-server:10443/
+url=https://localhost:10443/
 
 ## Some integration tests will fail without valid credentials.
-#username=admin
-#password_method=plain
-#password=caosdb
+username=admin
+password_method=plain
+password=caosdb
 
 ## Provide SSL certificate used by the CaosDB server for all tests to
 ## run. Some integration tests will fail without a valid certificate here.
-#cacert=/path/to/cert.pem
+cacert=./test_profile/custom/other/cert/caosdb.cert.pem
 
 #debug=0
 #ssl_insecure=True

requirements.txt

+jsonschema
+linkahead
+pytest
+pytest-cov
+python-dateutil
+setuptools

resources/subdir/.add_dir_to_git

+This directory should be created when cloning or pulling this git repository.

setup.py

-from setuptools import setup, find_packages
-setup(
-    name="PyCaosDB Integration Tests",
-    version="0.1.0",
-    packages=find_packages(),
-    tests_require=["nose>=1.0", "python-dateutil>=2.8.1"],
-)

test_profile/custom/caosdb-server/.add_dir_to_git

+This directory should be created when cloning or pulling this git repository.

test_profile/custom/caosdb-server/scripting/home/.pylinkahead.ini

+[Connection]
+url = https://localhost:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file

test_profile/custom/debug-scripting-bin

+../../resources
\ No newline at end of file

test_profile/paths/authtoken/.add_dir_to_git

+This directory should be created when cloning or pulling this git repository.

test_profile/paths/extroot/.add_dir_to_git

+This directory should be created when cloning or pulling this git repository.

test_profile/profile.yml

+default:
+  # Optionally, specify a base compose file. The default base compose file is
+  # located in ./compose/docker-compose-default.yml. A different compose file
+  # may be useful when other services shall be included, e.g. nginx or django.
+  # base_compose_file: "compose/docker-compose-default.yml"
+
+  # Paths to be mounted into Docker, all entries are optional.
+  paths:
+    # Needed for file-exchange integration tests
+    extroot:
+      "": "paths/extroot"
+
+  # Docker building configuration
+  # References can be either hashes of commits or branch names
+  refs:
+    # SERVER: dev
+    # PYLIB: dev
+    # MYSQLBACKEND: dev
+    # WEBUI: dev
+    # ADVANCEDUSERTOOLS: dev
+
+  # General configuration options
+  conf:
+    # Shall the SQL & caosroot dumps at custom/other/restore/ be used?
+    # restore: false
+    restore: true
+    # Shall the server run in debug mode?
+    # This will bind-mount the following directories from custom into the Docker
+    # container:
+    # - debug-authtoken :: Authentication tokens will be stored here.
+    # - debug-scripting-bin :: Used as the server-side scripting bin dir.
+    # debug: false
+    debug: true
+    debug_authtoken_dir: "/tmp/linkahead-pyinttest/authtoken/"
+    # URL of the docker registry.  Set to "" to look locally.
+    # registry_server: "gitlab.indiscale.com:5050"
+    # The account for accessing the registry server.  "" means no account.
+    # registry_account: ""
+    # The secret token for accessing the registry server.  "" means no token.
+    # registry_token: ""
+    # Name of the docker image
+    # image: "caosdb/src/caosdb-deploy"
+    # Tag of the docker image
+    # tag: "latest"
+    # Name of the main Docker container.  Set to "" to use an auto-generated
+    # name, which is necessary for running multiple instances
+    # container_name: "linkahead"
+    # The time zone for the server
+    timezone: "Cuba"
+
+    # Network settings.
+    network:
+      # The subnet for the Docker containers
+      # default: auto selected by docker
+      # You can set it with:
+      # subnet: 10.3.128.0/17
+      # Port for accessing LinkAhead via HTTPS
+      # port_ssl: 10443
+      # Port for accessing LinkAhead via plain HTTP (not recommended when
+      # accessible from untrusted networks, but ok for testing or when behind
+      # a proxy)
+      # port_plain: 8000
+      # Port for GRPC end-point via HTTPS
+      # port_grpc_ssl: 8443
+      # Port for GRPC end-point via plain HTTP
+      # port_grpc_plain: 8080
+      # Port for debugging the LinkAhead JVM
+      # port_debug: 9000
+      # Port for profiling the LinkAhead JVM via JMX
+      # port_profiler: 9090
+      # listen to ip address ("" means any)
+      # bind_ip: "127.0.0.1"
+
+    server:
+      # All the keys of conf are set as environment variables in the server
+      # container before the server start. This overrides the server.conf
+      # settings in any other files, even the settings from
+      # `custom/caosdb-server/conf/ext/server.conf.d/`
+      # Check out conf/core/server.conf in the caosdb-server repository for
+      # options.
+      #
+      # When the conf variables are unset, the server uses its default values
+      # or the values from the `server.conf.d` directory.
+      conf:
+        # uncomment to enable the anonymous user
+        # auth_optional: TRUE
+        # uncomment to use the your custom authtoken config. See
+        # `conf/core/authtoken.example.yaml` for examples.
+        # Note: The path is relative to the caosdb server's root directory.
+        authtoken_config: conf/core/authtoken.example.yaml
+
+        # Add an integration-test key which tells the server that
+        # tests using this key are allowed to, e.g., clear the
+        # database.
+        _CAOSDB_INTEGRATION_TEST_SUITE_KEY: "_CAOSDB_PYINTTEST_SUITE"
+        TRANSACTION_BENCHMARK_ENABLED: "TRUE"
+        # SERVER_SIDE_SCRIPTING_BIN_DIRS: ""
+
+        # HTTPS port of the grpc end-point
+        # grpc_server_port_https: 8443
+        # HTTP port of the grpc end-point
+        # grpc_server_port_http: 8080
+
+  # Development configuration options
+  # devel:
+    # Copy the caosdb-server jar from this location into the Docker container.
+    # Note that this is implemented by copying the file to
+    # custom/caosdb-server/target/, any file there will be overwritten.
+    # jar: /var/build/caosdb-server/0123abcd/target/caosdb-server-<version>-jar-with-dependencies.jar

tests/test_file.py

@@ -535,10 +535,9 @@ def test_insert_files_in_dir_with_symlink():
             flags={
                 "InsertFilesInDir": path_on_server +
                 "linked_subfolder"})
-        assert_is_not_none(c.messages["Warning", 6])
-        assert_equal(c.messages["Warning", 6][0],
-                     "Directory or file is symbolic link: " + path_on_server +
-                     "linked_subfolder")
+        assert c.messages.get("Warning", 6, exact=True) is not None
+        assert (c.messages.get("Warning", 6, exact=True).description ==
+                "Directory or file is symbolic link: " + path_on_server + "linked_subfolder")
 
         c = models.Container()
         c.retrieve(
@@ -548,11 +547,11 @@ def test_insert_files_in_dir_with_symlink():
                 "InsertFilesInDir": "--force-allow-symlinks " +
                 path_on_server +
                 "linked_subfolder"})
-        assert_is_not_none(c.messages["Info", 0])
-        assert_equal(c.messages["Info", 0][0],
-                     "Files count in linked_subfolder/: 1")
+        assert c.messages.get("Info", 0, exact=True) is not None
+        assert c.messages.get(
+            "Info", 0, exact=True).description == "Files count in linked_subfolder/: 1"
         assert c[0].name is None
-        assert_equal(c[0].path, "/linked_subfolder/test2.dat")
+        assert c[0].path == "/linked_subfolder/test2.dat"
 
         d = models.Container()
         d.insert(
@@ -562,12 +561,12 @@ def test_insert_files_in_dir_with_symlink():
                 "InsertFilesInDir": "--force-allow-symlinks " +
                 path_on_server +
                 "linked_subfolder"})
-        assert_is_not_none(c.messages["Info", 0])
-        assert_equal(d.messages["Info", 0][0],
-                     "Files count in linked_subfolder/: 1")
-        assert_true(d[0].is_valid())
+        assert d.messages.get("Info", 0, exact=True) is not None
+        assert d.messages.get(
+            "Info", 0, exact=True).description == "Files count in linked_subfolder/: 1"
+        assert d[0].is_valid() is True
         assert d[0].name is None
-        assert_equal(d[0].path, "/linked_subfolder/test2.dat")
+        assert d[0].path == "/linked_subfolder/test2.dat"
 
     finally:
         try:

tests/test_server_side_scripting.py

@@ -45,12 +45,6 @@ from pytest import mark, raises
 
 _TEST_SCRIPTS = ["not_executable", "ok", "err", "ok_anonymous"]
 
-try:
-    _SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL = get_config().get(
-        "IntegrationTests",
-        "test_server_side_scripting.bin_dir.local")
-except Exception:
-    _SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL = tempfile.mkdtemp()
 try:
     _SERVER_SIDE_SCRIPTING_BIN_DIR_SERVER = get_config().get(
         "IntegrationTests",
@@ -108,25 +102,6 @@ def setup_module():
         "SERVER_SIDE_SCRIPTING_BIN_DIRS")
     clean_database()
 
-    from os import makedirs
-    from os.path import exists, isdir, join
-    from shutil import copyfile, copymode
-    print("bin dir (local): " + str(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL))
-    print("bin dir (server): " + str(_SERVER_SIDE_SCRIPTING_BIN_DIR_SERVER))
-    print("tests scripts: " + str(_TEST_SCRIPTS))
-
-    if not exists(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL):
-        makedirs(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL)
-        _REMOVE_FILES_AFTERWARDS.append(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL)
-    assert isdir(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL)
-
-    for script_file in _TEST_SCRIPTS:
-        target = join(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL, script_file)
-        src = join(_TEST_SCRIPTS_DIR, script_file)
-        copyfile(src, target)
-        copymode(src, target)
-        _REMOVE_FILES_AFTERWARDS.append(target)
-
 
 def teardown_module():
     from os import remove
@@ -432,20 +407,16 @@ def test_anonymous_script_calling_success():
 
 @mark.local_server
 def test_evil_path():
-    subdir = os.path.join(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL, "subdir")
+    """Test that we can't "escape" from the defined bin dirs, i.e.,
+    can't execute scripts in arbitrary locations.
+
+    """
 
-    if not os.path.exists(subdir):
-        os.makedirs(subdir)
-        _REMOVE_FILES_AFTERWARDS.append(subdir)
+    # Set SSS bin dir to "subdir" sub directory
     admin.set_server_property("SERVER_SIDE_SCRIPTING_BIN_DIRS",
                               os.path.join(_SERVER_SIDE_SCRIPTING_BIN_DIR_SERVER,
                                            "subdir"))
-
-    # ok exists one level up from "subdir"
-    assert os.path.exists(
-        os.path.join(
-            _SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL,
-            "ok"))
-
+    # The parent directory is not in the list of SSS bin dirs, so the
+    # server must not allow the execution of ok.
     with raises(HTTPResourceNotFoundError):
         r = run_server_side_script("../ok")