Merge branch 'dev' into f-test_insert_files_in_dir

.gitlab-ci.yml

@@ -20,12 +20,6 @@
 
 variables:
    CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-pyint-testenv
-   # When using dind, it's wise to use the overlayfs driver for
-   # improved performance.
-   DOCKER_DRIVER: overlay2
-
-services:
-  - docker:19.03.0-dind
 
 stages:
   - setup
@@ -79,6 +73,9 @@ stages:
 # These volumes are inteded to be used for testing server-side scripting and
 # file-system features.
 #
+services:
+  - docker:19.03.0-dind
+
 test:
   tags: [docker]
   stage: test
@@ -88,11 +85,14 @@ test:
             CAOSDB_TAG=dev-latest;
         fi
       - echo $CAOSDB_TAG
-      - KNOWN_TAGS=$(curl -u testuser:$CI_REGISTRY_PASSWORD -X GET https://$CI_REGISTRY/v2/caosdb/tags/list)
+      - KNOWN_TAGS=$(curl -u indiscale:$CI_REGISTRY_PASSWORD -X GET https://$CI_REGISTRY/v2/caosdb/tags/list)
       - echo $KNOWN_TAGS
         # test if the caosdb registry knows our current tag
       - echo $KNOWN_TAGS | grep "$CAOSDB_TAG"
-      - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+      - time docker load < /image-cache/caosdb-pyint-testenv.tar || true
+      - time docker load < /image-cache/mariadb.tar || true
+      - time docker load < /image-cache/caosdb.tar || true
+      - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
       - docker pull $CI_REGISTRY_IMAGE:latest
       - cd .docker
         # here the server and the mysql backend docker are being started
@@ -115,19 +115,18 @@ test:
     expire_in: 1 week
 
 build-testenv:
-  tags: [notls]
-  image: docker:18.09
+  tags: [cached-dind]
+  image: docker:19.03
   stage: setup
-  only:
-      - schedules
   script:
       - df -h
-      - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+      - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
         # use here general latest or specific branch latest...
       - docker build 
         --file .docker/Dockerfile
         -t $CI_REGISTRY_IMAGE:latest .
       - docker push $CI_REGISTRY_IMAGE:latest
+      - docker save $CI_REGISTRY_IMAGE:latest > /image-cache/caosdb-pyint-testenv.tar
 
 cert:
   tags: [docker]
@@ -141,7 +140,7 @@ cert:
       - cd .docker
       - CAOSHOSTNAME=caosdb-server ./cert.sh
 style:
-  tags: [notls]
+  tags: [docker]
   stage: style
   image: $CI_REGISTRY_IMAGE:latest
   script:

tests/test_administration.py

@@ -363,7 +363,7 @@ def test_insert_user_failure_name_in_use():
     test_insert_user_success()
     with assert_raises(ClientErrorException) as cm:
         test_insert_user_success()
-    assert_equal(cm.exception.msg, "User name is yet in use.")
+    assert_equal(cm.exception.msg, "User name is already in use.")
 
 
 @with_setup(setup, teardown)

tests/test_authentication.py

@@ -27,14 +27,28 @@
 """
 
 import os
+from sys import hexversion
+from urllib.parse import urlparse
+from http.client import HTTPSConnection
+import ssl
 from subprocess import call, check_output
+from lxml import etree
 from pytest import skip
 from caosdb.exceptions import LoginFailedException
 import caosdb as h
-from nose.tools import assert_false, assert_true, assert_is_none, assert_raises, assert_equal, assert_is_not_none, nottest  # @UnresolvedImport
+from nose.tools import (assert_false, assert_true, assert_is_none,
+                        assert_raises, assert_equal, assert_is_not_none,
+                        nottest, with_setup)
 from caosdb.connection.connection import _Connection
 
 
+def setup():
+    try:
+        h.execute_query("FIND Test*").delete()
+    except Exception as e:
+        print(e)
+
+
 def test_pass():
     if not h.get_config().has_option("Connection", "password_method") or not h.get_config().get("Connection", "password_method") == "pass":
         skip()
@@ -43,22 +57,10 @@ def test_pass():
 
 
 def test_https_support():
-    from sys import hexversion
-    if hexversion < 0x02070900:
-        raise Exception("version " + str(hex(hexversion)))
-    elif 0x02999999 < hexversion < 0x03020000:
+    if 0x02999999 < hexversion < 0x03020000:
         raise Exception("version " + str(hex(hexversion)))
-    try:
-        # python2
-        from httplib import HTTPSConnection
-        from urlparse import urlparse
-    except:
-        # python 3
-        from urllib.parse import urlparse
-        from http.client import HTTPSConnection
-    import ssl
-
-    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)  # @UndefinedVariable
+
+    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
     context.verify_mode = ssl.CERT_REQUIRED
     if hasattr(context, "check_hostname"):
         context.check_hostname = True
@@ -82,3 +84,47 @@ def test_login_via_post_form_data_failure():
                 "username": h.get_config().get("Connection", "username"),
                 "password": "wrongpassphrase"
             })
+
+
+def test_anonymous_setter():
+    """ this test verifies that the "test_login_while_anonymous_is_active" is
+    effective."""
+
+    # activate anonymous user
+    h.administration.set_server_property("AUTH_OPTIONAL", "TRUE")
+
+    # connect without auth-token
+    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+    context.verify_mode = ssl.CERT_REQUIRED
+    context.load_verify_locations(h.get_config().get("Connection", "cacert"))
+
+    url = h.get_config().get("Connection", "url")
+    fullurl = urlparse(url)
+
+    http_con = HTTPSConnection(
+        str(fullurl.netloc), timeout=200, context=context)
+
+    http_con.request(method="GET", headers={}, url=str(fullurl.path) + "Info")
+    body = http_con.getresponse().read()
+
+    xml = etree.fromstring(body)
+    # verify unauthenticated
+    assert xml.xpath("/Response/UserInfo/Roles/Role")[0].text == "anonymous"
+
+
+@with_setup(setup, setup)
+def test_login_while_anonymous_is_active():
+    # activate anonymous user
+    h.administration.set_server_property("AUTH_OPTIONAL", "TRUE")
+
+    # logout
+    h.get_connection()._logout()
+
+    body = h.get_connection().retrieve(
+        entity_uri_segments=["Entity"],
+        reconnect=True).read()
+    xml = etree.fromstring(body)
+
+    # pylib did the login even though the anonymous user is active
+    assert xml.xpath(
+        "/Response/UserInfo/Roles/Role")[0].text == "administration"

tests/test_deletion.py

@@ -151,11 +151,7 @@ def test_deletion():
     assert_is_not_none(cr2.id)
 
     c.extend([cr1, sr, d])
-    try:
-        assert_raises(h.TransactionError, c.delete)
-    except:
-        import time
-        time.sleep(120)
+    assert_raises(h.TransactionError, c.delete)
     assert_true(c.has_errors())
     assert_equal(int(c.get_errors()[0].code), 12)
 

tests/test_file.py

@@ -577,6 +577,11 @@ def test_insert_files_in_dir_with_symlink():
 
 @with_setup(None, teardown)
 def test_insert_files_in_dir():
+    """ test if files in directories can be inserted as symlinks via the
+    InsertFilesInDir flag. This tests also verifies that the job can be
+    executed for the root directory after the child directory and only the new
+    files are being inserted.
+    """
     path = get_config().get("IntegrationTests",
                             "test_files.test_insert_files_in_dir.local") + "testfolder/"
     path_on_server = get_config().get("IntegrationTests",