Merge remote-tracking branch 'origin/dev' into f-readme-remote-server

.docker-base/Dockerfile

 # Use docker as parent image
-FROM docker:19.03.0
+FROM docker:20.10.3
 
 # http://bugs.python.org/issue19846
 ENV LANG C.UTF-8
@@ -9,6 +9,37 @@ RUN apk add --no-cache py3-pip python3 python3-dev gcc make \
     git bash curl gettext  py3-requests 
 RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \
     libxml2 libxml2-dev
+RUN apk add --no-cache ca-certificates
+
+# install rust (needed for compiling a docker-compose dependency)
+# This is necessary until alpine comes with an up to date RUST
+# copied from https://github.com/rust-lang/docker-rust/blob/bbc7feb12033da3909dced4e88ddbb6964fbc328/1.50.0/alpine3.13/Dockerfile
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.50.0
+
+RUN set -eux; \
+    apkArch="$(apk --print-arch)"; \
+    case "$apkArch" in \
+        x86_64) rustArch='x86_64-unknown-linux-musl'; rustupSha256='05c5c05ec76671d73645aac3afbccf2187352fce7e46fc85be859f52a42797f6' ;; \
+        aarch64) rustArch='aarch64-unknown-linux-musl'; rustupSha256='6a8a480d8d9e7f8c6979d7f8b12bc59da13db67970f7b13161ff409f0a771213' ;; \
+        *) echo >&2 "unsupported architecture: $apkArch"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.23.1/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+
+RUN pip3 install wheel
 RUN pip3 install docker-compose==1.25
 
 # Script for waiting on LA server

.docker/docker-compose.yml

@@ -7,7 +7,7 @@ services:
     networks:
       - caosnet
   caosdb-server:
-    image: "$CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+    image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
     user: 999:999
     depends_on:
       - sqldb

.gitlab-ci.yml

@@ -19,8 +19,8 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-pyinttest/testenv:latest
-   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/caosdb-pyinttest/base:latest
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/testenv:latest
+   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
    DEPLOY_REF: dev
 
 stages:
@@ -79,7 +79,7 @@ stages:
 # file-system features.
 #
 services:
-  - docker:19.03.0-dind
+  - docker:20.10.3-dind
 
 test:
   tags: [docker]
@@ -103,23 +103,22 @@ test:
       - echo $CI_COMMIT_REF_NAME
       - echo $CI_REGISTRY_IMAGE
 
-      - docker login -u gitlab+deploy-token-ci-pull -p $TOKEN_CI_PULL $CI_REGISTRY_INDISCALE
+      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       - time docker load < /image-cache/caosdb-${F_BRANCH}.tar || true
       - if [[ "$CAOSDB_TAG" == "" ]]; then
           if echo "$F_BRANCH" | grep -c "^f-" ; then
             CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH}-latest;
-            docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEPLOY_REF}-latest ;
+            docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEPLOY_REF}-latest ;
           else
             CAOSDB_TAG=${DEPLOY_REF}-latest;
           fi;
         fi
-      - docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev-latest ;
+      - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev-latest ;
       - echo $CAOSDB_TAG
 
       - time docker load < /image-cache/caosdb-pyint-testenv-${CI_COMMIT_REF_NAME}.tar || true
       - time docker load < /image-cache/mariadb-${F_BRANCH}.tar || true
       - time docker load < /image-cache/caosdb-${F_BRANCH}.tar || true
-      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       - docker pull $CI_REGISTRY_IMAGE
       - cd .docker
         # here the server and the mysql backend docker are being started
@@ -151,6 +150,7 @@ build-testenv:
   tags: [cached-dind]
   image: docker:19.03
   stage: setup
+  timeout: 2h
   needs: []
   script:
       - df -h
@@ -217,7 +217,5 @@ store-version:
   only:
     - dev
     - master
-    # TODO remove
-    - f-versions
     - web
 

tests/test_issues_server.py

@@ -29,8 +29,8 @@ import tempfile
 import time
 
 import caosdb as db
-from caosdb.exceptions import TransactionError
 import pytest
+from caosdb.exceptions import TransactionError
 
 
 def setup_module():
@@ -199,3 +199,37 @@ def test_issue_110():
     print(result[0])
     print(a1)
     assert result[0].id == a1.id
+
+
+def test_issue_120():
+    """Editing entities that were created with a no longer existing user leads
+    to a server error.
+
+    The server should throw an error when CHECK_ENTITY_ACL_ROLES_MODE=MUST,
+    otherwise a warning.
+    """
+    # insert an entity
+    entity = db.RecordType("TestRT").insert(flags={"ACL": None})
+
+    db.administration.set_server_property("CHECK_ENTITY_ACL_ROLES_MODE",
+                                          "SHOULD")
+    # update with non-existing user, realm and role
+    entity.deny(
+        realm="CaosDB",
+        username="NON_EXISTING_USER",
+        permission="USE:AS_REFERENCE")
+    entity.update(flags={"ACL": None})
+    assert entity.messages["Warning", 1104][0] == "User Role does not exist."
+
+    entity.deny(
+        realm="NON_EXISTING_REALM",
+        username="NON_EXISTING_USER",
+        permission="USE:AS_REFERENCE")
+    entity.update(flags={"ACL": None})
+    assert entity.messages["Warning", 1104][0] == "User Role does not exist."
+
+    entity.deny(
+        role="ALSO_NON_EXISTING_ROLE",
+        permission="USE:AS_REFERENCE")
+    entity.update(flags={"ACL": None})
+    assert entity.messages["Warning", 1104][0] == "User Role does not exist."

tests/test_query.py

@@ -1054,6 +1054,8 @@ def test_query_cache():
     response = query.execute()
     assert response == 1
     assert query.cached == False
+    assert query.etag is not None
+    etag = query.etag
 
     # cached == True
     query = h.Query("COUNT TestRT")
@@ -1061,6 +1063,7 @@ def test_query_cache():
     response = query.execute()
     assert response == 1
     assert query.cached == True
+    assert query.etag == etag
 
     # cached == True
     query = h.Query("FIND TestRT")
@@ -1068,6 +1071,7 @@ def test_query_cache():
     response = query.execute(unique=True)
     assert response.id == entity.id
     assert query.cached == True
+    assert query.etag == etag
 
     # cached == True
     query = h.Query("SELECT bla FROM TestRT")
@@ -1075,6 +1079,7 @@ def test_query_cache():
     response = query.execute(unique=True)
     assert response.id == entity.id
     assert query.cached == True
+    assert query.etag == etag
 
     # no cache flag
     query = h.Query("SELECT bla FROM TestRT")
@@ -1082,6 +1087,7 @@ def test_query_cache():
     response = query.execute(unique=True, cache=False)
     assert response.id == entity.id
     assert query.cached == False
+    assert query.etag == etag
 
     # cached == True
     query = h.Query("SELECT bla FROM TestRT")
@@ -1089,6 +1095,7 @@ def test_query_cache():
     response = query.execute(unique=True)
     assert response.id == entity.id
     assert query.cached == True
+    assert query.etag == etag
 
     # write resets cache
     another_entity = h.Record().add_parent("TestRT").insert()
@@ -1099,6 +1106,8 @@ def test_query_cache():
     response = query.execute()
     assert response == 2
     assert query.cached == False
+    assert query.etag != etag
+    new_etag = query.etag
 
     # cached == True
     query = h.Query("COUNT TestRT")
@@ -1106,6 +1115,7 @@ def test_query_cache():
     response = query.execute()
     assert response == 2
     assert query.cached == True
+    assert query.etag == new_etag
 
 
 def test_query_cache_with_permissions():

tests/test_server_side_scripting.py

@@ -395,6 +395,7 @@ def test_anonymous_script_calling_success():
     assert xml.xpath("/Response/script/@code")[0] == "0"
 
 
+@mark.local_server
 def test_evil_path():
     subdir = os.path.join(_SERVER_SIDE_SCRIPTING_BIN_DIR_LOCAL, "subdir")
     if not os.path.exists(subdir):