Merge branch 'dev' into f-string-ids

.gitlab-ci.yml

@@ -158,7 +158,7 @@ test:
       - cat hash_pylib
         # Run the actual tests. This starts a new docker container within which
         # the tests run. The return value is stored in .docker/result
-      - /bin/sh ./run.sh
+      - /bin/sh ./run.sh | tee ../tox_output.log
 
         # Save logs
       - docker logs docker-caosdb-server-1 &> ../caosdb_log.txt
@@ -177,6 +177,9 @@ test:
         # the pyinttest docker writes the return value of the tests into the
         # file result
       - rc=`cat .docker/result`
+      - if [ "$r" != "0" ]; then
+          awk '/=+ FAILURES/, /=+ / && ! /=+ FAILURES/' tox_output.log;
+        fi;
       - exit $rc
   dependencies: [cert]
   timeout: 3h

CHANGELOG.md

@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Tests for [caosdb-pylib#90](https://gitlab.com/caosdb/caosdb-pylib/-/issues/90): `Entity.get_parents_recursively()` did not work for unretrieved
   parents.
 * Test for [caosdb-server#192](https://gitlab.com/caosdb/caosdb-server/-/issues/192)
+* Test for miscellaneous *-too-long errors.
 
 ### Changed (for changes in existing functionality)
 
@@ -67,5 +68,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * Tests for NaN Double Values (see https://gitlab.com/caosdb/caosdb-server/issues/41)
 * Tests for name queries. [caosdb-server#51](https://gitlab.com/caosdb/caosdb-server/-/issues/51)
+* Server-side scripting is more tolerant to Pandas warnings now. (https://gitlab.indiscale.com/caosdb/src/caosdb-pyinttest/-/issues/21)
 
 ### Security (in case of vulnerabilities)

README.md

@@ -13,9 +13,17 @@ CaosDB project.
   be found elsewhere):
   - The CaosDB server must have debugging enabled.
   - The database should be empty.
+  - There should be a symlink named `debug-scripting-bin` inside the `custom` profile directory,
+    which points to the `resources` directory here in the pyinttest repository.  You can create a
+    symlink like that with the following command (adapt paths to match your setting):  
+    `custom$ ln -s ../../../../caosdb-pyinttest/resources debug-scripting-bin`
 - Modify `pycaosdb.ini.template` and save it as `pycaosdb.ini`, taking care of the following points:
   - Certificates must be valid and be specified in `pycaosdb.ini`.
   - Server-side scripting paths must be given, otherwise server-side scripting will be omitted.
+    - The local path `test_server_side_scripting.bin_dir.local` should point to a
+      `linkahead-server/scripting/bin` somwhere.
+    - The remote path `test_server_side_scripting.bin_dir.server` should probably be something like
+      `/opt/caosdb/git/caosdb-server/scripting/bin-debug`.
   - Paths for the file tests must exist, or be creatable by the testing script and the server.
 - Run the tests with `pytest` or `pytest-3` (depending on your system).
 - If you want to run just a single test, you can also select a single test file:  

tests/test_datatype_inheritance.py

@@ -28,7 +28,7 @@ from caosdb.connection.connection import get_connection
 from caosdb.common.models import is_temporary_id
 
 from caosdb.exceptions import TransactionError
-from pytest import raises
+from pytest import mark, raises
 
 
 def setup_function(function):
@@ -155,6 +155,7 @@ def test_datatype_overriding_update():
     assert str("DATETIME").lower() == rt.get_properties()[0].datatype.lower()
 
 
+@mark.xfail(reason="https://gitlab.com/linkahead/linkahead-server/-/issues/257")
 def test_recordtype_to_record():
     rt = RecordType(name="SimpleTextRecordType")
     rt.datatype = "TEXT"
@@ -170,9 +171,9 @@ def test_recordtype_to_record():
 
     rec = Record().add_parent(name="SimpleTextRecordType").insert()
     assert rec.is_valid()
-    # TODO
-    # assert rec.datatype is not None
-    # assert str("TEXT").lower() == rec.datatype.lower()
+    # This fails to inherit the datatype
+    assert rec.datatype is not None
+    assert str("TEXT").lower() == rec.datatype.lower()
 
 
 def test_concrete_property():

tests/test_error_stuff.py

@@ -30,7 +30,7 @@ Created on 19.02.2015.
 @author: tf
 
 """
-import caosdb as h
+import linkahead as db
 from caosdb.exceptions import (AmbiguousEntityError,
                                EntityDoesNotExistError, EntityError,
                                EntityHasNoDatatypeError,
@@ -43,7 +43,7 @@ import pytest
 
 def setup_function(function):
     try:
-        h.execute_query("FIND ENTITY").delete()
+        db.execute_query("FIND ENTITY").delete()
     except BaseException:
         pass
 
@@ -55,7 +55,7 @@ def teardown_function(function):
 
 def test_retrieval_no_exception_raised():
     """Test whether retrieval fails but error is suppressed."""
-    p = h.Property(name="TestNon-ExsistentProperty").retrieve(
+    p = db.Property(name="TestNon-ExsistentProperty").retrieve(
         unique=True, raise_exception_on_error=False)
     assert not p.is_valid()
     assert (p.id is None or is_temporary_id(p.id))
@@ -68,7 +68,7 @@ def test_retrieval_exception_raised():
     """
     propname = "TestNon-ExistentProperty"
     with pytest.raises(TransactionError) as te:
-        h.Property(name="TestNon-ExistentProperty").retrieve(unique=True,
+        db.Property(name="TestNon-ExistentProperty").retrieve(unique=True,
                                                               raise_exception_on_error=True)
     assert len(te.value.errors) == 1
     ee = te.value.errors[0]
@@ -90,19 +90,19 @@ def test_ambiguous_retrieval():
     raised correctly if there are two possible candidates.
 
     """
-    h.RecordType(name="TestType").insert()
-    h.Record(name="TestRec").add_parent(name="TestType").insert()
+    db.RecordType(name="TestType").insert()
+    db.Record(name="TestRec").add_parent(name="TestType").insert()
     # Insert twice, so unique=False
-    h.Record(name="TestRec").add_parent(name="TestType").insert(unique=False)
+    db.Record(name="TestRec").add_parent(name="TestType").insert(unique=False)
     with pytest.raises(TransactionError) as te:
-        h.Record(name="TestRec").retrieve()
+        db.Record(name="TestRec").retrieve()
     assert te.value.has_error(AmbiguousEntityError)
     assert te.value.errors[0].entity.name == "TestRec"
 
 
 def test_insertion_no_exception_raised():
     """Test whether insertion fails but no error is raised."""
-    p = h.Property(name="TestNoTypeProperty").insert(
+    p = db.Property(name="TestNoTypeProperty").insert(
         raise_exception_on_error=False)
     assert not p.is_valid()
     assert (p.id is None or is_temporary_id(p.id))
@@ -110,7 +110,7 @@ def test_insertion_no_exception_raised():
 
 def test_insertion_exception_raised():
     """Test insertion of a property with missing datatype."""
-    p = h.Property(name="TestNoTypeProperty")
+    p = db.Property(name="TestNoTypeProperty")
     with pytest.raises(TransactionError) as te:
         p.insert(raise_exception_on_error=True)
     assert te.value.has_error(EntityHasNoDatatypeError)
@@ -118,7 +118,7 @@ def test_insertion_exception_raised():
 
 def test_insertion_with_invalid_parents():
     with pytest.raises(TransactionError) as te:
-        p = h.Property(
+        p = db.Property(
             name="TestNoTypeProperty",
             datatype="Text").add_parent(
             id=-1)
@@ -137,7 +137,7 @@ def test_insertion_with_invalid_parents():
 
 def test_insertion_with_invalid_properties():
     with pytest.raises(TransactionError) as te:
-        p = h.Property(
+        p = db.Property(
             name="TestNoTypeProperty",
             datatype="Text").add_property(
             id=-1)
@@ -158,11 +158,11 @@ def test_entity_does_not_exist():
     EntityDoesNotExistErrors.
 
     """
-    p1 = h.Property(name="TestNon-ExistentProperty1").retrieve(
+    p1 = db.Property(name="TestNon-ExistentProperty1").retrieve(
         raise_exception_on_error=False)
-    p2 = h.Property(name="TestNon-ExistentProperty2").retrieve(
+    p2 = db.Property(name="TestNon-ExistentProperty2").retrieve(
         raise_exception_on_error=False)
-    p3 = h.Property(name="TestNon-ExistentProperty3").retrieve(
+    p3 = db.Property(name="TestNon-ExistentProperty3").retrieve(
         raise_exception_on_error=False)
     # None of them should exist
     assert not p1.is_valid()
@@ -172,17 +172,17 @@ def test_entity_does_not_exist():
     assert not p3.is_valid()
     assert (p3.id is None or is_temporary_id(p3.id))
 
-    pe = h.Property(name="TestExistentProperty", datatype="text").insert()
+    pe = db.Property(name="TestExistentProperty", datatype="text").insert()
 
-    c = h.Container().extend(
+    c = db.Container().extend(
         [
-            h.Property(
+            db.Property(
                 name="TestNon-ExistentProperty1"),
-            h.Property(
+            db.Property(
                 name="TestNon-ExistentProperty2"),
-            h.Property(
+            db.Property(
                 name="TestNon-ExistentProperty3"),
-            h.Property(
+            db.Property(
                 name="TestExistentProperty")])
 
     with pytest.raises(TransactionError) as te:
@@ -200,11 +200,11 @@ def test_insert_existent_entity():
     UniqueNamesError.
 
     """
-    p1 = h.Property(name="TestNon-ExistentProperty1").retrieve(
+    p1 = db.Property(name="TestNon-ExistentProperty1").retrieve(
         raise_exception_on_error=False)
-    p2 = h.Property(name="TestNon-ExistentProperty2").retrieve(
+    p2 = db.Property(name="TestNon-ExistentProperty2").retrieve(
         raise_exception_on_error=False)
-    p3 = h.Property(name="TestNon-ExistentProperty3").retrieve(
+    p3 = db.Property(name="TestNon-ExistentProperty3").retrieve(
         raise_exception_on_error=False)
     # None of them should exist
     assert not p1.is_valid()
@@ -214,21 +214,21 @@ def test_insert_existent_entity():
     assert not p3.is_valid()
     assert (p3.id is None or is_temporary_id(p3.id))
 
-    pe = h.Property(name="TestExistentProperty", datatype="text").insert()
+    pe = db.Property(name="TestExistentProperty", datatype="text").insert()
     assert pe.is_valid()
 
-    c = h.Container().extend(
+    c = db.Container().extend(
         [
-            h.Property(
+            db.Property(
                 name="TestNon-ExistentProperty1",
                 datatype="text"),
-            h.Property(
+            db.Property(
                 name="TestNon-ExistentProperty2",
                 datatype="text"),
-            h.Property(
+            db.Property(
                 name="TestNon-ExistentProperty3",
                 datatype="text"),
-            h.Property(
+            db.Property(
                 name="TestExistentProperty",
                 datatype="text")])
 
@@ -244,31 +244,31 @@ def test_insert_existent_entity():
 
 
 def test_double_insertion():
-    c1 = h.Container()
+    c1 = db.Container()
 
     c1.append(
-        h.Property(
+        db.Property(
             name="TestSimpleTextProperty",
             description="simple text property (from test_error_stuff.py)",
             datatype='text'))
     c1.append(
-        h.Property(
+        db.Property(
             name="TestSimpleDoubleProperty",
             description="simple double property (from test_error_stuff.py)",
             datatype='double'))
     c1.append(
-        h.Property(
+        db.Property(
             name="TestSimpleIntegerProperty",
             description="simple integer property (from test_error_stuff.py)",
             datatype='integer'))
     c1.append(
-        h.Property(
+        db.Property(
             name="TestSimpleDatetimeProperty",
             description="simple datetime property (from test_error_stuff.py)",
             datatype='datetime'))
 
     c1.append(
-        h.RecordType(
+        db.RecordType(
             name="TestSimpleRecordType",
             description="simple recordType (from test_error_stuff.py)").add_property(
                 name='TestSimpleTextProperty').add_property(
@@ -278,30 +278,30 @@ def test_double_insertion():
 
     c1.insert()
 
-    c2 = h.Container()
+    c2 = db.Container()
     c2.append(
-        h.Property(
+        db.Property(
             name="TestSimpleTextProperty",
             description="simple text property (from test_error_stuff.py)",
             datatype='text'))
     c2.append(
-        h.Property(
+        db.Property(
             name="TestSimpleDoubleProperty",
             description="simple double property (from test_error_stuff.py)",
             datatype='double'))
     c2.append(
-        h.Property(
+        db.Property(
             name="TestSimpleIntegerProperty",
             description="simple integer property (from test_error_stuff.py)",
             datatype='integer'))
     c2.append(
-        h.Property(
+        db.Property(
             name="TestSimpleDatetimeProperty",
             description="simple datetime property (from test_error_stuff.py)",
             datatype='datetime'))
 
     c2.append(
-        h.RecordType(
+        db.RecordType(
             name="TestSimpleRecordType",
             description="simple recordType (from test_error_stuff.py)").add_property(
                 name='TestSimpleTextProperty').add_property(
@@ -323,7 +323,7 @@ def test_update_acl_errors():
     `Entity.update_acl`
 
     """
-    rec_ne = h.Record("TestRecordNonExisting")
+    rec_ne = db.Record("TestRecordNonExisting")
 
     with pytest.raises(TransactionError) as te:
 
@@ -332,12 +332,36 @@ def test_update_acl_errors():
     assert te.value.has_error(EntityDoesNotExistError)
     assert te.value.errors[0].entity.name == rec_ne.name
 
-    rt = h.RecordType(name="TestType").insert()
-    rec = h.Record(name="TestRecord").add_parent(rt).insert()
-    h.Record(name=rec.name).add_parent(rt).insert(unique=False)
+    rt = db.RecordType(name="TestType").insert()
+    rec = db.Record(name="TestRecord").add_parent(rt).insert()
+    db.Record(name=rec.name).add_parent(rt).insert(unique=False)
 
     with pytest.raises(TransactionError) as te:
 
-        h.Record(name=rec.name).update_acl()
+        db.Record(name=rec.name).update_acl()
 
     assert te.value.has_error(AmbiguousEntityError)
+
+
+def test_URI_too_long():
+    """Some tests for variours URI too long commands.
+
+    See for example https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/180
+
+    """
+
+    short = 100
+    uri_long = 819
+    header_long = 815
+
+    with pytest.raises(db.TransactionError) as excinfo:
+        db.execute_query("0123456789" * short)
+    assert "Parsing" in excinfo.value.msg
+
+    with pytest.raises(db.HTTPURITooLongError) as excinfo:
+        db.execute_query("0123456789" * uri_long)
+    assert "414" in excinfo.value.msg
+
+    with pytest.raises(db.HTTPURITooLongError) as excinfo:
+        db.execute_query("0123456789" * header_long)
+    assert "431" in excinfo.value.msg

tests/test_issues_server.py

 # -*- coding: utf-8 -*-
 # This file is a part of the CaosDB Project.
 #
-# Copyright (c) 2020 - 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (c) 2020 - 2024 IndiScale GmbH <info@indiscale.com>
 # Copyright (c) 2022 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (c) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-# Copyright (c) 2021 - 2022 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (c) 2021 - 2024 Timm Fitschen <t.fitschen@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -19,18 +19,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-"""Tests for issues on gitlab.com, project caosdb-server."""
+"""Tests for issues on gitlab.com, project linkahead-server."""
 
 import math
 import os
 import tempfile
 import time
 
-import caosdb as db
+import linkahead as db
 import pytest
 
-from caosdb import administration as admin
-from caosdb.exceptions import (TransactionError, HTTPClientError)
+from linkahead import administration as admin
+from linkahead.exceptions import (TransactionError, HTTPClientError, HTTPURITooLongError)
 
 CURATOR_ROLE = "curator"
 
@@ -1405,6 +1405,22 @@ See https://gitlab.com/caosdb/caosdb-server/-/issues/220"""
     assert query.cached is False, "Query after name change of parent should not be cached."
 
 
+@pytest.mark.xfail(reason="Needs fix for intermediate length strings, "
+                   "see https://gitlab.com/linkahead/linkahead-server/-/issues/101")
+def test_101():
+    """Unexpected server errors in case of intermediate length strings,
+    https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/101
+
+    """
+    # This is ok
+    very_long_string = "Long string"*10000
+    with pytest.raises(HTTPURITooLongError):
+        db.execute_query(f"FIND RECORD WITH test=\"{very_long_string}\"")
+    # This is not
+    long_string = "Long string"*100
+    assert len(db.execute_query(f"FIND RECORD WITH test=\"{long_string}\"")) == 0
+
+
 @pytest.mark.xfail(reason="Needs fix for keeping datatype, "
                    "see https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/106")
 def test_indiscale_106():
@@ -1435,6 +1451,17 @@ See https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/106
     assert db.Record(id=r.id).retrieve().get_property("prop").datatype == db.TEXT
 
 
+@pytest.mark.xfail(reason="https://gitlab.com/caosdb/caosdb-server/-/issues/230")
+def test_230_name_duplicates_in_list_datatypes():
+    """https://gitlab.com/linkahead/linkahead-server/-/issues/230"""
+    prop = db.Property(name="Test", datatype=db.TEXT).insert()
+    # RT with same name, will be used as datatype
+    rt = db.RecordType(name="Test").insert(unique=False)
+    rec = db.Record(name="TestRec").add_parent(id=rt.id).insert()
+    rec.add_property(id=rt.id, datatype=rt.id).update()  # This works since it's not a list
+    rec.add_property(id=rt.id, datatype=db.LIST(rt.id)).update()  # This fails
+
+
 @pytest.mark.xfail(reason="https://gitlab.com/caosdb/caosdb-server/-/issues/235")
 def test_235_long_name():
     """Should give an appropriate error, not just unknown server/-/issues."""
@@ -1455,3 +1482,26 @@ def test_235_long_name():
     except Exception as exc:
         assert not isinstance(exc, db.HTTPServerError)
         # TODO more specific error should be asserted
+
+
+@pytest.mark.xfail(reason="https://gitlab.com/linkahead/linkahead-server/-/issues/248")
+def test_248():
+    """Querying for entities with property fails if using ID."""
+    rt = db.RecordType(name="RT1").insert()
+    prop = db.Property(name="prop", datatype=db.DOUBLE).insert()
+    rec = db.Record().add_parent(rt).add_property(prop, value=23).insert()
+    results = db.execute_query(f"FIND Entity with {prop.id}")
+    assert len(results) == 1
+
+
+@pytest.mark.xfail(reason="https://gitlab.com/linkahead/linkahead-server/-/issues/253")
+def test_253():
+    """Value in string queries may not start with large number of digits."""
+    test_strings = [
+        "0123456789",
+        "hello" + "0123456789" * 5 + "world",
+        "0123456789" * 5 + "world",
+    ]
+    for string in test_strings:
+        results = db.execute_query(f"FIND Entity with prop={string}")
+        assert len(results) == 0

tests/test_permissions.py

@@ -36,6 +36,7 @@ from nose.tools import (assert_equal,
                         assert_is_not_none)
 from pytest import raises, mark
 
+
 test_user = "test_user"
 test_role = "test_role"
 test_pw = "passphrase1P!"
@@ -187,6 +188,33 @@ def test_basic_acl_stuff():
     assert_false("DELETE" in other_role_permissions)
 
 
+@mark.xfail(reason="fix needed: https://gitlab.com/linkahead/linkahead-server/-/issues/247")
+def test_server_issue_247():
+    db.administration.set_server_property(
+        "QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS", "TRUE")
+    person = db.RecordType("TestPerson").insert()
+    db.Property("TestFirstName", datatype=db.TEXT).insert()
+    db.Property("TestConductor", datatype=person).insert()
+
+    dan = db.Record(
+        name="TestDaniel").add_property(
+        name="TestFirstName",
+        value="Daniel").add_parent(person).insert()
+    exp = db.RecordType(
+        name="TestExperiment").add_property(
+        name="TestConductor",
+        value=dan.id).insert()
+
+    grant_permission(person, "RETRIEVE:*")
+    grant_permission(exp, "RETRIEVE:*")
+    deny_permission(dan, "RETRIEVE:*")
+    switch_to_test_user()
+
+    assert db.execute_query(
+        "FIND ENTITY TestExperiment WHICH HAS A TestConductor=" + str(dan.id),
+        unique=True).id == exp.id
+
+
 def test_query():
     db.administration.set_server_property(
         "QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS", "TRUE")
@@ -239,10 +267,11 @@ def test_query():
             unique=True)
     '''... but works without the which clause'''
     assert db.execute_query("FIND ENTITY TestExperiment", unique=True).id == exp.id
+
     '''and with the id'''
-    assert db.execute_query(
-        "FIND ENTITY TestExperiment WHICH HAS A TestConductor=" + str(dan.id),
-        unique=True).id == exp.id
+    # assert db.execute_query(
+    #    "FIND ENTITY TestExperiment WHICH HAS A TestConductor=" + str(dan.id),
+    #    unique=True).id == exp.id
 
     '''failure - exp'''
     grant_permission(dan, "RETRIEVE:*")
@@ -1195,3 +1224,118 @@ def test_deny_update_role():
     p.name = "TestPropertyEvenNewer"
     with raises(db.TransactionError) as te:
         p.update()
+
+
+def test_query_with_invisible_reference():
+    """
+    Names of references that are not visible to the test user should not be usable as query
+    filters.
+    """
+    db.administration.set_server_property(
+        "QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS", "TRUE")
+
+    rt = db.RecordType(name="TestRT").insert()
+    rec_invisible = db.Record(name="TestInvisible").add_parent(rt).insert()
+    rec_visible = db.Record(name="TestVisible").add_parent(
+        rt).add_property(name=rt.name, value=rec_invisible.id).insert()
+    # test user is only allowed to see rec_visible, not rec_invisble
+    grant_permission(rec_visible, "RETRIEVE:*")
+    deny_permission(rec_invisible, "RETRIEVE:*")
+
+    # as admin, I'm allowed to filter this
+    switch_to_admin_user()
+    assert len(db.execute_query(f"FIND {rt.name} WITH {rt.name}={rec_invisible.name}")) == 1
+
+    switch_to_test_user()
+
+    # Retrival is forbidden
+    with raises(db.TransactionError) as te:
+        retrieved_invisible = db.Record(id=rec_invisible.id).retrieve()
+    assert te.value.has_error(db.AuthorizationError)
+
+    retrieved_visible = db.Record(id=rec_visible.id).retrieve()
+    assert retrieved_visible.name == rec_visible.name
+    assert retrieved_visible.get_property(rt.name) is not None
+    assert retrieved_visible.get_property(rt.name).value == rec_invisible.id
+
+    # We cant see rec_invisible, so its name can't be used as a valid query filter.
+    assert len(db.execute_query(
+        f"FIND {rt.name} WITH {rt.name} WITH name={rec_invisible.name}")) == 0
+    assert len(db.execute_query(f"FIND {rt.name} WITH {rt.name}={rec_invisible.name}")) == 0
+    assert len(db.execute_query(f"FIND {rt.name} WITH {rt.name} LIKE '*invis*'")) == 0
+
+
+def test_select_query_with_invisible_reference():
+    """SELECT queries must not leak property values of invisible referenced entities."""
+
+    visible_rt = db.RecordType(name="TestTypeVisible").insert()
+    invisible_rt = db.RecordType(name="TestTypeInvisible").insert()
+    other_rt = db.RecordType(name="TestTypeOther").insert()
+    prop = db.Property(name="TestProp", datatype=db.INTEGER).insert()
+
+    other_rec = db.Record(name="TestOther").add_parent(other_rt).insert()
+    referenced_visible = db.Record(name="TestReferencedVisible").add_parent(visible_rt).insert()
+    # invisible rec will have one int property, one reference to the (invisible)
+    # other rt and one to the visible rt.
+    invisible_rec = db.Record(name="TestInvisible").add_parent(invisible_rt)
+    invisible_rec.add_property(name=prop.name, value=42)
+    invisible_rec.add_property(name=other_rt.name, value=other_rec.id)
+    invisible_rec.add_property(name=visible_rt.name, value=referenced_visible.id)
+    invisible_rec.insert()
+    visible_rec = db.Record(name="TestVisible").add_parent(visible_rt.name)
+    visible_rec.add_property(name=invisible_rt.name, value=invisible_rec.id)
+    visible_rec.insert()
+
+    # Everything is there when queried as admin
+    select_query = (
+        f"SELECT name, {invisible_rt.name}, {invisible_rt.name}.name, "
+        f"{invisible_rt.name}.{prop.name}, {invisible_rt.name}.{other_rt.name}, "
+        f"{invisible_rt.name}.{other_rt.name}.name FROM {visible_rec.id}")
+    select_results = db.execute_query(select_query)
+    value_args = ["name", f"{invisible_rt.name}", (invisible_rt.name, "name"),
+                  (invisible_rt.name, prop.name), (invisible_rt.name, other_rt.name),
+                  (invisible_rt.name, other_rt.name, "name")]
+    values = select_results.get_property_values(*value_args)[0]
+    assert values[0] == visible_rec.name
+    assert values[1] == invisible_rec.id
+    assert values[2] == invisible_rec.name
+    assert values[3] == invisible_rec.get_property(prop.name).value
+    assert values[4] == other_rec.id
+    assert values[5] == other_rec.name
+
+    for rec in [referenced_visible, visible_rec]:
+        grant_permission(rec, "RETRIEVE:*")
+    for rec in [invisible_rec, other_rec]:
+        deny_permission(rec, "RETRIEVE:*")
+
+    switch_to_test_user()
+
+    select_results = db.execute_query(select_query)
+    values = select_results.get_property_values(*value_args)[0]
+    assert values[0] == visible_rec.name
+    assert values[1] == invisible_rec.id  # id is ok
+    assert values[2] == invisible_rt.name  # fall-back to property's name
+    assert values[3] is None  # prop isn't either
+    assert values[4] is None  # neither id ...
+    assert values[5] is None  # ... nor name of other rec referenced by invisible
+
+    # Special case of visible referencing invisible referencing visible
+
+    switch_to_admin_user()
+
+    select_query = (
+        f"SELECT {invisible_rt.name}.{visible_rt.name}, "
+        f"{invisible_rt.name}.{visible_rt.name}.name FROM {visible_rec.id}")
+    value_args = [(invisible_rt.name, visible_rt.name),
+                  (invisible_rt.name, visible_rt.name, "name")]
+    select_results = db.execute_query(select_query)
+    values = select_results.get_property_values(*value_args)[0]
+    assert values[0] == referenced_visible.id
+    assert values[1] == referenced_visible.name
+
+    switch_to_test_user()
+
+    select_results = db.execute_query(select_query)
+    values = select_results.get_property_values(*value_args)[0]
+    assert values[0] is None
+    assert values[1] is None

tests/test_query.py

@@ -1102,6 +1102,8 @@ def test_query_cache():
 
 
 def test_query_cache_with_permissions():
+    db.administration.set_server_property(
+        "QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS", "TRUE")
     db.RecordType("TestRT").insert()
     db.RecordType("TestRT2").insert()
     public_record = db.Record().add_parent("TestRT").insert()

tests/test_server_side_scripting.py

@@ -70,6 +70,28 @@ def clean_database():
         d.delete()
 
 
+def assert_stderr(stderr) -> None:
+    """Assert that ``stderr`` is either None or contains only the pyarrow deprecation warning.
+
+This can probably removed with Pandas 3.0, to be replaced by ``assert stderr is None``.
+
+Parameters
+----------
+stderr
+  The object to be tested.
+    """
+    if stderr is None:
+        return
+    assert isinstance(stderr, str)
+    assert stderr.split(sep="\n", maxsplit=1)[1] == """Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),
+(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)
+but was not found to be installed on your system.
+If this would cause problems for you,
+please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466
+        
+  i = __import__(m)"""
+
+
 def setup_function(function):
     clean_database()
 
@@ -148,7 +170,7 @@ def test_call_ok():
     xml = etree.parse(r)
     assert xml.xpath("/Response/script/call")[0].text == "ok"
     assert xml.xpath("/Response/script/stdout")[0].text == "ok"
-    assert xml.xpath("/Response/script/stderr")[0].text is None
+    assert_stderr(xml.xpath("/Response/script/stderr")[0].text)
     assert xml.xpath("/Response/script/@code")[0] == "0"
 
 
@@ -178,7 +200,7 @@ def test_run_server_side_script_with_file_as_positional_param():
                                       exit="123",
                                       query="COUNT ENTITY TestRT",
                                       files={"-p2": "test_file.txt"})
-    assert response.stderr is None
+    assert_stderr(response.stderr)
     assert response.code == 123
     assert response.call == ('administration/diagnostics.py '
                              '--exit=123 --query=COUNT ENTITY TestRT '
@@ -203,7 +225,7 @@ def test_run_server_side_script_with_additional_file():
                                       exit="123",
                                       query="COUNT ENTITY TestRT",
                                       files={"dummykey": "test_file.txt"})
-    assert response.stderr is None
+    assert_stderr(response.stderr)
     assert response.code == 123
     assert response.call == ('administration/diagnostics.py '
                              '--exit=123 --query=COUNT ENTITY TestRT '
@@ -250,7 +272,7 @@ def test_diagnostics_basic():
                                                              "with code 123.")
     assert xml.xpath("/Response/script/call")[0].text.startswith(
         "administration/diagnostics.py")
-    assert xml.xpath("/Response/script/stderr")[0].text is None
+    assert_stderr(xml.xpath("/Response/script/stderr")[0].text)
 
 
 def test_diagnostics_with_file_upload():
@@ -293,7 +315,7 @@ def test_diagnostics_with_file_upload():
                                                            "with code 0.")
     assert xml.xpath("/Response/script/call")[0].text.startswith(
         "administration/diagnostics.py")
-    assert xml.xpath("/Response/script/stderr")[0].text is None
+    assert_stderr(xml.xpath("/Response/script/stderr")[0].text)
 
 
 @mark.local_server
@@ -403,7 +425,7 @@ def test_anonymous_script_calling_success():
     assert xml.xpath("/Response/UserInfo/Roles/Role")[0].text == "anonymous"
     assert xml.xpath("/Response/script/call")[0].text == "ok_anonymous"
     assert xml.xpath("/Response/script/stdout")[0].text == "ok_anonymous"
-    assert xml.xpath("/Response/script/stderr")[0].text is None
+    assert_stderr(xml.xpath("/Response/script/stderr")[0].text)
     assert xml.xpath("/Response/script/@code")[0] == "0"