Merge branch 'release-v0.9.0' into 'main'

ENH: JsonSchemaExporter accepts do_not_create parameter.

See merge request !89

.coveragerc

+[html]
+show_contexts = True

.gitlab-ci.yml

@@ -131,7 +131,7 @@ unittest_py39:
   script:
     # First verify that system Python actually is 3.9
     - python3 -c "import sys; assert sys.version.startswith('3.9')"
-    - python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
+    - python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
     - tox
 
 unittest_py37:
@@ -161,7 +161,6 @@ unittest_py311:
   stage: unittest
   image: python:3.11
   script: *python_test_script
-  allow_failure: true
 
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages

.gitlab/merge_request_templates/Default.md

@@ -28,6 +28,7 @@ guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
 - [ ] Up-to-date CHANGELOG.md (or not necessary)
 - [ ] Up-to-date JSON schema (or not necessary)
 - [ ] Appropriate user and developer documentation (or not necessary)
+  - Update / write published documentation (`make doc`).
   - How do I use the software?  Assume "stupid" users.
   - How do I develop or debug the software?  Assume novice developers.
 - [ ] Annotations in code (Gitlab comments)
@@ -41,7 +42,8 @@ guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
 - [ ] I understand the intent of this MR
 - [ ] All automated tests pass
 - [ ] Up-to-date CHANGELOG.md (or not necessary)
-- [ ] Appropriate user and developer documentation (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+      documentation.
 - [ ] The test environment setup works and the intended behavior is reproducible in the test
   environment
 - [ ] In-code documentation and comments are up-to-date.

CHANGELOG.md

@@ -4,6 +4,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.9.0] - 2023-11-27 ##
+
+### Added ###
+
+* Added support for passing callables as `find_func` to the `BaseTableExporter`.
+* Added member `BaseTableExporter.all_keys`
+* Parsing from YAML now allows to give an existing model to which the YAML data model shall be
+  added.
+* The `json_schema_exporter` module which introduces tools to create a json
+  schema from a RecordType, e.g., for the usage in web forms.
+* `DataModel.get_deep(name: str)` method which uses the DataModel as a kind of cache pool.
+
+### Changed ###
+
+* A bit better error handling in the yaml model parser.
+* `TableImporter.check_datatypes` allows numeric values in string columns if
+  `strict=False` (default).
+
+### Fixed ###
+
+* `TableImporter.check_missing` in case of array-valued fields in table
+* YAML model parser has better description handling.
+
+### Documentation ###
+
+* Test coverage reports are now generated in `.tox/cov_html/` by tox.
+
 ## [0.8.0] - 2023-05-30 ##
 (Florian Spreckelsen)
 

CITATION.cff

@@ -20,6 +20,6 @@ authors:
     given-names: Stefan
     orcid: https://orcid.org/0000-0001-7214-8125
 title: CaosDB - Advanced User Tools
-version: 0.8.0
+version: 0.9.0
 doi: 10.3390/data4020083
-date-released: 2023-05-30
+date-released: 2023-11-27
\ No newline at end of file

README.md

@@ -4,6 +4,7 @@
 
 This is the **CaosDB Advanced User Tools** repository and a part of the
 CaosDB project.
+
 This project contains tools that are beyond the typical use of
 the CaosDB python client. Especially, this includes the crawler which will
 typically be used by a data curator.

README_SETUP.md

@@ -32,7 +32,10 @@ Optional h5-crawler:
 - `pip install .[h5-crawler] --user`
 
 ## Run Unit Tests
-`tox`
+
+- All tests: `tox`
+- One specific test with tox: `tox -- unittests/test_myusecase.py -k expression`
+- Or even using only pytest: `pytest unittests/test_myusecase.py -k expression`
 
 ## Run Integration Tests Locally
 

integrationtests/test.sh

@@ -16,7 +16,7 @@ fi
 OUT=/tmp/crawler.output
 ls
 cat pycaosdb.ini
-python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
+python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
 rm -rf /tmp/caosdb_identifiable_cache.db
 set -e
 echo "Clearing database"
@@ -96,5 +96,8 @@ python3 -m pytest test_json_schema_datamodel_parser.py
 echo "Testing yaml datamodel parser"
 python3 -m pytest test_yaml_parser.py
 
+echo "Testing json-schema exporter"
+python3 -m pytest test_json_schema_exporter.py
+
 # Obsolete due to teardown in the above test.
 # echo "/n/n/n YOU NEED TO RESTART THE SERVER TO REDO TESTS!!!"

integrationtests/test_cache.py

@@ -63,6 +63,12 @@ class CacheTest(unittest.TestCase):
 
         update = UpdateCache(db_file=self.cache)
         run_id = "a"
+        print(db.execute_query("FIND Record TestRecord", unique=True))
+        print(db.execute_query("FIND entity with id="+str(rec.id), unique=True))
+        try:
+            print(db.execute_query("FIND Record "+str(rec.id), unique=True))
+        except BaseException:
+            print("Query does not work as expected")
         update.insert(cont, run_id)
         assert len(update.get_updates(run_id)) == 1
 

integrationtests/test_data_model.py

 import unittest
+import pytest
 
 import caosdb as db
 from caosadvancedtools.models.data_model import DataModel
@@ -55,6 +56,19 @@ class DataModelTest(unittest.TestCase):
         assert len(exist) == 1
         assert exist[0].name == "TestRecord"
 
+    def test_large_data_model(self):
+        # create RT and one property
+        dm = DataModel()
+        long = "Long" * 50
+        first_RT = db.RecordType(name=f"TestRecord_first")
+        for index in range(20):
+            this_RT = db.RecordType(name=f"TestRecord_{long}_{index:02d}")
+            first_RT.add_property(this_RT)
+            dm.append(this_RT)
+        dm.append(first_RT)
+        dm.sync_data_model(noquestion=True)  # Insert
+        dm.sync_data_model(noquestion=True)  # Check again
+
     def tearDown(self):
         try:
             tests = db.execute_query("FIND ENTITY test*")

integrationtests/test_json_schema_exporter.py

+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import linkahead as db
+
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema as rtjs
+
+
+def _delete_everything():
+    ents = db.execute_query("FIND ENTITY WITH ID > 99")
+    if ents:
+        ents.delete()
+
+
+def setup_module():
+    _delete_everything()
+
+
+def teardown_module():
+    _delete_everything()
+
+
+def test_uniqueness_of_reference_types():
+    parent_type = db.RecordType(name="ParentType").insert()
+    int_prop = db.Property(name="IntegerProp", datatype=db.INTEGER).insert()
+    sub_type = db.RecordType(name="SubType").add_parent(parent_type).add_property(
+        int_prop, importance=db.RECOMMENDED).insert()
+    referencing_type = db.RecordType(name="ReferencingType")
+    referencing_type.add_property(int_prop, importance=db.OBLIGATORY)
+    referencing_type.add_property(parent_type)
+    referencing_type.insert()
+    recA = db.Record(name="RecAParent").add_parent(parent_type).insert()
+    recB = db.Record(name="RecBSub").add_parent(sub_type).insert()
+
+    rt = db.execute_query(f"FIND RECORDTYPE WITH name='{referencing_type.name}'", unique=True)
+
+    schema = rtjs(rt)
+    assert schema["title"] == referencing_type.name
+    assert schema["type"] == "object"
+    assert len(schema["required"]) == 1
+    assert "IntegerProp" in schema["required"]
+    assert "IntegerProp" in schema["properties"]
+    assert schema["properties"]["IntegerProp"]["type"] == "integer"
+    assert parent_type.name in schema["properties"]
+    assert "oneOf" in schema["properties"][parent_type.name]
+    one_of = schema["properties"][parent_type.name]["oneOf"]
+    assert len(one_of) == 2
+    enum_index = 0
+    if "enum" not in one_of[enum_index]:
+        # As in unittests, we can't rely on the order of oneOf.
+        enum_index = 1 - enum_index
+    assert "enum" in one_of[enum_index]
+    assert len(one_of[enum_index]["enum"]) == 2
+    assert recA.name in one_of[enum_index]["enum"]
+    assert recB.name in one_of[enum_index]["enum"]
+    assert one_of[1 - enum_index]["type"] == "object"
+    # No properties in parent_type
+    assert len(one_of[1 - enum_index]["properties"]) == 0

manual_tests/test_labfolder_import.py

@@ -32,7 +32,7 @@ from caosadvancedtools.converter import labfolder_export as labfolder
 
 def main(args):
     """The main function."""
-    model = parse_model_from_yaml("./model.yml")
+    model = parse_model_from_yaml("./models/model.yml")
 
     model.sync_data_model()
     labfolder.import_data(args.folder)

manual_tests/test_labfolder_retrieve.py

@@ -31,7 +31,7 @@ from caosadvancedtools.converter.labfolder_api import Importer
 
 def main(args):
     """The main function."""
-    model = parse_model_from_yaml("./model.yml")
+    model = parse_model_from_yaml("./models/model.yml")
 
     # model.sync_data_model()
     importer = Importer()

release.sh

 #!/bin/bash
 rm -rf dist/ build/ .eggs/
 python setup.py sdist bdist_wheel
-python -m twine upload -s dist/*
+python -m twine upload dist/*

setup.py

@@ -46,7 +46,7 @@ from setuptools import find_packages, setup
 ########################################################################
 
 MAJOR = 0
-MINOR = 8
+MINOR = 9
 MICRO = 0
 PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
 ISRELEASED = True
@@ -155,9 +155,9 @@ def setup_package():
         author='Henrik tom Wörden',
         author_email='h.tomwoerden@indiscale.com',
         python_requires='>=3.7',
-        install_requires=["caosdb>=0.11.0",
+        install_requires=["linkahead>=0.13.1",
                           "jsonref",
-                          "jsonschema>=4.4.0",
+                          "jsonschema[format]>=4.4.0",
                           "numpy>=1.17.3",
                           "openpyxl>=3.0.7",
                           "pandas>=1.2.0",

src/caosadvancedtools/bloxberg/swagger_client/__init__.py

@@ -13,22 +13,23 @@
 """
 
 from __future__ import absolute_import
+from swagger_client.models.validation_error import ValidationError
+from swagger_client.models.http_validation_error import HTTPValidationError
+from swagger_client.models.controller_cert_tools_generate_unsigned_certificate_json_certificate import ControllerCertToolsGenerateUnsignedCertificateJsonCertificate
+from swagger_client.models.controller_cert_tools_generate_pdf_json_certificate import ControllerCertToolsGeneratePdfJsonCertificate
+from swagger_client.models.batch import Batch
+from swagger_client.configuration import Configuration
+from swagger_client.api_client import ApiClient
+from swagger_client.api.pdf_api import PdfApi
+from swagger_client.api.certificate_api import CertificateApi
 
 # Fake the installation
-import sys, pathlib
+import sys
+import pathlib
 __this_dir = str(pathlib.Path(__file__).parent.parent)
 if __this_dir not in sys.path:
     sys.path.append(__this_dir)
 
 # import apis into sdk package
-from swagger_client.api.certificate_api import CertificateApi
-from swagger_client.api.pdf_api import PdfApi
 # import ApiClient
-from swagger_client.api_client import ApiClient
-from swagger_client.configuration import Configuration
 # import models into sdk package
-from swagger_client.models.batch import Batch
-from swagger_client.models.controller_cert_tools_generate_pdf_json_certificate import ControllerCertToolsGeneratePdfJsonCertificate
-from swagger_client.models.controller_cert_tools_generate_unsigned_certificate_json_certificate import ControllerCertToolsGenerateUnsignedCertificateJsonCertificate
-from swagger_client.models.http_validation_error import HTTPValidationError
-from swagger_client.models.validation_error import ValidationError

src/caosadvancedtools/bloxberg/swagger_client/models/batch.py

@@ -15,6 +15,7 @@ import re  # noqa: F401
 
 import six
 
+
 class Batch(object):
     """NOTE: This class is auto generated by the swagger code generator program.
 

src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py

@@ -15,6 +15,7 @@ import re  # noqa: F401
 
 import six
 
+
 class ControllerCertToolsGeneratePdfJsonCertificate(object):
     """NOTE: This class is auto generated by the swagger code generator program.
 

src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py

@@ -15,6 +15,7 @@ import re  # noqa: F401
 
 import six
 
+
 class ControllerCertToolsGenerateUnsignedCertificateJsonCertificate(object):
     """NOTE: This class is auto generated by the swagger code generator program.