BUILD: Release 0.14.0

Merge branch 'release-0.14.0' into 'main'

Release 0.14.0

See merge request !138

.docker/Dockerfile

@@ -10,7 +10,8 @@ RUN apt-get update && \
 
 
 COPY .docker/wait-for-it.sh /wait-for-it.sh
-ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
+# See https://gitlab.indiscale.com/caosdb/src/caosdb-pylib for project ID (=97)
+ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/dev \
   pylib_version.json
 RUN pip install --break-system-packages -U git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
 COPY . /git

.docker/docker-compose.yml

@@ -27,6 +27,8 @@ services:
       - "10080:10080"
     environment:
       DEBUG: 1
+      CAOSDB_CONFIG__CAOSDB_INTEGRATION_TEST_SUITE_KEY: "_CAOSDB_ADV_TEST_SUITE"
+      CAOSDB_CONFIG_TIMEZONE: "Cuba"
 
 networks:
   caosnet:

.gitlab-ci.yml

@@ -162,6 +162,28 @@ unittest_py313:
   image: python:3.13
   script: *python_test_script
 
+unittest_py314:
+  allow_failure: true   # remove on release
+  tags: [docker]
+  stage: unittest
+  image: python:3.14-rc
+  script:               # replace by '*python_test_script' on release
+    # Install cargo manually, source its env, and set it to accept 3.14 as interpreter
+    - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+    - . "$HOME/.cargo/env"
+    - export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
+    # Install hdf5 dependencies
+    - apt-get update -y
+    - apt-get install -y pkg-config libhdf5-dev
+    # Install packages
+    - pip install --break-system-packages git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
+    - pip install --break-system-packages .[all]
+    # Force pytest to current version as 6.x is incompatible with 3.14
+    - pip install --upgrade pip
+    - pip install pytest==8.3.4
+    # Run tests
+    - pytest --cov=caosadvancedtools unittests
+
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
 pages_prepare: &pages_prepare

CHANGELOG.md

 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.0] - 2025-06-02 ##
+
+### Added ###
+
+- Added `table_json_conversion.export_import_xlsx` with a public function
+  `export_container_to_xlsx`, which exports the data of a given Entity
+  Container to a XLSX file.
+- Added parameters to the JsonSchemaExporter to
+  - add `id` property to all RecordTypes
+  - guess when a reference property probably should be treated like an enum.
+- Added `table_json_conversion.convert.records_from_json()` to create a Records Container from the
+  JSON which was extracted from an XLSX file.
+- Add `tools/test` module with utility functions which are useful for testing.
+
+### Fixed ###
+
+- Validation against schema in table json converter fixed.
+
 ## [0.13.0] - 2025-01-16 ##
 
 ### Added ###

CITATION.cff

@@ -19,7 +19,7 @@ authors:
   - family-names: Luther
     given-names: Stefan
     orcid: https://orcid.org/0000-0001-7214-8125
-title: CaosDB - Advanced User Tools
-version: 0.13.0
+title: LinkAhead - Advanced User Tools
+version: 0.14.0
 doi: 10.3390/data4020083
-date-released: 2025-01-16
+date-released: 2025-06-02
\ No newline at end of file

README_SETUP.md

@@ -33,16 +33,11 @@ entries `install_requires` and `extras_require`.
 
 ## Run Integration Tests Locally
 
-1. Change directory to `integrationtests/`.
-2. Mount `extroot` to the folder that will be used as extroot. E.g. `sudo mount
-   -o bind extroot ../../caosdb-deploy/profiles/debug/paths/extroot` (or
-   whatever path the extroot of the empty profile to be used is located at).
-3. Start (or restart) an empty (!) CaosDB instance (with the mounted
-   extroot). The database will be cleared during testing, so it's important to
-   use an empty instance. Make sure your configuration for the python caosdb
-   module is correct and allows to connect to the server.
-4. Run `test.sh`. Note that this may modify content of the
+1. Start LinkAhead using the profile in `integrationtests/test_profile/profile.yaml`
+2. Change directory to `integrationtests/`.
+3. Run `test.sh`. Note that this may modify content of the
    `integrationtest/extroot/` directory.
+4. Alternatively, run single tests: `pyest integrationtest/test_foo.py`
 
 ## Code Formatting
 

integrationtests/data/multiple_refs_data.json

+{
+  "Training": [
+    {
+      "name": "Example training with multiple organizations.",
+      "participant": [
+        {
+          "name": null,
+          "full_name": "Petra Participant",
+          "email": "petra@indiscale.com"
+        },
+        {
+          "name": null,
+          "full_name": "Peter",
+          "email": "peter@getlinkahead.com"
+        }
+      ],
+      "date": "2024-03-21 14:12:00+00:00",
+      "url": "www.indiscale.com",
+      "Organisation": [
+        {
+          "name": "World Training Organization",
+          "Country": "US",
+          "Person": [
+            {
+              "name": null,
+              "full_name": "Henry Henderson",
+              "email": "henry@organization.org"
+            },
+            {
+              "name": null,
+              "full_name": "Harry Hamburg",
+              "email": "harry@organization.org"
+            }
+          ]
+        },
+        {
+          "name": "European Training Organisation",
+          "Country": "UK",
+          "Person": [
+            {
+              "name": null,
+              "full_name": "Hermione Harvard",
+              "email": "hermione@organisation.org.uk"
+            },
+            {
+              "name": null,
+              "full_name": "Hazel Harper",
+              "email": "hazel@organisation.org.uk"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

integrationtests/test.sh

@@ -71,8 +71,8 @@ cd ..
 echo "Done."
 python3 test_table.py
 # TODO the following test deletes lots of the data inserted by the crawler
-echo "Testing im and export"
-python3 test_im_und_export.py
+echo "Testing export_related and import_from_xml"
+python3 test_export_related_import_xml.py
 
 # automated analysis
 # for some reason the loadFiles of sim data has to be called again
@@ -99,5 +99,8 @@ python3 -m pytest test_yaml_parser.py
 echo "Testing json-schema exporter"
 python3 -m pytest test_json_schema_exporter.py
 
+echo "Testing XLSX export/import"
+python3 -m pytest test_xlsx_export_import_xlsx.py
+
 # Obsolete due to teardown in the above test.
 # echo "/n/n/n YOU NEED TO RESTART THE SERVER TO REDO TESTS!!!"

integrationtests/test_profile/paths/extroot/README.md

+This directory is mounted into the LinkAhead docker container when the debug
+profile is used, to allow the inclusion of external file systems.

integrationtests/test_profile/profile.yaml

+default:
+  paths:
+    extroot:
+      "": "../extroot"
+
+  refs:
+    # SERVER: dev
+    # PYLIB: dev
+    # MYSQLBACKEND: dev
+    # WEBUI: dev
+    # ADVANCEDUSERTOOLS: dev
+
+  # General configuration options
+  conf:
+    restore: true
+    debug: true
+    timezone: "Cuba"
+
+    network:
+
+    server:
+      conf:
+        _CAOSDB_INTEGRATION_TEST_SUITE_KEY: "_CAOSDB_ADV_TEST_SUITE"
+        TRANSACTION_BENCHMARK_ENABLED: "TRUE"
+        # SERVER_SIDE_SCRIPTING_BIN_DIRS: ""
+
+  # Development configuration options
+  # devel:
+    # jar: /var/build/caosdb-server/0123abcd/target/caosdb-server-<version>-jar-with-dependencies.jar

setup.py

@@ -46,7 +46,7 @@ from setuptools import find_packages, setup
 ########################################################################
 
 MAJOR = 0
-MINOR = 13
+MINOR = 14
 MICRO = 0
 PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
 ISRELEASED = True
@@ -155,7 +155,7 @@ def setup_package():
         author='Henrik tom Wörden',
         author_email='h.tomwoerden@indiscale.com',
         python_requires='>=3.9',
-        install_requires=["linkahead>=0.13.1",
+        install_requires=["linkahead>0.17.0",
                           "jsonref",
                           "jsonschema[format]>=4.4.0",
                           "numpy>=1.24.0, < 2",

src/caosadvancedtools/table_json_conversion/_validation_utils.py

+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2025 Indiscale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Utilities for validation of conversion / import / export results.
+For internal use.
+"""
+
+import datetime
+import json
+from copy import deepcopy
+from typing import Union
+
+import jsonschema
+
+
+def validate_jsonschema(instance: Union[dict, int, str, bool],
+                        schema: Union[str, dict]):
+    """
+    A table_json_conversion compatible variant of jsonschema.validate().
+    Accepts instances with datetime instances and None in not-nullable entries.
+
+    Parameters
+    ----------
+    instance : dict, int, str, bool
+        Either a dict or a json entry to check against the given schema.
+    schema : str, dict
+        Either a dict with the jsonschema to check against, or a path to a file
+        containing the same.
+    """
+    # Helper Functions
+    def _in_schema(key, val, schema):
+        """
+        Checks whether a key: value pair is in the given schema or fulfills the
+        criteria of a direct subschema (anyOf, allOf, oneOf).
+        """
+        if schema.get(key, None) == val:
+            return True
+        if 'anyOf' in schema:
+            return any([_in_schema(key, val, sub) for sub in schema['anyOf']])
+        if 'allOf' in schema:
+            return all([_in_schema(key, val, sub) for sub in schema['allOf']])
+        if 'oneOf' in schema:
+            return [_in_schema(key, val, sub) for sub in schema['oneOf']].count(True) == 1
+        return False
+
+    def _remove_incompatible_vals(iterable, schema):
+        """
+        Removes Key: None and datetime instances from nested dicts and lists of
+        any depth. Key: None is currently valid as there is no 'obligatory with
+        value', and datetime cannot be checked by jsonschema.
+        """
+        if isinstance(iterable, list):
+            schema = schema.get('items', schema)
+            for elem in iterable:
+                _remove_incompatible_vals(elem, schema)
+        elif isinstance(iterable, dict):
+            schema = schema.get('properties', schema)
+            for key, elem in list(iterable.items()):
+                if elem is None:
+                    iterable.pop(key)
+                elif isinstance(elem, (datetime.date, datetime.datetime)):
+                    if (_in_schema('format', 'date', schema[key]) or
+                            _in_schema('format', 'date-time', schema[key])):
+                        iterable.pop(key)
+                elif isinstance(iterable, (dict, list)):
+                    try:
+                        _remove_incompatible_vals(elem, schema[key])
+                    except KeyError:
+                        pass
+        return iterable
+
+    # If jsonschema is a file, load its content
+    if isinstance(schema, str) and schema.endswith(".json"):
+        with open(schema, encoding="utf-8") as content:
+            schema = json.load(content)
+    assert isinstance(schema, dict)
+    # If instance is not a dict, remove_incompatible_values would not remove
+    # the value if it is valid, so we need to check manually by wrapping
+    instance = deepcopy(instance)
+    if not isinstance(instance, dict):
+        if _remove_incompatible_vals({'key': instance}, {'key': schema}) == {}:
+            return
+    # Clean dict and validate
+    instance = _remove_incompatible_vals(deepcopy(instance), schema)
+    jsonschema.validate(instance, schema=schema)