diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000000000000000000000000000000000000..1dc4f3d8c8d375ba3f7b352aa3e18702ec731d83
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[html]
+show_contexts = True
diff --git a/.gitignore b/.gitignore
index e2526574b37539d054397d49bbefcadcc9dce654..2b50c0fc33c80b9d83bc913c1e23836b51049f1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
src/caosadvancedtools/version.py
# compiled python and dist stuff
+.venv
*.egg
.eggs
*.egg-info/
@@ -17,3 +18,5 @@ build/
# documentation
_apidoc
+/dist/
+*~
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 68fb90fc027f1d706430358822b7aa8d5c4e2959..61c68e67a7107457e2f3c780b54366a23eae1e78 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ variables:
CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
-stages:
+stages:
- setup
- cert
- style
@@ -53,15 +53,15 @@ test:
- time docker load < /image-cache/mariadb.tar || true
- time docker load < /image-cache/caosdb-dev.tar || true
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
+ - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
-f .docker/docker-compose.yml up -d
- - cd .docker
+ - cd .docker
- /bin/sh ./run.sh
- - cd ..
+ - cd ..
- docker logs docker-caosdb-server-1 &> caosdb_log.txt
- docker logs docker-sqldb-1 &> mariadb_log.txt
- docker-compose -f .docker/docker-compose.yml down
- - rc=`cat .docker/result`
+ - rc=`cat .docker/result`
- exit $rc
dependencies: [cert]
needs: [cert]
@@ -76,19 +76,19 @@ build-testenv:
tags: [cached-dind]
image: docker:18.09
stage: setup
- # Hint: do not use only here; the image needs always to be build since it
+ # Hint: do not use only here; the image needs always to be build since it
# contains the repo code
#only:
- script:
+ script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- - docker build
+ - docker build
--file .docker/Dockerfile
-t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE
- docker save $CI_REGISTRY_IMAGE > /image-cache/caosdb-advanced-testenv.tar
- cd .docker-base
- - docker build
+ - docker build
-t $CI_REGISTRY_IMAGE_BASE .
- docker push $CI_REGISTRY_IMAGE_BASE
@@ -123,14 +123,57 @@ linting:
- make lint
allow_failure: true
-unittest:
+unittest_py39:
tags: [docker]
stage: unittest
image: $CI_REGISTRY_IMAGE
needs: [build-testenv]
script:
- - python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
- - tox
+ # First verify that system Python actually is 3.9
+ - python3 -c "import sys; assert sys.version.startswith('3.9')"
+ - python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
+ - tox
+
+unittest_py38:
+ tags: [docker]
+ stage: unittest
+ image: python:3.8
+ script: &python_test_script
+ - pip install pynose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
+ - pip install .
+ - pytest --cov=caosadvancedtools unittests
+
+unittest_py310:
+ tags: [docker]
+ stage: unittest
+ image: python:3.10
+ script: *python_test_script
+
+unittest_py311:
+ tags: [docker]
+ stage: unittest
+ image: python:3.11
+ script: *python_test_script
+
+unittest_py312:
+ tags: [docker]
+ stage: unittest
+ image: python:3.12
+ script: *python_test_script
+
+unittest_py313:
+ tags: [docker]
+ stage: unittest
+ image: python:3.13-rc
+ script:
+ # TODO: Replace by '*python_test_script' as soon as 3.13 has been officially released.
+ - apt update && apt install -y cargo || true
+ - pip install meson[ninja] meson-python || true
+ - pip install pynose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py || true
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev || true
+ - pip install . || true
+ - pytest --cov=caosadvancedtools unittests || true
# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md
new file mode 100644
index 0000000000000000000000000000000000000000..aa1a65aca363b87aff50280e1a86824009d2098b
--- /dev/null
+++ b/.gitlab/issue_templates/Default.md
@@ -0,0 +1,28 @@
+## Summary
+
+*Please give a short summary of what the issue is.*
+
+## Expected Behavior
+
+*What did you expect how the software should behave?*
+
+## Actual Behavior
+
+*What did the software actually do?*
+
+## Steps to Reproduce the Problem
+
+*Please describe, step by step, how others can reproduce the problem. Please try these steps for yourself on a clean system.*
+
+1.
+2.
+3.
+
+## Specifications
+
+- Version: *Which version of this software?*
+- Platform: *Which operating system, which other relevant software versions?*
+
+## Possible fixes
+
+*Do you have ideas how the issue can be resolved?*
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
new file mode 100644
index 0000000000000000000000000000000000000000..3629e0ca3695000863d8c254516f64bf59a7bf60
--- /dev/null
+++ b/.gitlab/merge_request_templates/Default.md
@@ -0,0 +1,56 @@
+# Summary
+
+*Insert a meaningful description for this merge request here: What is the new/changed behavior?
+Which bug has been fixed? Are there related issues?*
+
+
+# Focus
+
+*Point the reviewer to the core of the code change. Where should they start reading? What should
+they focus on (e.g. security, performance, maintainability, user-friendliness, compliance with the
+specs, finding more corner cases, concrete questions)?*
+
+
+# Test Environment
+
+*How to set up a test environment for manual testing?*
+
+
+# Check List for the Author
+
+Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab
+comments for the reviewer. They should guide the reviewer through the changes, explain your changes
+and also point out open questions. For further good practices have a look at [our review
+guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
+
+- [ ] All automated tests pass
+- [ ] Reference related issues
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Up-to-date JSON schema (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary)
+ - Update / write published documentation (`make doc`).
+ - How do I use the software? Assume "stupid" users.
+ - How do I develop or debug the software? Assume novice developers.
+- [ ] Annotations in code (Gitlab comments)
+ - Intent of new code
+ - Problems with old code
+ - Why this implementation?
+
+
+# Check List for the Reviewer
+
+- [ ] I understand the intent of this MR
+- [ ] All automated tests pass
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+ documentation.
+- [ ] The test environment setup works and the intended behavior is reproducible in the test
+ environment
+- [ ] In-code documentation and comments are up-to-date.
+- [ ] Check: Are there specifications? Are they satisfied?
+
+For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
+
+
+/assign me
+/target_branch dev
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f88538c8b1d4bff0109f68847430aeb53e7a01e9..6d90e03e5f79070ccbb1da12b5fed42c0b07a756 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,12 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
-- Unauthorized inserts can now be cached. Note that the Crawler cannot postpone
- inserts but the Cache has the functionality now.
-- caosdbignore; You can add one or more `.caosdbignore` files to the directory
- structure that you want to make available in CaosDB and the run loadFiles.
- The syntax is that of `.gitignore` files. For more information see `loadFiles`
- section of the Crawler in the documentation.
+- XLSX handling: conversion from XLSX to Json
### Changed ###
@@ -25,6 +20,111 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security ###
+### Documentation ###
+
+## [0.10.0] - 2024-04-24 ##
+
+### Added ###
+
+- XLSX handling: template generator and conversion from Json to XLSX
+- Json schema exporter:
+ - has new parameter `use_rt_pool`
+ - propagates more properties in the `make_array` function
+- Support for Python 3.12 and experimental support for 3.13
+
+### Changed ###
+
+* `table_converter.to_table` now returns an empty DataFrame instead of raising a
+ ValueError when called with an empty container.
+
+### Removed ###
+
+* The deprecated `parent` keyword from the YAML datamodel specification. Use
+ `inherit_from_{obligatory|recommended|suggested}` instead.
+* Support for Python 3.7
+
+### Fixed ###
+
+- Json schema exporter handles reference properties better.
+- [#59](https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/59)
+ `to_table` failed on lists as values.
+
+### Documentation ###
+
+* Added documentation for json-schema datamodel export
+
+## [0.9.0] - 2023-11-27 ##
+
+### Added ###
+
+* Added support for passing callables as `find_func` to the `BaseTableExporter`.
+* Added member `BaseTableExporter.all_keys`
+* Parsing from YAML now allows to give an existing model to which the YAML data model shall be
+ added.
+* The `json_schema_exporter` module which introduces tools to create a json
+ schema from a RecordType, e.g., for the usage in web forms.
+* `DataModel.get_deep(name: str)` method which uses the DataModel as a kind of cache pool.
+
+### Changed ###
+
+* A bit better error handling in the yaml model parser.
+* `TableImporter.check_datatypes` allows numeric values in string columns if
+ `strict=False` (default).
+
+### Fixed ###
+
+* `TableImporter.check_missing` in case of array-valued fields in table
+* YAML model parser has better description handling.
+
+### Documentation ###
+
+* Test coverage reports are now generated in `.tox/cov_html/` by tox.
+
+## [0.8.0] - 2023-05-30 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- TableImporter now accepts a `existing_columns` argument which demands that certain columns exist
+- The `JsonSchemaParser` class supports `patternProperties`
+- The `JsonSchemaParser` calss supports json-schema references (`$ref`)
+
+### Changed ###
+
+- The converters and datatype arguments of TableImporter now may have keys for nonexisting columns
+- The `JsonSchemaParser` class does not require the top-level entry of a json
+ schema definition to specify a RecordType.
+
+### Fixed ###
+
+- refactored to work with the new default key word in FIND queries: RECORD
+
+## [0.7.0] - 2023-03-09 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- `create_entity_link` function to create html links to entities; useful for
+ logging
+
+## [0.6.1] - 2023-01-20##
+
+### Added ###
+
+* Re-introduced support for Python 3.7
+
+## [0.6.0] - 2022-10-11 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- Unauthorized inserts can now be cached. Note that the Crawler cannot postpone
+ inserts but the Cache has the functionality now.
+- caosdbignore; You can add one or more `.caosdbignore` files to the directory
+ structure that you want to make available in CaosDB and the run loadFiles.
+ The syntax is that of `.gitignore` files. For more information see `loadFiles`
+ section of the Crawler in the documentation.
+
## [0.5.0] - 2022-09-05 ##
(Florian Spreckelsen)
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000000000000000000000000000000000000..6f7b69b665ada694ec5756fd0a6bb16c490c23a5
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+ - family-names: Fitschen
+ given-names: Timm
+ orcid: https://orcid.org/0000-0002-4022-432X
+ - family-names: Schlemmer
+ given-names: Alexander
+ orcid: https://orcid.org/0000-0003-4124-9649
+ - family-names: Hornung
+ given-names: Daniel
+ orcid: https://orcid.org/0000-0002-7846-6375
+ - family-names: tom Wörden
+ given-names: Henrik
+ orcid: https://orcid.org/0000-0002-5549-578X
+ - family-names: Parlitz
+ given-names: Ulrich
+ orcid: https://orcid.org/0000-0003-3058-1435
+ - family-names: Luther
+ given-names: Stefan
+ orcid: https://orcid.org/0000-0001-7214-8125
+title: CaosDB - Advanced User Tools
+version: 0.10.0
+doi: 10.3390/data4020083
+date-released: 2024-04-24
\ No newline at end of file
diff --git a/Makefile b/Makefile
index d9b182cbd0b17490e9d81b900d6ba8cefadb1b64..26f5c8182545b2a57b3921f3745d16ff6305a0cc 100644
--- a/Makefile
+++ b/Makefile
@@ -36,10 +36,10 @@ unittest:
pytest-3 unittests
style:
- pycodestyle --count src unittests --exclude=swagger_client
- autopep8 -ar --diff --exit-code --exclude swagger_client .
+ pycodestyle --count --exclude=swagger_client src unittests
+ autopep8 -ar --diff --exit-code --exclude swagger_client src unittests
.PHONY: style
lint:
- pylint --unsafe-load-any-extension=y -d all -e E,F --ignore=swagger_client src/caosadvancedtools
+ pylint --unsafe-load-any-extension=y --fail-under=9.72 -d R,C --ignore=swagger_client src/caosadvancedtools
.PHONY: lint
diff --git a/README.md b/README.md
index 83a767476286acba98d113b8fa7ab6b482751230..e652daf17b938fdd1226c192d514724dea1f2a62 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,7 @@
This is the **CaosDB Advanced User Tools** repository and a part of the
CaosDB project.
+
This project contains tools that are beyond the typical use of
the CaosDB python client. Especially, this includes the crawler which will
typically be used by a data curator.
@@ -43,7 +44,7 @@ Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
[documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/), the
preferred way is also a merge request as describe above (the documentation
resides in `src/doc`). However, you can also create an issue for it.
-- You can also contact us at **info (AT) caosdb.de** and join the CaosDB
+- You can also contact us at **info (AT) caosdb.org** and join the CaosDB
community on
[#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
@@ -51,7 +52,7 @@ Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
diff --git a/README_SETUP.md b/README_SETUP.md
index 43047d554afbe8ffba11aef67b20dde44d29bdcf..3a7f0197a4b06694c7ae787d0baa6e8a89de0e5e 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -11,7 +11,7 @@ git clone 'https://gitlab.com/caosdb/caosdb-advanced-user-tools'
Dependencies will be installed automatically if you use the below described
procedure.
- `caosdb>=0.6.0`
-- `openpyxl>=3.0.0`
+- `openpyxl>=3.0.7`
- `xlrd>=1.2.0`
- `pandas>=1.2.0`
- `numpy>=1.17.3`
@@ -32,7 +32,10 @@ Optional h5-crawler:
- `pip install .[h5-crawler] --user`
## Run Unit Tests
-`tox`
+
+- All tests: `tox`
+- One specific test with tox: `tox -- unittests/test_myusecase.py -k expression`
+- Or even using only pytest: `pytest unittests/test_myusecase.py -k expression`
## Run Integration Tests Locally
@@ -52,6 +55,8 @@ Optional h5-crawler:
`make style`
## Documentation #
+We use sphinx to create the documentation. Docstrings in the code should comply
+with the Googly style (see link below).
Build documentation in `build/` with `make doc`.
@@ -59,4 +64,11 @@ Build documentation in `build/` with `make doc`.
- `sphinx`
- `sphinx-autoapi`
+- `sphinx-rtd-theme`
- `recommonmark >= 0.6.0`
+
+### How to contribute ###
+
+- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
+- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index 8e4897e7e42c02e2c2620661c9f51e39630f6b50..adeab4ddf8a321bdf4e2794f07f18f5c0f4425b3 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -24,6 +24,7 @@ guidelines of the CaosDB Project
- `version` variables in `src/doc/conf.py`
- Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
`ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
+ - `CITATION.cff` (update version and date)
5. Merge the release branch into the main branch.
diff --git a/integrationtests/test.sh b/integrationtests/test.sh
index 36730cc948d308659f01f6153f86a917ab1909d0..a31afcfd2f74770b656eef41002b2f444b7962de 100755
--- a/integrationtests/test.sh
+++ b/integrationtests/test.sh
@@ -14,9 +14,9 @@ then
fi
fi
OUT=/tmp/crawler.output
-ls
+ls
cat pycaosdb.ini
-python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
+python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
rm -rf /tmp/caosdb_identifiable_cache.db
set -e
echo "Clearing database"
@@ -43,21 +43,23 @@ mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx \
DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back
cd ..
echo "run crawler"
-./crawl.py / | tee $OUT
+./crawl.py / | tee "$OUT"
# rename the moved file
mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back \
extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx
# check whether there was something UNAUTHORIZED
-grep "There where unauthorized changes" $OUT
+grep "There where unauthorized changes" "$OUT"
# get the id of the run which is the last field of the output string
-RUN_ID=$(grep "run id:" $OUT | awk '{ print $NF }')
+RUN_ID=$(grep "run id:" "$OUT" | awk '{ print $NF }')
echo $RUN_ID
echo "run crawler again"
echo "./crawl.py -a $RUN_ID /"
-./crawl.py -a $RUN_ID / | tee $OUT
+./crawl.py -a $RUN_ID / | tee "$OUT"
set +e
-if grep "There where unauthorized changes" $OUT
-then
+if grep "There where unauthorized changes" "$OUT"
+then
+ echo "There still were unauthorized changes, which should not have happend!"
+ echo "Test FAILED"
exit 1
fi
set -e
@@ -94,5 +96,8 @@ python3 -m pytest test_json_schema_datamodel_parser.py
echo "Testing yaml datamodel parser"
python3 -m pytest test_yaml_parser.py
+echo "Testing json-schema exporter"
+python3 -m pytest test_json_schema_exporter.py
+
# Obsolete due to teardown in the above test.
# echo "/n/n/n YOU NEED TO RESTART THE SERVER TO REDO TESTS!!!"
diff --git a/integrationtests/test_assure_functions.py b/integrationtests/test_assure_functions.py
index b1c731dbbf25f33b54fc3a005402f292525d2d05..e04d481f230936ae96b02fe910401f50e7138a01 100644
--- a/integrationtests/test_assure_functions.py
+++ b/integrationtests/test_assure_functions.py
@@ -33,7 +33,7 @@ from caosadvancedtools.guard import (global_guard, RETRIEVE, UPDATE)
def setup():
"""Delete all test entities."""
- db.execute_query("FIND Test*").delete(raise_exception_on_error=False)
+ db.execute_query("FIND ENTITY Test*").delete(raise_exception_on_error=False)
def setup_module():
@@ -105,13 +105,13 @@ def test_add_to_empty_list():
db.Record(name="TestReferencingRecord").add_parent(
referencing_rt).add_property(list_prop, value=[]).insert()
- referenced_rec = db.execute_query("FIND TestReferencedRecord", unique=True)
+ referenced_rec = db.execute_query("FIND ENTITY TestReferencedRecord", unique=True)
referencing_rec = db.execute_query(
- "FIND TestReferencingRecord", unique=True)
+ "FIND ENTITY TestReferencingRecord", unique=True)
assure_object_is_in_list(referenced_rec, referencing_rec, list_prop.name)
referencing_rec = db.execute_query(
- "FIND TestReferencingRecord", unique=True)
+ "FIND ENTITY TestReferencingRecord", unique=True)
assert referencing_rec.get_property(list_prop.name).value == [
referenced_rec.id]
diff --git a/integrationtests/test_base_table_exporter_integration.py b/integrationtests/test_base_table_exporter_integration.py
index 9d79e857fe706d78103ade3b92ee38498a2a1607..5af9caa3e83184f77c37c24073d85ee5aae2184b 100644
--- a/integrationtests/test_base_table_exporter_integration.py
+++ b/integrationtests/test_base_table_exporter_integration.py
@@ -81,7 +81,7 @@ def insert_entities():
def setup_module():
"""Clear all test entities"""
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND ENTITY Test*").delete()
except BaseException:
pass
@@ -146,7 +146,7 @@ def test_queries():
"Test_Property_2").value
# test guessing of selector
- del(export_dict["Test_Property_2"]["selector"])
+ del (export_dict["Test_Property_2"]["selector"])
my_exporter = te.BaseTableExporter(
export_dict=export_dict, record=rec1, raise_error_if_missing=True)
assert my_exporter.export_dict["Test_Property_2"]["selector"] == "Test_Property_2"
diff --git a/integrationtests/test_cache.py b/integrationtests/test_cache.py
index 4b0a6cedc390b1268e8d2d89393e19a27a83b3be..aacef1792e6028bf056093c517f45f6367f471d6 100644
--- a/integrationtests/test_cache.py
+++ b/integrationtests/test_cache.py
@@ -33,7 +33,7 @@ from caosadvancedtools.cache import UpdateCache
class CacheTest(unittest.TestCase):
def empty_db(self):
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND ENTITY Test*").delete()
except Exception:
pass
@@ -63,6 +63,12 @@ class CacheTest(unittest.TestCase):
update = UpdateCache(db_file=self.cache)
run_id = "a"
+ print(db.execute_query("FIND Record TestRecord", unique=True))
+ print(db.execute_query("FIND entity with id="+str(rec.id), unique=True))
+ try:
+ print(db.execute_query("FIND Record "+str(rec.id), unique=True))
+ except BaseException:
+ print("Query does not work as expected")
update.insert(cont, run_id)
assert len(update.get_updates(run_id)) == 1
diff --git a/integrationtests/test_crawl_with_datamodel_problems.py b/integrationtests/test_crawl_with_datamodel_problems.py
index 0c6a145afdab682f82af09a17fb9aa0770769959..8623d57d60ded38987953ffaf78b1d30e15a8011 100644
--- a/integrationtests/test_crawl_with_datamodel_problems.py
+++ b/integrationtests/test_crawl_with_datamodel_problems.py
@@ -74,7 +74,7 @@ def test_crawler_with_data_model_problems():
deleted_entities = {"Experiment", "Poster", "results"}
for ent in deleted_entities:
- db.execute_query("FIND "+ent).delete()
+ db.execute_query("FIND ENTITY "+ent).delete()
# Do the crawling
def access(x): return "extroot" + x
diff --git a/integrationtests/test_crawler_basics.py b/integrationtests/test_crawler_basics.py
index 7da90844f14cf0d1eaded9d4fc8f37320da46aad..60c09d73e954c39d752b5fa4ae5e272d28000ca1 100644
--- a/integrationtests/test_crawler_basics.py
+++ b/integrationtests/test_crawler_basics.py
@@ -40,7 +40,7 @@ def setup_module():
"""Clear all test entities. Allow insertions."""
guard.set_level(INSERT)
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND ENTITY Test*").delete()
except Exception:
pass
diff --git a/integrationtests/test_crawler_with_cfoods.py b/integrationtests/test_crawler_with_cfoods.py
index 19b1f8ff10e365031d6940c7b904e3656eda2861..1fa5eaa5a4f050d7282b863aae626982ff738c43 100755
--- a/integrationtests/test_crawler_with_cfoods.py
+++ b/integrationtests/test_crawler_with_cfoods.py
@@ -30,7 +30,7 @@ from caosdb.apiutils import retrieve_entity_with_id
def get_entity_with_id(eid):
- return db.execute_query("FIND "+str(eid), unique=True)
+ return db.execute_query("FIND ENTITY "+str(eid), unique=True)
class LoadFilesTest(unittest.TestCase):
@@ -49,7 +49,7 @@ class CrawlerTest(unittest.TestCase):
# # dummy for dependency test experiment # #
########################
exp = db.execute_query(
- "FIND Experiment with date=2019-02-04 and identifier=empty_identifier",
+ "FIND ENTITY Experiment with date=2019-02-04 and identifier=empty_identifier",
unique=True)
########################
@@ -59,7 +59,7 @@ class CrawlerTest(unittest.TestCase):
# vanishing of the property
# thus an x is used here. Needs to be fixed.
exp = db.execute_query(
- "FIND Experiment with date=2019-02-03 and identifier=empty_identifier",
+ "FIND ENTITY Experiment with date=2019-02-03 and identifier=empty_identifier",
unique=True)
# There should be a Project with name TestProject which is referenced
@@ -99,7 +99,7 @@ class CrawlerTest(unittest.TestCase):
# # second experiment # #
#########################
exp = db.execute_query(
- "FIND Experiment with date=2019-02-03 and identifier='something'",
+ "FIND ENTITY Experiment with date=2019-02-03 and identifier='something'",
unique=True)
# Should be the same project
@@ -120,7 +120,7 @@ class CrawlerTest(unittest.TestCase):
# # first analysis # #
######################
ana = db.execute_query(
- "FIND Analysis with date=2019-02-03 and identifier='empty_identifier'",
+ "FIND ENTITY Analysis with date=2019-02-03 and identifier='empty_identifier'",
unique=True)
# There should be a Project with name TestProject which is referenced
@@ -164,7 +164,7 @@ class CrawlerTest(unittest.TestCase):
# # second analysis # #
#######################
ana = db.execute_query(
- "FIND Analysis with date=2019-02-03 and identifier='something'",
+ "FIND ENTITY Analysis with date=2019-02-03 and identifier='something'",
unique=True)
# Should be the same project
@@ -197,7 +197,7 @@ class CrawlerTest(unittest.TestCase):
# # first simulation # #
######################
sim = db.execute_query(
- "FIND Simulation with date=2019-02-03 and identifier='empty_identifier'",
+ "FIND ENTITY Simulation with date=2019-02-03 and identifier='empty_identifier'",
unique=True)
# There should be a Project with name TestProject which is referenced
@@ -228,7 +228,7 @@ class CrawlerTest(unittest.TestCase):
# # second simulation # #
#########################
sim = db.execute_query(
- "FIND Simulation with date=2019-02-03 and identifier='something'",
+ "FIND ENTITY Simulation with date=2019-02-03 and identifier='something'",
unique=True)
sources = [get_entity_with_id(el) for el in
@@ -273,7 +273,7 @@ class CrawlerTest(unittest.TestCase):
#########################
# # first publication # #
#########################
- pub = db.execute_query("FIND *really_cool_finding", unique=True)
+ pub = db.execute_query("FIND ENTITY *really_cool_finding", unique=True)
# There should be a file as result attached with path poster.pdf
datfile_id = pub.get_property("results").value[0]
@@ -291,7 +291,7 @@ class CrawlerTest(unittest.TestCase):
##########################
# # second publication # #
##########################
- pub = db.execute_query("FIND *paper_on_exciting_stuff ", unique=True)
+ pub = db.execute_query("FIND ENTITY *paper_on_exciting_stuff ", unique=True)
# Test type
self.assertEqual(pub.parents[0].name, "Thesis")
@@ -311,10 +311,10 @@ class CrawlerTest(unittest.TestCase):
# # first software version # #
##############################
ana = db.execute_query(
- "FIND Software with version='V1.0-rc1'", unique=True)
+ "FIND ENTITY Software with version='V1.0-rc1'", unique=True)
sw = db.execute_query(
- "FIND Software with name='2010_TestSoftware'", unique=True)
+ "FIND ENTITY Software with name='2010_TestSoftware'", unique=True)
assert sw.get_property("alias").value == "TestSoftware"
# The software record should inherit from the correct software
@@ -360,10 +360,10 @@ class CrawlerTest(unittest.TestCase):
# # second software version # #
#######################
ana = db.execute_query(
- "FIND Software with version='v0.1'", unique=True)
+ "FIND ENTITY Software with version='v0.1'", unique=True)
sw = db.execute_query(
- "FIND Software with name='2010_TestSoftware'", unique=True)
+ "FIND ENTITY Software with name='2010_TestSoftware'", unique=True)
# The software record should inherit from the correct software
assert sw.id == ana.get_parents()[0].id
@@ -393,11 +393,11 @@ class CrawlerTest(unittest.TestCase):
# # third software version # #
#######################
ana = db.execute_query(
- "FIND Software with date='2020-02-04' and not version",
+ "FIND ENTITY Software with date='2020-02-04' and not version",
unique=True)
sw = db.execute_query(
- "FIND Software with name='2020NewProject0X'", unique=True)
+ "FIND ENTITY Software with name='2020NewProject0X'", unique=True)
# The software record should inherit from the correct software
assert sw.id == ana.get_parents()[0].id
@@ -438,11 +438,11 @@ class CrawlerTest(unittest.TestCase):
# # fourth software version # #
#######################
ana = db.execute_query(
- "FIND Software with date='2020-02-03' and not version",
+ "FIND ENTITY Software with date='2020-02-03' and not version",
unique=True)
sw = db.execute_query(
- "FIND Software with name='2020NewProject0X'", unique=True)
+ "FIND ENTITY Software with name='2020NewProject0X'", unique=True)
assert sw.get_property("alias").value == "NewProject0X"
# The software record should inherit from the correct software
@@ -479,10 +479,10 @@ class CrawlerTest(unittest.TestCase):
# # fifth software version # #
##############################
ana = db.execute_query(
- "FIND Software with version='second'", unique=True)
+ "FIND ENTITY Software with version='second'", unique=True)
sw = db.execute_query(
- "FIND Software with name='2020NewProject0X'", unique=True)
+ "FIND ENTITY Software with name='2020NewProject0X'", unique=True)
assert sw.get_property("alias").value == "NewProject0X"
# The software record should inherit from the correct software
diff --git a/integrationtests/test_data_model.py b/integrationtests/test_data_model.py
index 2949fa81727a6c61a8646a48c249204fa87542d8..bd74a40bde2540bb57245de1de464a1bfd84bc72 100644
--- a/integrationtests/test_data_model.py
+++ b/integrationtests/test_data_model.py
@@ -1,4 +1,5 @@
import unittest
+import pytest
import caosdb as db
from caosadvancedtools.models.data_model import DataModel
@@ -55,9 +56,22 @@ class DataModelTest(unittest.TestCase):
assert len(exist) == 1
assert exist[0].name == "TestRecord"
+ def test_large_data_model(self):
+ # create RT and one property
+ dm = DataModel()
+ long = "Long" * 50
+ first_RT = db.RecordType(name=f"TestRecord_first")
+ for index in range(20):
+ this_RT = db.RecordType(name=f"TestRecord_{long}_{index:02d}")
+ first_RT.add_property(this_RT)
+ dm.append(this_RT)
+ dm.append(first_RT)
+ dm.sync_data_model(noquestion=True) # Insert
+ dm.sync_data_model(noquestion=True) # Check again
+
def tearDown(self):
try:
- tests = db.execute_query("FIND test*")
+ tests = db.execute_query("FIND ENTITY test*")
tests.delete()
except Exception:
pass
diff --git a/integrationtests/test_datamodel_problems.py b/integrationtests/test_datamodel_problems.py
index 3bca302dd2a337cee7fd023ee6a64c5185bc99f5..855170338fbc81493e407fbe235415d60958c0f0 100644
--- a/integrationtests/test_datamodel_problems.py
+++ b/integrationtests/test_datamodel_problems.py
@@ -39,7 +39,7 @@ def setup_module():
"""Clear problem sets and delete possible test entities"""
DataModelProblems.missing.clear()
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND Entity Test*").delete()
except Exception as delete_exc:
print(delete_exc)
diff --git a/integrationtests/test_im_und_export.py b/integrationtests/test_im_und_export.py
index 8ea45fd2cebbcb2c3be6c8cb79805204486f7862..407faa1a1d3eb609ffd01b9c78d74f1c6a9b231b 100644
--- a/integrationtests/test_im_und_export.py
+++ b/integrationtests/test_im_und_export.py
@@ -8,13 +8,14 @@ from caosadvancedtools.import_from_xml import import_xml
if __name__ == "__main__":
print("Conducting im- and export tests")
- rec = db.execute_query("FIND 2019-02-03_really_cool_finding", unique=True)
+ rec = db.execute_query("FIND ENTITY 2019-02-03_really_cool_finding", unique=True)
directory = TemporaryDirectory()
export_related_to(rec.id, directory=directory.name)
# delete everything
print("Clearing database")
recs = db.execute_query("FIND entity with id>99")
- recs.delete()
+ if len(recs) > 0:
+ recs.delete()
assert 0 == len(db.execute_query("FIND File which is stored at "
"**/poster.pdf"))
print("Importing stored elements")
@@ -22,7 +23,7 @@ if __name__ == "__main__":
# The following tests the existence of some required entities.
# However, this is not a full list.
- db.execute_query("FIND 2019-02-03_really_cool_finding", unique=True)
+ db.execute_query("FIND ENTITY 2019-02-03_really_cool_finding", unique=True)
db.execute_query("FIND RecordType Poster", unique=True)
db.execute_query("FIND RecordType Analysis", unique=True)
db.execute_query("FIND RecordType Person", unique=True)
diff --git a/integrationtests/test_json_schema_exporter.py b/integrationtests/test_json_schema_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..44b428263ebbd9696fc2a171ea356d764482d5e3
--- /dev/null
+++ b/integrationtests/test_json_schema_exporter.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import json
+
+import linkahead as db
+
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema as rtjs
+from caosadvancedtools.models.parser import parse_model_from_string
+
+
+def _delete_everything():
+ ents = db.execute_query("FIND ENTITY WITH ID > 99")
+ if ents:
+ ents.delete()
+
+
+def setup_module():
+ _delete_everything()
+
+
+def teardown_module():
+ _delete_everything()
+
+
+def test_uniqueness_of_reference_types():
+ parent_type = db.RecordType(name="ParentType").insert()
+ int_prop = db.Property(name="IntegerProp", datatype=db.INTEGER).insert()
+ sub_type = db.RecordType(name="SubType").add_parent(parent_type).add_property(
+ int_prop, importance=db.RECOMMENDED).insert()
+ referencing_type = db.RecordType(name="ReferencingType")
+ referencing_type.add_property(int_prop, importance=db.OBLIGATORY)
+ referencing_type.add_property(parent_type)
+ referencing_type.insert()
+ recA = db.Record(name="RecAParent").add_parent(parent_type).insert()
+ recB = db.Record(name="RecBSub").add_parent(sub_type).insert()
+
+ rt = db.execute_query(f"FIND RECORDTYPE WITH name='{referencing_type.name}'", unique=True)
+
+ schema = rtjs(rt)
+ assert schema["title"] == referencing_type.name
+ assert schema["type"] == "object"
+ assert len(schema["required"]) == 1
+ assert "IntegerProp" in schema["required"]
+ assert "IntegerProp" in schema["properties"]
+ assert schema["properties"]["IntegerProp"]["type"] == "integer"
+ assert parent_type.name in schema["properties"]
+ assert "oneOf" in schema["properties"][parent_type.name]
+ one_of = schema["properties"][parent_type.name]["oneOf"]
+ assert len(one_of) == 2
+ enum_index = 0
+ if "enum" not in one_of[enum_index]:
+ # As in unittests, we can't rely on the order of oneOf.
+ enum_index = 1 - enum_index
+ assert "enum" in one_of[enum_index]
+ assert len(one_of[enum_index]["enum"]) == 2
+ assert recA.name in one_of[enum_index]["enum"]
+ assert recB.name in one_of[enum_index]["enum"]
+ assert one_of[1 - enum_index]["type"] == "object"
+ # No properties in parent_type
+ assert len(one_of[1 - enum_index]["properties"]) == 0
+
+
+def test_reference_property():
+ model_string = """
+RT1:
+ description: Some recordtype
+RT2:
+ obligatory_properties:
+ prop1:
+ description: Some reference property
+ datatype: RT1
+ """
+ model = parse_model_from_string(model_string)
+ model.sync_data_model(noquestion=True)
+ schema = rtjs(db.RecordType(name="RT2").retrieve())
+ assert json.dumps(schema, indent=2) == """{
+ "type": "object",
+ "required": [
+ "prop1"
+ ],
+ "additionalProperties": true,
+ "title": "RT2",
+ "properties": {
+ "prop1": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": true,
+ "description": "Some reference property",
+ "title": "prop1",
+ "properties": {}
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
diff --git a/integrationtests/test_table.py b/integrationtests/test_table.py
index 15b851fb5c81611d0faba93edfc58f46f9d75e79..b8dfe349f3dac3be9bb741f937f2be4f73b6b2af 100644
--- a/integrationtests/test_table.py
+++ b/integrationtests/test_table.py
@@ -34,14 +34,14 @@ if __name__ == "__main__":
table = pd.read_csv("example_table.csv")
- assert 0 == len(db.execute_query("FIND Person with firstname=Henrik"))
+ assert 0 == len(db.execute_query("FIND ENTITY Person with firstname=Henrik"))
first = table.loc[table.firstName == "Henrik"]
tcr = TableCrawler(table=first, unique_cols=["firstName", "lastName"],
recordtype="Person", interactive=False)
tcr.crawl(security_level=UPDATE)
- assert 1 == len(db.execute_query("FIND Person with firstname=Henrik"))
+ assert 1 == len(db.execute_query("FIND ENTITY Person with firstname=Henrik"))
tcr = TableCrawler(table=table, unique_cols=["firstName", "lastName"],
recordtype="Person", interactive=False)
tcr.crawl(security_level=UPDATE)
- assert 1 == len(db.execute_query("FIND Person with firstname=Henrik"))
- assert 1 == len(db.execute_query("FIND Person with firstname=Max"))
+ assert 1 == len(db.execute_query("FIND ENTITY Person with firstname=Henrik"))
+ assert 1 == len(db.execute_query("FIND ENTITY Person with firstname=Max"))
diff --git a/integrationtests/update_analysis.py b/integrationtests/update_analysis.py
index bd18ab375437bec02320dcfd269896c2ba7e2bb0..ddebc049f449026400278a26226d341d64e678c8 100644
--- a/integrationtests/update_analysis.py
+++ b/integrationtests/update_analysis.py
@@ -39,7 +39,7 @@ from caosadvancedtools.serverside.generic_analysis import run
def main():
- da = db.execute_query("FIND Analysis with identifier=TEST", unique=True)
+ da = db.execute_query("FIND ENTITY Analysis with identifier=TEST", unique=True)
run(da)
diff --git a/manual_tests/test_labfolder_import.py b/manual_tests/test_labfolder_import.py
index e1e9d3266478900b7fae02b3493fbc3d41ea2bd5..c767feb55cdf3958343d8d9780d01fa10c70f6ec 100644
--- a/manual_tests/test_labfolder_import.py
+++ b/manual_tests/test_labfolder_import.py
@@ -32,7 +32,7 @@ from caosadvancedtools.converter import labfolder_export as labfolder
def main(args):
"""The main function."""
- model = parse_model_from_yaml("./model.yml")
+ model = parse_model_from_yaml("./models/model.yml")
model.sync_data_model()
labfolder.import_data(args.folder)
diff --git a/manual_tests/test_labfolder_retrieve.py b/manual_tests/test_labfolder_retrieve.py
index 8c3f12d84a8990412d0d19cd6026a3452677f943..5bbaf91d0221a402e3a39246a129413adfa5f871 100644
--- a/manual_tests/test_labfolder_retrieve.py
+++ b/manual_tests/test_labfolder_retrieve.py
@@ -31,7 +31,7 @@ from caosadvancedtools.converter.labfolder_api import Importer
def main(args):
"""The main function."""
- model = parse_model_from_yaml("./model.yml")
+ model = parse_model_from_yaml("./models/model.yml")
# model.sync_data_model()
importer = Importer()
diff --git a/pylintrc b/pylintrc
index 625f83ce950841f7a239538123ef7b5812fc5c5f..d3a2e89ae1990480e5377daf443e0a63224342bc 100644
--- a/pylintrc
+++ b/pylintrc
@@ -1,5 +1,3 @@
-# -*- mode:conf; -*-
-
[FORMAT]
# Good variable names which should always be accepted, separated by a comma
good-names=ii,rt,df
@@ -17,3 +15,8 @@ init-hook=
import sys; sys.path.extend(["src/caosadvancedtools"]);
import astroid; astroid.context.InferenceContext.max_inferred = 500;
+[MESSAGES CONTROL]
+disable=
+ fixme,
+ logging-format-interpolation,
+ logging-not-lazy,
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index e65efaf9aaf061a8a1ec0040f87d682536fac4c2..0000000000000000000000000000000000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[pytest]
-testpaths = unittests
-addopts = -vv
diff --git a/release.sh b/release.sh
index 1af097f014de6cd9eb3d3e8ba5da34aea0fe1671..f6335ae20d0c29e760b508aac831a35460a59ef3 100755
--- a/release.sh
+++ b/release.sh
@@ -1,4 +1,4 @@
#!/bin/bash
rm -rf dist/ build/ .eggs/
python setup.py sdist bdist_wheel
-python -m twine upload -s dist/*
+python -m twine upload dist/*
diff --git a/setup.py b/setup.py
index 39d806c1dcd4abca6f6f2d756d7b9f1e41c4ba95..6f482155c25a19039b8afc4343d10186db321cd0 100755
--- a/setup.py
+++ b/setup.py
@@ -46,7 +46,7 @@ from setuptools import find_packages, setup
########################################################################
MAJOR = 0
-MINOR = 5
+MINOR = 10
MICRO = 1
PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = False
@@ -154,15 +154,17 @@ def setup_package():
long_description_content_type="text/markdown",
author='Henrik tom Wörden',
author_email='h.tomwoerden@indiscale.com',
- install_requires=["caosdb>=0.7.0",
- "jsonschema>=4.4.0",
- "numpy>=1.17.3",
- "openpyxl>=3.0.0",
+ python_requires='>=3.8',
+ install_requires=["linkahead>=0.13.1",
+ "jsonref",
+ "jsonschema[format]>=4.4.0",
+ "numpy>=1.24.0",
+ "openpyxl>=3.0.7",
"pandas>=1.2.0",
"xlrd>=2.0",
],
extras_require={"h5-crawler": ["h5py>=3.3.0", ],
- "gitignore-parser ": ["gitignore-parser >=0.1.0", ],
+ "gitignore-parser": ["gitignore-parser >=0.1.0", ],
},
packages=find_packages('src'),
package_dir={'': 'src'},
diff --git a/src/caosadvancedtools/bloxberg/bloxberg.py b/src/caosadvancedtools/bloxberg/bloxberg.py
index 42af1e11a23a37214ec294b8032517bb5c70bb5b..6aa2eaab94a1beb7735a09de6b22ab7745c0b078 100644
--- a/src/caosadvancedtools/bloxberg/bloxberg.py
+++ b/src/caosadvancedtools/bloxberg/bloxberg.py
@@ -145,6 +145,7 @@ filename : str
Write the JSON to this file.
"""
content = {}
+ print(certificate, filename)
return content
@@ -188,7 +189,6 @@ def demo_run():
print("Making sure that the remote data model is up to date.")
ensure_data_model()
print("Data model is up to date.")
- import caosdb as db
CertRT = db.RecordType(name="BloxbergCertificate").retrieve()
print("Certifying the `BloxbergCertificate` RecordType...")
json_filename = "/tmp/cert.json"
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/__init__.py b/src/caosadvancedtools/bloxberg/swagger_client/__init__.py
index 136c5b27a37cfbd9135230468ae5a29cb0eb2b77..255d6d3124dc352f10366e22f1eb8b461ff6593d 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/__init__.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/__init__.py
@@ -13,22 +13,23 @@
"""
from __future__ import absolute_import
+from swagger_client.models.validation_error import ValidationError
+from swagger_client.models.http_validation_error import HTTPValidationError
+from swagger_client.models.controller_cert_tools_generate_unsigned_certificate_json_certificate import ControllerCertToolsGenerateUnsignedCertificateJsonCertificate
+from swagger_client.models.controller_cert_tools_generate_pdf_json_certificate import ControllerCertToolsGeneratePdfJsonCertificate
+from swagger_client.models.batch import Batch
+from swagger_client.configuration import Configuration
+from swagger_client.api_client import ApiClient
+from swagger_client.api.pdf_api import PdfApi
+from swagger_client.api.certificate_api import CertificateApi
# Fake the installation
-import sys, pathlib
+import sys
+import pathlib
__this_dir = str(pathlib.Path(__file__).parent.parent)
if __this_dir not in sys.path:
sys.path.append(__this_dir)
# import apis into sdk package
-from swagger_client.api.certificate_api import CertificateApi
-from swagger_client.api.pdf_api import PdfApi
# import ApiClient
-from swagger_client.api_client import ApiClient
-from swagger_client.configuration import Configuration
# import models into sdk package
-from swagger_client.models.batch import Batch
-from swagger_client.models.controller_cert_tools_generate_pdf_json_certificate import ControllerCertToolsGeneratePdfJsonCertificate
-from swagger_client.models.controller_cert_tools_generate_unsigned_certificate_json_certificate import ControllerCertToolsGenerateUnsignedCertificateJsonCertificate
-from swagger_client.models.http_validation_error import HTTPValidationError
-from swagger_client.models.validation_error import ValidationError
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/api_client.py b/src/caosadvancedtools/bloxberg/swagger_client/api_client.py
index 25e6501a4e36b09bca266f2eb375807053a58870..7337ca334c545b2c2502a20cb5369db331149037 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/api_client.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/api_client.py
@@ -591,7 +591,7 @@ class ApiClient(object):
)
def __hasattr(self, object, name):
- return name in object.__class__.__dict__
+ return name in object.__class__.__dict__
def __deserialize_model(self, data, klass):
"""Deserializes list or dict to model.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py b/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py
index 7a347cf7ac9148df8ec9a43200f4058f127447b9..474ca01a69a6a06c93b7e9a640695fa709890997 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class Batch(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py
index 2d7fd2d763ba40c9a384203301aa3e70efdf7783..8c1b50d8816b09c1a466cf7d11cee1ca605dfd3a 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class ControllerCertToolsGeneratePdfJsonCertificate(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py
index 4a6d2d3f0e15faa8672f001e964d66c6e0a27780..fa0da3cb0c09e384cdddbd4ce458a4baf14f4b5d 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class ControllerCertToolsGenerateUnsignedCertificateJsonCertificate(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py b/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py
index 21c9e467311c596499f3f408c5ac670b5852c6fa..67c23fba87467a7888bff82fc7f11e9d90e15f15 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class HTTPValidationError(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py b/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py
index 7ae6bf0900449ff3612798a4503692c4e38e1c11..96d1e23734698efbdad8423c33012473e9aac03b 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class ValidationError(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/cache.py b/src/caosadvancedtools/cache.py
index db189b16e5755094ff0d6816aa0806b197b1e883..bf1287ba35d9c0af28d440db56e38173d38fcacf 100644
--- a/src/caosadvancedtools/cache.py
+++ b/src/caosadvancedtools/cache.py
@@ -27,16 +27,15 @@
# something to replace this.
import os
import sqlite3
-from copy import deepcopy
+import tempfile
+import warnings
from abc import ABC, abstractmethod
+from copy import deepcopy
from hashlib import sha256
-import warnings
import caosdb as db
from lxml import etree
-import tempfile
-
def put_in_container(stuff):
if isinstance(stuff, list):
@@ -151,13 +150,18 @@ class AbstractCache(ABC):
finally:
conn.close()
- def run_sql_commands(self, commands, fetchall=False):
- """
- Run a list of SQL commands on self.db_file.
+ def run_sql_commands(self, commands, fetchall: bool = False):
+ """Run a list of SQL commands on self.db_file.
+
+Parameters
+----------
+
+commands:
+ List of sql commands (tuples) to execute
- commands: list of sql commands (tuples) to execute
- fetchall: When True, run fetchall as last command and return the results.
- Otherwise nothing is returned.
+fetchall: bool, optional
+ When True, run fetchall as last command and return the results.
+ Otherwise nothing is returned.
"""
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
@@ -344,7 +348,7 @@ class UpdateCache(AbstractCache):
old_ones = db.Container()
for ent in cont:
- old_ones.append(db.execute_query("FIND {}".format(ent.id),
+ old_ones.append(db.execute_query("FIND ENTITY WITH ID={}".format(ent.id),
unique=True))
return old_ones
diff --git a/src/caosadvancedtools/cfood.py b/src/caosadvancedtools/cfood.py
index 4a9f955a17fc429deb6cdd10c3645700e579b4df..588476bd624f7e24a5e0f49380fb1e66f5bd1b17 100644
--- a/src/caosadvancedtools/cfood.py
+++ b/src/caosadvancedtools/cfood.py
@@ -27,12 +27,14 @@
CaosDB can automatically be filled with Records based on some structure, a file
structure, a table or similar.
+
The Crawler will iterate over the respective items and test for each item
whether a CFood class exists that matches the file path, i.e. whether CFood
class wants to treat that pariticular item. If one does, it is instanciated to
treat the match. This occurs in basically three steps:
+
1. Create a list of identifiables, i.e. unique representation of CaosDB Records
-(such as an experiment belonging to a project and a date/time).
+ (such as an experiment belonging to a project and a date/time).
2. The identifiables are either found in CaosDB or they are created.
3. The identifiables are update based on the date in the file structure.
"""
@@ -333,7 +335,7 @@ class AbstractFileCFood(AbstractCFood):
out : str
The regular expression, starting with ``.*\\.`` and ending with the EOL dollar
character. The actual extension will be accessible in the
- :py:attribute:`pattern group name<python:re.Pattern.groupindexe>` ``ext``.
+ :py:attr:`pattern group name <python:re.Pattern.groupindex>` ``ext``.
"""
if not extensions:
@@ -807,7 +809,7 @@ class RowCFood(AbstractCFood):
def update_identifiables(self):
rec = self.identifiables[0]
- for key, value in self.item.iteritems():
+ for key, value in self.item.items():
if key in self.unique_cols:
continue
assure_property_is(rec, key,
diff --git a/src/caosadvancedtools/cfoods/h5.py b/src/caosadvancedtools/cfoods/h5.py
index 4e6832f2e96e0950ed99146d4907f1ffb70d8494..dfd6f2905c955c1b4e493da1f2cbf45583ee68dc 100644
--- a/src/caosadvancedtools/cfoods/h5.py
+++ b/src/caosadvancedtools/cfoods/h5.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2020,2021 IndiScale GmbH <www.indiscale.com>
-# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2020-2025 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
# Copyright (C) 2021 Alexander Kreft
# Copyright (C) 2021 Laboratory for Fluid Physics and Biocomplexity,
@@ -22,8 +22,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-"""A CFood for hdf5 files
-
+"""A CFood for hdf5 files.
This module allows to parse hdf5 files and reproduce their structure in form
of Records that reference each other.
@@ -33,19 +32,14 @@ attributes. Groups and datasets are mapped to Records and attributes to
Properties.
"""
-import re
from copy import deepcopy
import caosdb as db
import h5py
import numpy as np
from caosadvancedtools.cfood import fileguide
-from caosdb.common.datatype import is_reference
-from caosdb.common.utils import uuid
-from ..cfood import (AbstractFileCFood, assure_has_description,
- assure_has_parent, assure_has_property,
- assure_property_is)
+from ..cfood import AbstractFileCFood
from ..structure_mapping import (EntityMapping, collect_existing_structure,
update_structure)
@@ -100,8 +94,7 @@ def h5_attr_to_property(val):
if hasattr(val, 'ndim'):
if not isinstance(val, np.ndarray) and val.ndim != 0:
print(val, val.ndim)
- raise Exception(
- "Implementation assumes that only np.arrays have ndim.")
+ raise RuntimeError("Implementation assumes that only np.arrays have ndim.")
return val, dtype
@@ -127,6 +120,8 @@ class H5CFood(AbstractFileCFood):
self.identifiable_root = None
self.root_name = "root"
self.hdf5Container = db.Container()
+ self.to_be_inserted = db.Container()
+ self.structure = db.Container()
self.em = EntityMapping()
def collect_information(self):
@@ -134,7 +129,7 @@ class H5CFood(AbstractFileCFood):
@staticmethod
def get_re():
- """Return a regular expression string to match *.h5, *.nc, *.hdf, *.hdf5."""
+ """Return a regular expression string to match ``*.h5``, ``*.nc``, ``*.hdf``, ``*.hdf5``."""
extensions = [
"h5",
"nc",
@@ -165,7 +160,8 @@ class H5CFood(AbstractFileCFood):
"""
- self.structure._cuid = "root element"
+ # TODO Why do we need a protected member here?
+ self.structure._cuid = "root element" # pylint: disable=protected-access
self.em.add(self.structure, self.identifiable_root)
collect_existing_structure(self.structure, self.identifiable_root,
self.em)
@@ -282,7 +278,7 @@ class H5CFood(AbstractFileCFood):
return rec
def insert_missing_structure(self, target_structure: db.Record):
- if target_structure._cuid not in self.em.to_existing:
+ if target_structure._cuid not in self.em.to_existing: # pylint: disable=protected-access
self.to_be_inserted.append(target_structure)
for prop in target_structure.get_properties():
diff --git a/src/caosadvancedtools/converter/labfolder_export.py b/src/caosadvancedtools/converter/labfolder_export.py
index 172ed993ecf0bc96d39a57f06082d7f83716ba19..6e282218b6d451749ad3070693d38299663f1bee 100644
--- a/src/caosadvancedtools/converter/labfolder_export.py
+++ b/src/caosadvancedtools/converter/labfolder_export.py
@@ -22,17 +22,7 @@
""" Imports labfolder exports """
import os
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-import time
-import warnings
-from io import BytesIO, StringIO
-
-import requests
-import yaml
+
from bs4 import BeautifulSoup
import caosdb as db
@@ -73,8 +63,8 @@ def get_author_from_entry(entry):
def val_or_none(stuff):
if len(stuff) == 0:
return None
- else:
- return stuff[0].getText()
+
+ return stuff[0].getText()
def add_property_from_data_element(dbrecord, element):
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 085cd8d27f261644b38061d26fb10e37ac5465fd..5e84bc8a60c1b358150c4db389efb62656af0631 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -58,6 +58,7 @@ from .guard import RETRIEVE, ProhibitedException
from .guard import global_guard as guard
from .serverside.helper import send_mail as main_send_mail
from .suppressKnown import SuppressKnown
+from .utils import create_entity_link
logger = logging.getLogger(__name__)
@@ -103,15 +104,8 @@ def apply_list_of_updates(to_be_updated, update_flags={},
info = "UPDATE: updating the following entities\n"
- baseurl = db.configuration.get_config()["Connection"]["url"]
-
- def make_clickable(txt, id):
- return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt)
-
for el in to_be_updated:
- info += str("\t" + make_clickable(el.name, el.id)
- if el.name is not None
- else "\t" + make_clickable(str(el.id), el.id))
+ info += str("\t" + create_entity_link(el))
info += "\n"
logger.info(info)
@@ -218,15 +212,14 @@ class Crawler(object):
new_cont.insert(unique=False)
logger.info("Successfully inserted {} records!".format(len(new_cont)))
all_inserts += len(new_cont)
- logger.info("Finished with authorized updates.")
+ logger.info("Finished with authorized inserts.")
changes = cache.get_updates(run_id)
for _, _, old, new, _ in changes:
- new_cont = db.Container()
- new_cont = new_cont.from_xml(new)
+ new_cont = db.Container.from_xml(new)
ids = []
- tmp = []
+ tmp = db.Container()
update_incomplete = False
# remove duplicate entities
for el in new_cont:
@@ -236,14 +229,14 @@ class Crawler(object):
else:
update_incomplete = True
new_cont = tmp
- if new[0].version:
+ if new_cont[0].version:
valids = db.Container()
nonvalids = db.Container()
for ent in new_cont:
remote_ent = db.Entity(id=ent.id).retrieve()
if ent.version == remote_ent.version:
- valids.append(remote_ent)
+ valids.append(ent)
else:
update_incomplete = True
nonvalids.append(remote_ent)
diff --git a/src/caosadvancedtools/export_related.py b/src/caosadvancedtools/export_related.py
index 69b588c34cc7c8123ab4291f6d8f76f06e7400be..7ae3a4dbba65faed551f75a1627eb504a3275f48 100755
--- a/src/caosadvancedtools/export_related.py
+++ b/src/caosadvancedtools/export_related.py
@@ -99,7 +99,7 @@ def invert_ids(entities):
def export_related_to(rec_id, directory="."):
if not isinstance(rec_id, int):
raise ValueError("rec_id needs to be an integer")
- ent = db.execute_query("FIND {}".format(rec_id), unique=True)
+ ent = db.execute_query("FIND ENTITY {}".format(rec_id), unique=True)
cont = recursively_collect_related(ent)
export(cont, directory=directory)
diff --git a/src/caosadvancedtools/json_schema_exporter.py b/src/caosadvancedtools/json_schema_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4c6de164aede0ce2e35b2f47fc30275552066b5
--- /dev/null
+++ b/src/caosadvancedtools/json_schema_exporter.py
@@ -0,0 +1,771 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"""Convert a data model into a json schema.
+
+Sometimes you may want to have a `json schema <https://json-schema.org>`_ which describes a
+LinkAhead data model, for example for the automatic generation of user interfaces with third-party
+tools like `rjsf <https://rjsf-team.github.io/react-jsonschema-form/docs/>`_. Then this is the
+right module for you!
+
+The :mod:`json_schema_exporter <caosadvancedtools.json_schema_exporter>` module has one main class,
+:class:`JsonSchemaExporter`, and a few utility and wrapper functions.
+
+For easy usage, you may simply import `recordtype_to_json_schema` and use it on a fully referenced
+RecordType like this::
+
+ import caosadvancedtools.models.parser as parser
+ import caosadvancedtools.json_schema_exporter as jsex
+
+ model = parser.parse_model_from_yaml("my_model.yml")
+
+ # get the data model schema for the "Journey" recordtype
+ schema, ui_schema = recordtype_to_json_schema(
+ rt=model.get_deep("Journey"),
+ do_not_create=["Continent"], # only choose from existing Records
+ multiple_choice=["visited_cities"],
+ rjsf=True # also create a UI schema
+ )
+
+For more details on how to use this wrapper, read the `function documentation
+<recordtype_to_json_schema>`.
+
+Other useful functions are `make_array`, which creates an array out of a single schema, and
+`merge_schemas`, which as the name suggests allows to combine multiple schema definitions into a
+single schema.
+
+"""
+
+from collections import OrderedDict
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+
+import linkahead as db
+from linkahead.cached import cached_query, cache_clear
+from linkahead.common.datatype import get_list_datatype, is_list_datatype
+from .models.data_model import DataModel
+
+
+class JsonSchemaExporter:
+ """A class which collects everything needed for the conversion.
+ """
+
+ def __init__(self, additional_properties: bool = True,
+ name_property_for_new_records: bool = False,
+ description_property_for_new_records: bool = False,
+ additional_options_for_text_props: dict = None,
+ additional_json_schema: Dict[str, dict] = None,
+ additional_ui_schema: Dict[str, dict] = None,
+ units_in_description: bool = True,
+ do_not_create: List[str] = None,
+ do_not_retrieve: List[str] = None,
+ no_remote: bool = False,
+ use_rt_pool: DataModel = None,
+ multiple_choice: List[str] = None,
+ wrap_files_in_objects: bool = False,
+ ):
+ """Set up a JsonSchemaExporter, which can then be applied on RecordTypes.
+
+ Parameters
+ ----------
+ additional_properties : bool, optional
+ Whether additional properties will be admitted in the resulting
+ schema. Optional, default is True.
+ name_property_for_new_records : bool, optional
+ Whether objects shall generally have a `name` property in the generated schema.
+ Optional, default is False.
+ description_property_for_new_records : bool, optional
+ Whether objects shall generally have a `description` property in the generated schema.
+ Optional, default is False.
+ additional_options_for_text_props : dict, optional
+ Dictionary containing additional "pattern" or "format" options for
+ string-typed properties. Optional, default is empty.
+ additional_json_schema : dict[str, dict], optional
+ Additional schema content for elements of the given names.
+ additional_ui_schema : dict[str, dict], optional
+ Additional ui schema content for elements of the given names.
+ units_in_description : bool, optional
+ Whether to add the unit of a LinkAhead property (if it has any) to the
+ description of the corresponding schema entry. If set to false, an
+ additional `unit` key is added to the schema itself which is purely
+ annotational and ignored, e.g., in validation. Default is True.
+ do_not_create : list[str], optional
+ A list of reference Property names, for which there should be no option
+ to create them. Instead, only the choice of existing elements should
+ be given.
+ do_not_retrieve : list[str], optional
+ A list of RedcordType names, for which no Records shall be retrieved. Instead, only an
+ object description should be given. If this list overlaps with the `do_not_create`
+ parameter, the behavior is undefined.
+ no_remote : bool, optional
+ If True, do not attempt to connect to a LinkAhead server at all. Default is False. Note
+ that the exporter may fail if this option is activated and the data model is not
+ self-sufficient.
+ use_rt_pool : models.data_model.DataModel, optional
+ If given, do not attempt to retrieve RecordType information remotely but from this parameter
+ instead.
+ multiple_choice : list[str], optional
+ A list of reference Property names which shall be denoted as multiple choice properties.
+ This means that each option in this property may be selected at most once. This is not
+ implemented yet if the Property is not in ``do_not_create`` as well.
+ wrap_files_in_objects : bool, optional
+ Whether (lists of) files should be wrapped into an array of objects
+ that have a file property. The sole purpose of this wrapping is to
+ provide a workaround for a `react-jsonschema-form
+ bug<https://github.com/rjsf-team/react-jsonschema-form/issues/3957>`_
+ so only set this to True if you're using the exported schema with
+ react-json-form and you are experiencing the bug. Default is False.
+ """
+ if not additional_options_for_text_props:
+ additional_options_for_text_props = {}
+ if not additional_json_schema:
+ additional_json_schema = {}
+ if not additional_ui_schema:
+ additional_ui_schema = {}
+ if not do_not_create:
+ do_not_create = []
+ if not do_not_retrieve:
+ do_not_retrieve = []
+ if not multiple_choice:
+ multiple_choice = []
+
+ cache_clear()
+
+ self._additional_properties = additional_properties
+ self._name_property_for_new_records = name_property_for_new_records
+ self._description_property_for_new_records = description_property_for_new_records
+ self._additional_options_for_text_props = additional_options_for_text_props
+ self._additional_json_schema = additional_json_schema
+ self._additional_ui_schema = additional_ui_schema
+ self._units_in_description = units_in_description
+ self._do_not_create = do_not_create
+ self._do_not_retrieve = do_not_retrieve
+ self._no_remote = no_remote
+ self._use_rt_pool = use_rt_pool
+ self._multiple_choice = multiple_choice
+ self._wrap_files_in_objects = wrap_files_in_objects
+
+ @staticmethod
+ def _make_required_list(rt: db.RecordType):
+ """Return the list of names of properties with importance db.OBLIGATORY."""
+ required_list = []
+ for prop in rt.properties:
+ if rt.get_importance(prop.name) != db.OBLIGATORY:
+ continue
+ prop_name = prop.name
+ required_list.append(prop_name)
+
+ return required_list
+
+ def _make_segment_from_prop(self, prop: db.Property) -> Tuple[OrderedDict, dict]:
+ """Return the JSON Schema and ui schema segments for the given property.
+
+The result may either be a simple json schema segment, such as a `string
+<https://json-schema.org/understanding-json-schema/reference/string>`_ element (or another
+simple type), a combination such as `anyOf
+<https://json-schema.org/understanding-json-schema/reference/combining#anyof>`_ or an `array
+<https://json-schema.org/understanding-json-schema/reference/array>`_ element
+
+Parameters
+----------
+prop : db.Property
+ The property to be transformed.
+
+Returns
+-------
+
+json_schema : OrderedDict
+ The Json schema.
+
+ui_schema : dict
+ An appropriate UI schema.
+ """
+ json_prop = OrderedDict()
+ ui_schema: dict = {}
+ if prop.datatype == db.TEXT or prop.datatype == db.DATETIME:
+ text_format = None
+ text_pattern = None
+ if prop.name in self._additional_options_for_text_props:
+ if "pattern" in self._additional_options_for_text_props[prop.name]:
+ text_pattern = self._additional_options_for_text_props[prop.name]["pattern"]
+ if "format" in self._additional_options_for_text_props[prop.name]:
+ text_format = self._additional_options_for_text_props[prop.name]["format"]
+ elif prop.datatype == db.DATETIME:
+ # Set the date or datetime format if only a pattern is given ...
+ text_format = ["date", "date-time"]
+ elif prop.datatype == db.DATETIME:
+ # ... again, for those props that don't appear in the additional
+ # options list.
+ text_format = ["date", "date-time"]
+
+ json_prop = self._make_text_property(prop.description, text_format, text_pattern)
+ return self._customize(json_prop, ui_schema, prop)
+
+ if prop.description:
+ json_prop["description"] = prop.description
+ if self._units_in_description and prop.unit:
+ if "description" in json_prop:
+ json_prop["description"] += f" Unit is {prop.unit}."
+ else:
+ json_prop["description"] = f"Unit is {prop.unit}."
+ elif prop.unit:
+ json_prop["unit"] = prop.unit
+
+ if prop.datatype == db.BOOLEAN:
+ json_prop["type"] = "boolean"
+ elif prop.datatype == db.INTEGER:
+ json_prop["type"] = "integer"
+ elif prop.datatype == db.DOUBLE:
+ json_prop["type"] = "number"
+ elif is_list_datatype(prop.datatype) and not (
+ self._wrap_files_in_objects and get_list_datatype(prop.datatype,
+ strict=True) == db.FILE):
+ json_prop["type"] = "array"
+ list_element_prop = db.Property(
+ name=prop.name, datatype=get_list_datatype(prop.datatype, strict=True))
+ json_prop["items"], inner_ui_schema = self._make_segment_from_prop(list_element_prop)
+ if "type" in json_prop["items"] and (
+ json_prop["items"]["type"] in ["boolean", "integer", "number", "string"]
+ ):
+ json_prop["items"]["type"] = [json_prop["items"]["type"], "null"]
+
+ if prop.name in self._multiple_choice and prop.name in self._do_not_create:
+ # TODO: if not multiple_choice, but do_not_create:
+ # "ui:widget" = "radio" & "ui:inline" = true
+ # TODO: set threshold for number of items.
+ json_prop["uniqueItems"] = True
+ ui_schema["ui:widget"] = "checkboxes"
+ ui_schema["ui:inline"] = True
+ if inner_ui_schema:
+ ui_schema["items"] = inner_ui_schema
+ elif prop.is_reference():
+ if prop.datatype == db.REFERENCE:
+ # No Record creation since no RT is specified and we don't know what
+ # schema to use, so only enum of all Records and all Files.
+ values = self._retrieve_enum_values("RECORD") + self._retrieve_enum_values("FILE")
+ json_prop["enum"] = values
+ if prop.name in self._multiple_choice:
+ json_prop["uniqueItems"] = True
+ elif prop.datatype == db.FILE or (
+ self._wrap_files_in_objects and
+ is_list_datatype(prop.datatype) and
+ get_list_datatype(prop.datatype, strict=True) == db.FILE
+ ):
+ # Singular FILE (wrapped or unwrapped), or wrapped LIST<FILE>
+ if self._wrap_files_in_objects:
+ # Workaround for react-jsonschema-form bug
+ # https://github.com/rjsf-team/react-jsonschema-form/issues/3957:
+ # Wrap all FILE references (regardless whether lists or
+ # scalars) in an array of objects that have a file property,
+ # since objects can be deleted, files can't.
+ json_prop["type"] = "array"
+ json_prop["items"] = {
+ "type": "object",
+ "title": "Next file",
+ # TODO Why can't it be empty?
+ # The wrapper object must wrap a file and can't be empty.
+ "required": [ # "file"
+ ],
+ # Wrapper objects must only contain the wrapped file.
+ "additionalProperties": False,
+ "properties": {
+ "file": {
+ "title": "Enter your file.",
+ "type": "string",
+ "format": "data-url"
+ }
+ }
+ }
+ if not is_list_datatype(prop.datatype):
+ # Scalar file, so the array has maximum length 1
+ json_prop["maxItems"] = 1
+ else:
+ json_prop["type"] = "string"
+ json_prop["format"] = "data-url"
+ else:
+ prop_name = prop.datatype
+ if isinstance(prop.datatype, db.Entity):
+ prop_name = prop.datatype.name
+ if prop.name in self._do_not_retrieve:
+ values = []
+ else:
+ values = self._retrieve_enum_values(f"RECORD '{prop_name}'")
+ if prop.name in self._do_not_create:
+ # Only a simple list of values
+ json_prop["enum"] = values
+ else:
+ if self._use_rt_pool:
+ rt = self._use_rt_pool.get_deep(prop_name)
+ elif self._no_remote:
+ rt = prop.datatype
+ else:
+ results = cached_query(f"FIND RECORDTYPE WITH name='{prop_name}'")
+ assert len(results) <= 1
+ if len(results):
+ rt = results[0]
+ else:
+ rt = db.Entity()
+ subschema, ui_schema = self._make_segment_from_recordtype(rt)
+ if prop.is_reference():
+ if prop.name:
+ subschema["title"] = prop.name
+ if prop.description:
+ subschema["description"] = prop.description
+
+ # if inner_ui_schema:
+ # ui_schema = inner_ui_schema
+ if values:
+ subschema["title"] = "Create new"
+ json_prop["oneOf"] = [
+ {
+ "title": "Existing entries",
+ "enum": values,
+ },
+ subschema
+ ]
+ else:
+ json_prop = subschema
+
+ else:
+ raise ValueError(
+ f"Unknown or no property datatype. Property {prop.name} with type {prop.datatype}")
+
+ return self._customize(json_prop, ui_schema, prop)
+
+ @staticmethod
+ def _make_text_property(description="", text_format=None, text_pattern=None) -> OrderedDict:
+ """Create a text element.
+
+ Can be a `string <https://json-schema.org/understanding-json-schema/reference/string>`_
+ element or an `anyOf
+ <https://json-schema.org/understanding-json-schema/reference/combining#anyof>`_ combination
+ thereof.
+
+ Example:
+
+ .. code-block:: json
+
+ {
+ "type": "string",
+ "description": "Some description",
+ "pattern": "[0-9]{2..4}-[0-9]{2-4}",
+ "format": "hostname",
+ }
+ """
+ prop: OrderedDict[str, Union[str, list]] = OrderedDict({
+ "type": "string"
+ })
+ if description:
+ prop["description"] = description
+ if text_format is not None:
+ if isinstance(text_format, list):
+ # We want the type inside the options, not in the head:
+ # "datetime property": {
+ # "anyOf": [
+ # {
+ # "type": "string",
+ # "format": "date"
+ # },
+ # {
+ # "type": "string",
+ # "format": "date-time"
+ # }]}
+ prop.pop("type")
+ prop["anyOf"] = [{"type": "string", "format": tf} for tf in text_format]
+ else:
+ prop["format"] = text_format
+ if text_pattern is not None:
+ prop["pattern"] = text_pattern
+
+ return prop
+
+ def _retrieve_enum_values(self, role: str):
+
+ if self._no_remote:
+ return []
+
+ possible_values = cached_query(f"SELECT name, id FROM {role}")
+
+ vals = []
+ for val in possible_values:
+ if val.name:
+ vals.append(f"{val.name}")
+ else:
+ vals.append(f"{val.id}")
+
+ return vals
+
+ def _make_segment_from_recordtype(self, rt: db.RecordType) -> Tuple[OrderedDict, dict]:
+ """Return Json schema and uischema segments for the given RecordType.
+
+ The result is an element of type `object
+ <https://json-schema.org/understanding-json-schema/reference/object>`_ and typically
+ contains more properties:
+
+ .. code-block:: json
+
+ {
+ "type": "object",
+ "title": "MyRecordtypeName",
+ "properties": {
+ "number": { "type": "number" },
+ "street_name": { "type": "string" },
+ "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+ }
+ }
+ """
+ schema: OrderedDict[str, Any] = OrderedDict({
+ "type": "object"
+ })
+ ui_schema = {}
+
+ schema["required"] = self._make_required_list(rt)
+ schema["additionalProperties"] = self._additional_properties
+ if rt.description:
+ schema["description"] = rt.description
+
+ if rt.name:
+ schema["title"] = rt.name
+
+ props = OrderedDict()
+ if self._name_property_for_new_records:
+ props["name"] = self._make_text_property("The name of the Record to be created")
+ if self._description_property_for_new_records:
+ props["description"] = self._make_text_property(
+ "The description of the Record to be created")
+
+ for prop in rt.properties:
+ if prop.name in props:
+ # Multi property
+ raise NotImplementedError(
+ "Creating a schema for multi-properties is not specified. "
+ f"Property {prop.name} occurs more than once."
+ )
+ props[prop.name], inner_ui_schema = self._make_segment_from_prop(prop)
+ if inner_ui_schema:
+ ui_schema[prop.name] = inner_ui_schema
+
+ schema["properties"] = props
+
+ return schema, ui_schema
+
+ def _customize(self, schema: OrderedDict, ui_schema: dict, entity: db.Entity = None) -> (
+ Tuple[OrderedDict, dict]):
+ """Generic customization method.
+
+Walk over the available customization stores and apply all applicable ones. No specific order is
+guaranteed (as of now).
+
+ Parameters
+ ----------
+ schema, ui_schema : dict
+ The input schemata.
+ entity: db.Entity : , optional
+ An Entity object, may be useful in the future for customizers.
+
+ Returns
+ -------
+ out : Tuple[dict, dict]
+ The modified input schemata.
+ """
+
+ name = schema.get("title", None)
+ if entity and entity.name:
+ name = entity.name
+ for key, add_schema in self._additional_json_schema.items():
+ if key == name:
+ schema.update(add_schema)
+ for key, add_schema in self._additional_ui_schema.items():
+ if key == name:
+ ui_schema.update(add_schema)
+
+ return schema, ui_schema
+
+ def recordtype_to_json_schema(self, rt: db.RecordType, rjsf: bool = False) -> Union[
+ dict, Tuple[dict, dict]]:
+ """Create a jsonschema from a given RecordType that can be used, e.g., to
+ validate a json specifying a record of the given type.
+
+ Parameters
+ ----------
+ rt : RecordType
+ The RecordType from which a json schema will be created.
+ rjsf : bool, optional
+ If True, uiSchema definitions for react-jsonschema-forms will be output as the second
+ return value. Default is False
+
+ Returns
+ -------
+ schema : dict
+ A dict containing the json schema created from the given RecordType's properties.
+
+ ui_schema : dict, optional
+ A ui schema. Only if a parameter asks for it (e.g. ``rjsf``).
+ """
+ if rt is None:
+ raise ValueError(
+ "recordtype_to_json_schema(...) cannot be called with a `None` RecordType.")
+ schema, inner_uischema = self._make_segment_from_recordtype(rt)
+ schema["$schema"] = "https://json-schema.org/draft/2020-12/schema"
+ if rt.description:
+ schema["description"] = rt.description
+ schema, inner_uischema = self._customize(schema, inner_uischema, rt)
+
+ if rjsf:
+ uischema = {}
+ if inner_uischema:
+ uischema = inner_uischema
+ return schema, uischema
+ return schema
+
+
+def recordtype_to_json_schema(rt: db.RecordType, additional_properties: bool = True,
+ name_property_for_new_records: bool = False,
+ description_property_for_new_records: bool = False,
+ additional_options_for_text_props: Optional[dict] = None,
+ additional_json_schema: Dict[str, dict] = None,
+ additional_ui_schema: Dict[str, dict] = None,
+ units_in_description: bool = True,
+ do_not_create: List[str] = None,
+ do_not_retrieve: List[str] = None,
+ no_remote: bool = False,
+ use_rt_pool: DataModel = None,
+ multiple_choice: List[str] = None,
+ rjsf: bool = False,
+ wrap_files_in_objects: bool = False
+ ) -> Union[dict, Tuple[dict, dict]]:
+ """Create a jsonschema from a given RecordType that can be used, e.g., to
+ validate a json specifying a record of the given type.
+
+ This is a standalone function which works without manually creating a
+ JsonSchemaExporter object.
+
+ Parameters
+ ----------
+ rt : RecordType
+ The RecordType from which a json schema will be created.
+ additional_properties : bool, optional
+ Whether additional properties will be admitted in the resulting
+ schema. Optional, default is True.
+ name_property_for_new_records : bool, optional
+ Whether objects shall generally have a `name` property in the generated schema. Optional,
+ default is False.
+ description_property_for_new_records : bool, optional
+ Whether objects shall generally have a `description` property in the generated schema.
+ Optional, default is False.
+ additional_options_for_text_props : dict, optional
+ Dictionary containing additional "pattern" or "format" options for
+ string-typed properties. Optional, default is empty.
+ additional_json_schema : dict[str, dict], optional
+ Additional schema content for elements of the given names.
+ additional_ui_schema : dict[str, dict], optional
+ Additional ui schema content for elements of the given names.
+ units_in_description : bool, optional
+ Whether to add the unit of a LinkAhead property (if it has any) to the
+ description of the corresponding schema entry. If set to false, an
+ additional `unit` key is added to the schema itself which is purely
+ annotational and ignored, e.g., in validation. Default is True.
+ do_not_create : list[str], optional
+ A list of reference Property names, for which there should be no option
+ to create them. Instead, only the choice of existing elements should
+ be given.
+ do_not_retrieve : list[str], optional
+ A list of RedcordType names, for which no Records shall be retrieved. Instead, only an
+ object description should be given. If this list overlaps with the `do_not_create`
+ parameter, the behavior is undefined.
+ no_remote : bool, optional
+ If True, do not attempt to connect to a LinkAhead server at all. Default is False.
+ use_rt_pool : models.data_model.DataModel, optional
+ If given, do not attempt to retrieve RecordType information remotely but from this parameter
+ instead.
+ multiple_choice : list[str], optional
+ A list of reference Property names which shall be denoted as multiple choice properties.
+ This means that each option in this property may be selected at most once. This is not
+ implemented yet if the Property is not in ``do_not_create`` as well.
+ rjsf : bool, optional
+ If True, uiSchema definitions for react-jsonschema-forms will be output as the second return
+ value. Default is False.
+ wrap_files_in_objects : bool, optional
+ Whether (lists of) files should be wrapped into an array of objects that
+ have a file property. The sole purpose of this wrapping is to provide a
+ workaround for a `react-jsonschema-form bug
+ <https://github.com/rjsf-team/react-jsonschema-form/issues/3957>`_ so
+ only set this to True if you're using the exported schema with
+ react-json-form and you are experiencing the bug. Default is False.
+
+
+ Returns
+ -------
+ schema : dict
+ A dict containing the json schema created from the given RecordType's properties.
+
+ ui_schema : dict, optional
+ A ui schema. Only if a parameter asks for it (e.g. ``rjsf``).
+ """
+
+ exporter = JsonSchemaExporter(
+ additional_properties=additional_properties,
+ name_property_for_new_records=name_property_for_new_records,
+ description_property_for_new_records=description_property_for_new_records,
+ additional_options_for_text_props=additional_options_for_text_props,
+ additional_json_schema=additional_json_schema,
+ additional_ui_schema=additional_ui_schema,
+ units_in_description=units_in_description,
+ do_not_create=do_not_create,
+ do_not_retrieve=do_not_retrieve,
+ no_remote=no_remote,
+ use_rt_pool=use_rt_pool,
+ multiple_choice=multiple_choice,
+ wrap_files_in_objects=wrap_files_in_objects
+ )
+ return exporter.recordtype_to_json_schema(rt, rjsf=rjsf)
+
+
+def make_array(schema: dict, rjsf_uischema: dict = None) -> Union[dict, Tuple[dict, dict]]:
+ """Create an array of the given schema.
+
+The result will look like this:
+
+.. code:: js
+
+ { "type": "array",
+ "items": {
+ // the schema
+ }
+ }
+
+Parameters
+----------
+
+schema : dict
+ The JSON schema which shall be packed into an array.
+
+rjsf_uischema : dict, optional
+ A react-jsonschema-forms ui schema that shall be wrapped as well.
+
+Returns
+-------
+
+schema : dict
+ A JSON schema dict with a top-level array which contains instances of the given schema.
+
+ui_schema : dict, optional
+ The wrapped ui schema. Only returned if ``rjsf_uischema`` is given as parameter.
+ """
+ result = {
+ "type": "array",
+ "items": schema,
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ }
+
+ if schema.get("description"):
+ result["description"] = schema["description"]
+
+ if rjsf_uischema is not None:
+ ui_schema = {"items": rjsf_uischema}
+ # Propagate ui: options up one level.
+ for key in rjsf_uischema.keys():
+ if key.startswith("ui:"):
+ ui_schema[key] = rjsf_uischema[key]
+
+ return result, ui_schema
+ return result
+
+
+def merge_schemas(schemas: Union[Dict[str, dict], Iterable[dict]],
+ rjsf_uischemas: Union[Dict[str, dict], Sequence[dict]] = None) -> (
+ Union[dict, Tuple[dict, dict]]):
+ """Merge the given schemata into a single schema.
+
+The result will look like this:
+
+.. code:: js
+
+ {
+ "type": "object",
+ "properties": {
+ // A, B, C
+ },
+ "required": [
+ // "A", "B", "C"
+ ],
+ "additionalProperties": false
+ }
+
+
+Parameters
+----------
+
+schemas : dict[str, dict] | Iterable[dict]
+ A dict or iterable of schemata which shall be merged together. If this is a dict, the keys will
+ be used as property names, otherwise the titles of the submitted schemata. If they have no title,
+ numbers will be used as a fallback. Note that even with a dict, the original schema's "title" is
+ not changed.
+rjsf_uischemas : dict[str, dict] | Iterable[dict], optional
+ If given, also merge the react-jsonschema-forms from this argument and return as the second return
+ value. If ``schemas`` is a dict, this parameter must also be a dict, if ``schemas`` is only an
+ iterable, this paramater must support numerical indexing.
+
+Returns
+-------
+
+schema : dict
+ A JSON schema dict with a top-level object which contains the given schemata as properties.
+
+uischema : dict
+ If ``rjsf_uischemas`` was given, this contains the merged UI schemata.
+ """
+ sub_schemas: dict[str, dict] = OrderedDict()
+ required = []
+ ui_schema = None
+
+ if isinstance(schemas, dict):
+ sub_schemas = schemas
+ required = [str(k) for k in schemas.keys()]
+ if rjsf_uischemas is not None:
+ if not isinstance(rjsf_uischemas, dict):
+ raise ValueError("Parameter `rjsf_uischemas` must be a dict, because `schemas` is "
+ f"as well, but it is a {type(rjsf_uischemas)}.")
+ ui_schema = {k: rjsf_uischemas[k] for k in schemas.keys()}
+ else:
+ for i, schema in enumerate(schemas, start=1):
+ title = schema.get("title", str(i))
+ sub_schemas[title] = schema
+ required.append(title)
+ if rjsf_uischemas is not None:
+ if not isinstance(rjsf_uischemas, Sequence):
+ raise ValueError("Parameter `rjsf_uischemas` must be a sequence, because `schemas` "
+ f"is as well, but it is a {type(rjsf_uischemas)}.")
+ ui_schema = {}
+ for i, title in enumerate(sub_schemas.keys()):
+ ui_schema[title] = rjsf_uischemas[i]
+ # ui_schema = {"index": ui_schema}
+
+ result = {
+ "type": "object",
+ "properties": sub_schemas,
+ "required": required,
+ "additionalProperties": False,
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ }
+
+ if ui_schema is not None:
+ return result, ui_schema
+ return result
diff --git a/src/caosadvancedtools/loadFiles.py b/src/caosadvancedtools/loadFiles.py
index 210732aa2bd31b7f493af263df24cc3e7a56611b..405b3d135c8af89e32c74015bd04f76f21828e20 100755
--- a/src/caosadvancedtools/loadFiles.py
+++ b/src/caosadvancedtools/loadFiles.py
@@ -175,10 +175,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi
for f in files:
totalsize += f.size
- print("Made in total {} new files with a combined size of {} "
- "accessible.".format(len(files), convert_size(totalsize)))
- logger.info("Made in total {} new files with a combined size of {} "
- "accessible.".format(len(files), convert_size(totalsize)))
+ logger.info(
+ f"Made new files accessible: {len(files)}, combined size: {convert_size(totalsize)} ")
return
@@ -249,6 +247,10 @@ exclude is given preference over include.
raise ValueError(
"Do not use a localpath and in- or exclude simultaneously!")
+ # configure logging
+ logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+ logger.setLevel(logging.INFO)
+
con = db.get_connection()
con.timeout = float(args.timeout)
con._login()
diff --git a/src/caosadvancedtools/models/data_model.py b/src/caosadvancedtools/models/data_model.py
index d9079e6196b4751ca86ba41275108330b946d57c..266414893bcdf1ab45ee1345fc549e15f4a66250 100644
--- a/src/caosadvancedtools/models/data_model.py
+++ b/src/caosadvancedtools/models/data_model.py
@@ -29,8 +29,9 @@ from copy import deepcopy
# remove this, when we drop support for old Python versions.
from typing import List
-import caosdb as db
-from caosdb.apiutils import compare_entities, describe_diff
+import linkahead as db
+import linkahead.common.models as models
+from linkahead.apiutils import compare_entities, describe_diff, merge_entities
CAOSDB_INTERNAL_PROPERTIES = [
@@ -60,7 +61,8 @@ class DataModel(dict):
different purpose (e.g. someone else's experiment).
DataModel inherits from dict. The keys are always the names of the
- entities. Thus you cannot have unnamed entities in your model.
+ entities. Thus you cannot have unnamed or ambiguously named entities in your
+ model.
Example:
@@ -141,7 +143,7 @@ class DataModel(dict):
# in via the extern keyword:
ref = db.Property(name=ent.name).retrieve()
else:
- query = db.Query(f"FIND * with id={ent.id}")
+ query = db.Query(f"FIND ENTITY with id={ent.id}")
ref = query.execute(unique=True)
diff = (describe_diff(*compare_entities(ent, ref
), name=ent.name))
@@ -261,3 +263,73 @@ class DataModel(dict):
all_ents[prop.name] = prop
return list(all_ents.values())
+
+ def get_deep(self, name: str, visited_props: dict = None, visited_parents: set = None):
+ """Attempt to resolve references for the given ``name``.
+
+ The returned entity has all the properties it inherits from its ancestry and all properties
+ have the correct descriptions and datatypes. This methods only uses data which is available
+ in this DataModel, which acts kind of like a cache pool.
+
+ Note that this may change this data model (subsequent "get" like calls may also return
+ deeper content.)
+
+ """
+ entity = self.get(name)
+ if not entity:
+ return entity
+ if not visited_props:
+ visited_props = {}
+ if not visited_parents:
+ visited_parents = set()
+
+ importances = {
+ models.OBLIGATORY: 0,
+ models.RECOMMENDED: 1,
+ models.SUGGESTED: 2,
+ }
+
+ for parent in list(entity.get_parents()): # Make a change-resistant list copy.
+ if parent.name in visited_parents:
+ continue
+ visited_parents.add(parent.name)
+ parent_importance = importances.get(parent._flags.get("inheritance"), 999)
+ if parent.name in self:
+ deep_parent = self.get_deep(parent.name, # visited_props=visited_props,
+ visited_parents=visited_parents
+ )
+
+ for prop in deep_parent.properties:
+ importance = importances[deep_parent.get_importance(prop.name)]
+ if (importance <= parent_importance
+ and prop.name not in [prop.name for prop in entity.properties]):
+ entity.add_property(prop)
+ else:
+ print(f"Referenced parent \"{parent.name}\" not found in data model.")
+
+ for prop in list(entity.get_properties()): # Make a change-resistant list copy.
+ if prop.name in visited_props:
+ if visited_props[prop.name]:
+ deep_prop = visited_props[prop.name]
+ merge_entities(prop, deep_prop)
+ prop.datatype = deep_prop.datatype
+ prop.value = deep_prop.value
+ prop.unit = deep_prop.unit
+ continue
+ visited_props[prop.name] = None
+ if prop.name in self:
+ deep_prop = self.get_deep(prop.name, visited_props=visited_props,
+ visited_parents=visited_parents)
+ linked_prop = entity.get_property(prop)
+ if not linked_prop.datatype:
+ if deep_prop.role == "Property":
+ linked_prop.datatype = deep_prop.datatype
+ elif deep_prop.role == "RecordType":
+ linked_prop.datatype = deep_prop
+ if deep_prop.description:
+ linked_prop.description = deep_prop.description
+ visited_props[prop.name] = deep_prop
+ else:
+ print(f"Referenced property \"{prop.name}\" not found in data model.")
+
+ return entity
diff --git a/src/caosadvancedtools/models/parser.py b/src/caosadvancedtools/models/parser.py
index c9b890de570d29e4a013b14ebe4579e956277ed2..f9bea92455e948eb40e337a43ad87b6d79156fce 100644
--- a/src/caosadvancedtools/models/parser.py
+++ b/src/caosadvancedtools/models/parser.py
@@ -1,8 +1,8 @@
# This file is a part of the CaosDB Project.
#
-# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2023 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -35,24 +35,24 @@ not defined, simply the name can be supplied with no value.
Parents can be provided under the 'inherit_from_xxxx' keywords. The value needs
to be a list with the names. Here, NO NEW entities can be defined.
"""
-import json
import argparse
+import json
+import jsonref
import re
import sys
import yaml
-from typing import List
+from typing import List, Optional
from warnings import warn
import jsonschema
-import caosdb as db
+import linkahead as db
+from linkahead.common.datatype import get_list_datatype
from .data_model import CAOSDB_INTERNAL_PROPERTIES, DataModel
# Keywords which are allowed in data model descriptions.
-KEYWORDS = ["parent", # deprecated, use inherit_from_* instead:
- # https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36
- "importance",
+KEYWORDS = ["importance",
"datatype", # for example TEXT, INTEGER or REFERENCE
"unit",
"description",
@@ -76,27 +76,11 @@ JSON_SCHEMA_ATOMIC_TYPES = [
"string",
"boolean",
"integer",
- "number"
+ "number",
+ "null"
]
-def _get_listdatatype(dtype):
- """matches a string to check whether the type definition is a list
-
- returns the type within the list or None, if it cannot be matched with a
- list definition
- """
- # TODO: string representation should be the same as used by the server:
- # e.g. LIST<TEXT>
- # this should be changed in the module and the old behavour should be
- # marked as depricated
- match = re.match(r"^LIST[(<](?P<dt>.*)[)>]$", dtype)
-
- if match is None:
- return None
- else:
- return match.group("dt")
-
# Taken from https://stackoverflow.com/a/53647080, CC-BY-SA, 2018 by
# https://stackoverflow.com/users/2572431/augurar
@@ -138,30 +122,82 @@ class JsonSchemaDefinitionError(RuntimeError):
super().__init__(msg)
-def parse_model_from_yaml(filename):
- """Shortcut if the Parser object is not needed."""
- parser = Parser()
+def parse_model_from_yaml(filename, existing_model: Optional[dict] = None, debug: bool = False):
+ """Parse a data model from a YAML file.
+
+This is a convenience function if the Parser object is not needed, it calls
+``Parser.parse_model_from_yaml(...)`` internally.
- return parser.parse_model_from_yaml(filename)
+Parameters
+----------
-def parse_model_from_string(string):
- """Shortcut if the Parser object is not needed."""
- parser = Parser()
+existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
+debug : bool, optional
+ If True, turn on miscellaneous debugging. Default is False.
+ """
+ parser = Parser(debug=debug)
+
+ return parser.parse_model_from_yaml(filename, existing_model=existing_model)
+
+
+def parse_model_from_string(string, existing_model: Optional[dict] = None, debug: bool = False):
+ """Parse a data model from a YAML string
+
+This is a convenience function if the Parser object is not needed, it calls
+``Parser.parse_model_from_string(...)`` internally.
+
+Parameters
+----------
+
+existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
+debug : bool, optional
+ If True, turn on miscellaneous debugging. Default is False.
+ """
+ parser = Parser(debug=debug)
- return parser.parse_model_from_string(string)
+ return parser.parse_model_from_string(string, existing_model=existing_model)
-def parse_model_from_json_schema(filename: str):
+def parse_model_from_json_schema(
+ filename: str,
+ top_level_recordtype: bool = True,
+ types_for_missing_array_items: dict = {},
+ ignore_unspecified_array_items: bool = False,
+ existing_model: Optional[dict] = None
+):
"""Return a datamodel parsed from a json schema definition.
Parameters
----------
+
filename : str
The path of the json schema file that is to be parsed
+ top_level_recordtype : bool, optional
+ Whether there is a record type defined at the top level of the
+ schema. Default is true.
+
+ types_for_missing_array_items : dict, optional
+ dictionary containing fall-back types for json entries with `type:
+ array` but without `items` specification. Default is an empty dict.
+
+ ignore_unspecified_array_items : bool, optional
+ Whether to ignore `type: array` entries the type of which is not
+ specified by their `items` property or given in
+ `types_for_missing_array_items`. An error is raised if they are not
+ ignored. Default is False.
+
+ existing_model : dict, optional
+ An existing model to which the created model shall be added. Not implemented yet.
+
Returns
-------
+
out : Datamodel
The datamodel generated from the input schema which then can be used for
synchronizing with CaosDB.
@@ -172,24 +208,34 @@ def parse_model_from_json_schema(filename: str):
about the limitations of the current implementation.
"""
+ if existing_model is not None:
+ raise NotImplementedError("Adding to an existing model is not implemented yet.")
+
# @author Florian Spreckelsen
# @date 2022-02-17
- # @review Daniel Hornung 2022-02-18
- parser = JsonSchemaParser()
+ # @review Timm Fitschen 2023-05-25
+ parser = JsonSchemaParser(types_for_missing_array_items, ignore_unspecified_array_items)
- return parser.parse_model_from_json_schema(filename)
+ return parser.parse_model_from_json_schema(filename, top_level_recordtype)
class Parser(object):
- def __init__(self):
+ def __init__(self, debug: bool = False):
"""Initialize an empty parser object and initialize the dictionary of entities and the list of
treated elements.
+Parameters
+----------
+
+debug : bool, optional
+ If True, turn on miscellaneous debugging. Default is False.
+
"""
self.model = {}
self.treated = []
+ self.debug = debug
- def parse_model_from_yaml(self, filename):
+ def parse_model_from_yaml(self, filename, existing_model: Optional[dict] = None):
"""Create and return a data model from the given file.
Parameters
@@ -197,17 +243,20 @@ class Parser(object):
filename : str
The path to the YAML file.
+ existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
Returns
-------
- out : DataModel
+ out : data_model.DataModel
The created DataModel
"""
with open(filename, 'r') as outfile:
ymlmodel = yaml.load(outfile, Loader=SafeLineLoader)
- return self._create_model_from_dict(ymlmodel)
+ return self._create_model_from_dict(ymlmodel, existing_model=existing_model)
- def parse_model_from_string(self, string):
+ def parse_model_from_string(self, string, existing_model: Optional[dict] = None):
"""Create and return a data model from the given YAML string.
Parameters
@@ -215,16 +264,19 @@ class Parser(object):
string : str
The YAML string.
+ existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
Returns
-------
- out : DataModel
+ out : data_model.DataModel
The created DataModel
"""
ymlmodel = yaml.load(string, Loader=SafeLineLoader)
- return self._create_model_from_dict(ymlmodel)
+ return self._create_model_from_dict(ymlmodel, existing_model=existing_model)
- def _create_model_from_dict(self, ymlmodel):
+ def _create_model_from_dict(self, ymlmodel, existing_model: Optional[dict] = None):
"""Create and return a data model out of the YAML dict `ymlmodel`.
Parameters
@@ -232,15 +284,21 @@ class Parser(object):
ymlmodel : dict
The dictionary parsed from a YAML file.
+ existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
Returns
-------
- out : DataModel
+ out : data_model.DataModel
The created DataModel
"""
if not isinstance(ymlmodel, dict):
raise ValueError("Yaml file should only contain one dictionary!")
+ if existing_model is not None:
+ self.model.update(existing_model)
+
# Extern keyword:
# The extern keyword can be used to include Properties and RecordTypes
# from existing CaosDB datamodels into the current model.
@@ -258,9 +316,9 @@ class Parser(object):
self.model[name] = db.Property(name=name).retrieve()
continue
for role in ("Property", "RecordType", "Record", "File"):
- if db.execute_query("COUNT {} {}".format(role, name)) > 0:
+ if db.execute_query("COUNT {} \"{}\"".format(role, name)) > 0:
self.model[name] = db.execute_query(
- "FIND {} WITH name={}".format(role, name), unique=True)
+ f"FIND {role} WITH name=\"{name}\"", unique=True)
break
else:
raise Exception("Did not find {}".format(name))
@@ -276,7 +334,12 @@ class Parser(object):
self._check_and_convert_datatypes()
for name, entity in ymlmodel.items():
- self._treat_entity(name, entity, line=ymlmodel["__line__"])
+ try:
+ self._treat_entity(name, entity, line=ymlmodel["__line__"])
+ except ValueError as err:
+ err_str = err.args[0].replace("invalid keyword:",
+ f"invalid keyword in line {entity['__line__']}:", 1)
+ raise ValueError(err_str, *err.args[1:]) from err
return DataModel(self.model.values())
@@ -327,13 +390,12 @@ class Parser(object):
if definition is None:
return
- if (self.model[name] is None
- and isinstance(definition, dict)
+ if (self.model[name] is None and isinstance(definition, dict)
# is it a property
and "datatype" in definition
# but not simply an RT of the model
- and not (_get_listdatatype(definition["datatype"]) == name and
- _get_listdatatype(definition["datatype"]) in self.model)):
+ and not (get_list_datatype(definition["datatype"]) == name and
+ get_list_datatype(definition["datatype"]) in self.model)):
# and create the new property
self.model[name] = db.Property(name=name,
@@ -383,6 +445,9 @@ class Parser(object):
raise YamlDefinitionError(line) from None
raise
+ if self.debug and self.model[name] is not None:
+ self.model[name].__line__ = definition["__line__"]
+
def _add_to_recordtype(self, ent_name, props, importance):
"""Add properties to a RecordType.
@@ -416,9 +481,9 @@ class Parser(object):
n = self._stringify(n)
if isinstance(e, dict):
- if "datatype" in e and _get_listdatatype(e["datatype"]) is not None:
+ if "datatype" in e and get_list_datatype(e["datatype"]) is not None:
# Reuse the existing datatype for lists.
- datatype = db.LIST(_get_listdatatype(e["datatype"]))
+ datatype = db.LIST(get_list_datatype(e["datatype"]))
else:
# Ignore a possible e["datatype"] here if it's not a list
# since it has been treated in the definition of the
@@ -440,6 +505,9 @@ class Parser(object):
def _inherit(self, name, prop, inheritance):
if not isinstance(prop, list):
+ if isinstance(prop, str):
+ raise YamlDefinitionError(
+ f"Parents must be a list but is given as string: {name} > {prop}")
raise YamlDefinitionError("Parents must be a list, error in line {}".format(
prop["__line__"]))
@@ -463,9 +531,13 @@ class Parser(object):
if not isinstance(definition, dict):
return
- if ("datatype" in definition
- and definition["datatype"].startswith("LIST")):
+ # These definition items must be handled even for list props.
+ for prop_name, prop in definition.items():
+ if prop_name == "description":
+ self.model[name].description = prop
+ # For lists, everything else is not needed at this level.
+ if ("datatype" in definition and definition["datatype"].startswith("LIST")):
return
if name in self.treated:
@@ -483,7 +555,8 @@ class Parser(object):
self.model[name].value = prop
elif prop_name == "description":
- self.model[name].description = prop
+ # Handled above
+ continue
elif prop_name == "recommended_properties":
self._add_to_recordtype(
@@ -520,16 +593,6 @@ class Parser(object):
self._inherit(name, prop, db.RECOMMENDED)
elif prop_name == "inherit_from_suggested":
self._inherit(name, prop, db.SUGGESTED)
- elif prop_name == "parent":
- warn(
- DeprecationWarning(
- "The `parent` keyword is deprecated and will be "
- "removed in a future version. Use "
- "`inherit_from_{obligatory|recommended|suggested}` "
- "instead."
- )
- )
- self._inherit(name, prop, db.OBLIGATORY)
else:
raise ValueError("invalid keyword: {}".format(prop_name))
@@ -557,15 +620,19 @@ class Parser(object):
dtype = value.datatype
is_list = False
- if _get_listdatatype(value.datatype) is not None:
- dtype = _get_listdatatype(value.datatype)
+ if get_list_datatype(dtype) is not None:
+ dtype = get_list_datatype(dtype)
is_list = True
- if dtype in self.model:
+ dtype_name = dtype
+ if not isinstance(dtype_name, str):
+ dtype_name = dtype.name
+
+ if dtype_name in self.model:
if is_list:
- value.datatype = db.LIST(self.model[dtype])
+ value.datatype = db.LIST(self.model[dtype_name])
else:
- value.datatype = self.model[dtype]
+ value.datatype = self.model[dtype_name]
continue
@@ -587,7 +654,7 @@ class Parser(object):
continue
raise ValueError("Property {} has an unknown datatype: {}".format(
- value.name, value.datatype))
+ value.name, dtype_name))
def _set_recordtypes(self):
""" properties are defined in first iteration; set remaining as RTs """
@@ -600,14 +667,13 @@ class Parser(object):
class JsonSchemaParser(Parser):
"""Extends the yaml parser to read in datamodels defined in a json schema.
- **EXPERIMENTAL:** While this calss can already be used to create data models
+ **EXPERIMENTAL:** While this class can already be used to create data models
from basic json schemas, there are the following limitations and missing
features:
* Due to limitations of json-schema itself, we currently do not support
inheritance in the imported data models
* The same goes for suggested properties of RecordTypes
- * Currently, ``$defs`` and ``$ref`` in the input schema are not resolved.
* Already defined RecordTypes and (scalar) Properties can't be re-used as
list properties
* Reference properties that are different from the referenced RT. (Although
@@ -615,15 +681,18 @@ class JsonSchemaParser(Parser):
* Values
* Roles
* The extern keyword from the yaml parser
- * Currently, a json-schema cannot be transformed into a data model if its
- root element isn't a RecordType (or Property) with ``title`` and ``type``.
"""
# @author Florian Spreckelsen
# @date 2022-02-17
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
- def parse_model_from_json_schema(self, filename: str):
+ def __init__(self, types_for_missing_array_items={}, ignore_unspecified_array_items=False):
+ super().__init__()
+ self.types_for_missing_array_items = types_for_missing_array_items
+ self.ignore_unspecified_array_items = ignore_unspecified_array_items
+
+ def parse_model_from_json_schema(self, filename: str, top_level_recordtype: bool = True):
"""Return a datamodel created from the definition in the json schema in
`filename`.
@@ -631,21 +700,24 @@ class JsonSchemaParser(Parser):
----------
filename : str
The path to the json-schema file containing the datamodel definition
+ top_level_recordtype : bool, optional
+ Whether there is a record type defined at the top level of the
+ schema. Default is true.
Returns
-------
- out : DataModel
+ out : data_model.DataModel
The created DataModel
"""
# @author Florian Spreckelsen
# @date 2022-02-17
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
with open(filename, 'r') as schema_file:
- model_dict = json.load(schema_file)
+ model_dict = jsonref.load(schema_file)
- return self._create_model_from_dict(model_dict)
+ return self._create_model_from_dict(model_dict, top_level_recordtype=top_level_recordtype)
- def _create_model_from_dict(self, model_dict: [dict, List[dict]]):
+ def _create_model_from_dict(self, model_dict: [dict, List[dict]], top_level_recordtype: bool = True):
"""Parse a dictionary and return the Datamodel created from it.
The dictionary was typically created from the model definition in a json schema file.
@@ -654,36 +726,68 @@ class JsonSchemaParser(Parser):
----------
model_dict : dict or list[dict]
One or several dictionaries read in from a json-schema file
+ top_level_recordtype : bool, optional
+ Whether there is a record type defined at the top level of the
+ schema. Default is true.
Returns
-------
- our : DataModel
+ our : data_model.DataModel
The datamodel defined in `model_dict`
"""
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
if isinstance(model_dict, dict):
model_dict = [model_dict]
for ii, elt in enumerate(model_dict):
- if "title" not in elt:
- raise JsonSchemaDefinitionError(
- f"Object {ii+1} is lacking the `title` key word")
- if "type" not in elt:
- raise JsonSchemaDefinitionError(
- f"Object {ii+1} is lacking the `type` key word")
- # Check if this is a valid Json Schema
try:
jsonschema.Draft202012Validator.check_schema(elt)
except jsonschema.SchemaError as err:
+ key = elt["title"] if "title" in elt else f"element {ii}"
raise JsonSchemaDefinitionError(
- f"Json Schema error in {elt['title']}:\n{str(err)}") from err
- name = self._stringify(elt["title"], context=elt)
- self._treat_element(elt, name)
+ f"Json Schema error in {key}:\n{str(err)}") from err
+
+ if top_level_recordtype:
+ if "title" not in elt:
+ raise JsonSchemaDefinitionError(
+ f"Object {ii+1} is lacking the `title` key word")
+ if "type" not in elt:
+ raise JsonSchemaDefinitionError(
+ f"Object {ii+1} is lacking the `type` key word")
+ # Check if this is a valid Json Schema
+ name = self._stringify(elt["title"], context=elt)
+ self._treat_element(elt, name)
+ elif "properties" in elt or "patternProperties" in elt:
+ # No top-level type but there are entities
+ if "properties" in elt:
+ for key, prop in elt["properties"].items():
+ name = self._get_name_from_property(key, prop)
+ self._treat_element(prop, name)
+ if "patternProperties" in elt:
+ # See also treatment in ``_treat_record_type``. Since here,
+ # there is no top-level RT we use the prefix `__Pattern`,
+ # i.e., the resulting Record Types will be called
+ # `__PatternElement`.
+ self._treat_pattern_properties(
+ elt["patternProperties"], name_prefix="__Pattern")
+ else:
+ # Neither RecordType itself, nor further properties in schema,
+ # so nothing to do here. Maybe add something in the future.
+ continue
return DataModel(self.model.values())
+ def _get_name_from_property(self, key: str, prop: dict):
+ # @review Timm Fitschen 2023-05-25
+ if "title" in prop:
+ name = self._stringify(prop["title"])
+ else:
+ name = self._stringify(key)
+
+ return name
+
def _get_atomic_datatype(self, elt):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
if elt["type"] == "string":
if "format" in elt and elt["format"] in ["date", "date-time"]:
return db.DATETIME
@@ -695,11 +799,15 @@ class JsonSchemaParser(Parser):
return db.DOUBLE
elif elt["type"] == "boolean":
return db.BOOLEAN
+ elif elt["type"] == "null":
+ # This could be any datatype since a valid json will never have a
+ # value in a null property. We use TEXT for convenience.
+ return db.TEXT
else:
raise JsonSchemaDefinitionError(f"Unkown atomic type in {elt}.")
def _treat_element(self, elt: dict, name: str):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
force_list = False
if name in self.model:
return self.model[name], force_list
@@ -710,12 +818,17 @@ class JsonSchemaParser(Parser):
if name == "name":
# This is identified with the CaosDB name property as long as the
# type is correct.
- if not elt["type"] == "string":
+ if not elt["type"] == "string" and "string" not in elt["type"]:
raise JsonSchemaDefinitionError(
"The 'name' property must be string-typed, otherwise it cannot "
"be identified with CaosDB's name property."
)
return None, force_list
+ # LinkAhead suports null for all types, so in the very special case of
+ # `"type": ["null", "<other_type>"]`, only consider the other type:
+ if isinstance(elt["type"], list) and len(elt["type"]) == 2 and "null" in elt["type"]:
+ elt["type"].remove("null")
+ elt["type"] = elt["type"][0]
if "enum" in elt:
ent = self._treat_enum(elt, name)
elif elt["type"] in JSON_SCHEMA_ATOMIC_TYPES:
@@ -733,11 +846,12 @@ class JsonSchemaParser(Parser):
# treat_something function
ent.description = elt["description"]
- self.model[name] = ent
+ if ent is not None:
+ self.model[name] = ent
return ent, force_list
def _treat_record_type(self, elt: dict, name: str):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
rt = db.RecordType(name=name)
if "required" in elt:
required = elt["required"]
@@ -745,10 +859,7 @@ class JsonSchemaParser(Parser):
required = []
if "properties" in elt:
for key, prop in elt["properties"].items():
- if "title" in prop:
- name = self._stringify(prop["title"])
- else:
- name = self._stringify(key)
+ name = self._get_name_from_property(key, prop)
prop_ent, force_list = self._treat_element(prop, name)
if prop_ent is None:
# Nothing to be appended since the property has to be
@@ -762,6 +873,17 @@ class JsonSchemaParser(Parser):
rt.add_property(prop_ent, importance=importance,
datatype=db.LIST(prop_ent))
+ if "patternProperties" in elt:
+
+ pattern_property_rts = self._treat_pattern_properties(
+ elt["patternProperties"], name_prefix=name)
+ for ppr in pattern_property_rts:
+ # add reference to pattern property type. These can never be
+ # obligatory since pattern properties cannot be required in the
+ # original schema (since their actual names are not known a
+ # priori).
+ rt.add_property(ppr)
+
if "description" in elt:
rt.description = elt["description"]
return rt
@@ -783,28 +905,96 @@ class JsonSchemaParser(Parser):
return rt
def _treat_list(self, elt: dict, name: str):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
- if "items" not in elt:
+ if "items" not in elt and name not in self.types_for_missing_array_items:
+ if self.ignore_unspecified_array_items:
+ return None, False
raise JsonSchemaDefinitionError(
f"The definition of the list items is missing in {elt}.")
- items = elt["items"]
- if "enum" in items:
- return self._treat_enum(items, name), True
- if items["type"] in JSON_SCHEMA_ATOMIC_TYPES:
- datatype = db.LIST(self._get_atomic_datatype(items))
+ if "items" in elt:
+ items = elt["items"]
+ if "enum" in items:
+ return self._treat_enum(items, name), True
+ if items["type"] in JSON_SCHEMA_ATOMIC_TYPES:
+ datatype = db.LIST(self._get_atomic_datatype(items))
+ return db.Property(name=name, datatype=datatype), False
+ if items["type"] == "object":
+ if "title" not in items or self._stringify(items["title"]) == name:
+ # Property is RecordType
+ return self._treat_record_type(items, name), True
+ else:
+ # List property will be an entity of its own with a name
+ # different from the referenced RT
+ ref_rt = self._treat_record_type(
+ items, self._stringify(items["title"]))
+ self.model[ref_rt.name] = ref_rt
+ return db.Property(name=name, datatype=db.LIST(ref_rt)), False
+ else:
+ # Use predefined type:
+ datatype = db.LIST(self.types_for_missing_array_items[name])
return db.Property(name=name, datatype=datatype), False
- if items["type"] == "object":
- if "title" not in items or self._stringify(items["title"]) == name:
- # Property is RecordType
- return self._treat_record_type(items, name), True
+
+ def _get_pattern_prop(self):
+ # @review Timm Fitschen 2023-05-25
+ if "__pattern_property_pattern_property" in self.model:
+ return self.model["__pattern_property_pattern_property"]
+ pp = db.Property(name="__matched_pattern", datatype=db.TEXT)
+ self.model["__pattern_property_pattern_property"] = pp
+ return pp
+
+ def _treat_pattern_properties(self, pattern_elements, name_prefix=""):
+ """Special Treatment for pattern properties: A RecordType is created for
+ each pattern property. In case of a `type: object` PatternProperty, the
+ remaining properties of the JSON entry are appended to the new
+ RecordType; in case of an atomic type PatternProperty, a single value
+ Property is added to the RecordType.
+
+ Raises
+ ------
+ NotImplementedError
+ In case of patternProperties with non-object, non-atomic type, e.g.,
+ array.
+
+ """
+ # @review Timm Fitschen 2023-05-25
+ num_patterns = len(pattern_elements)
+ pattern_prop = self._get_pattern_prop()
+ returns = []
+ for ii, (key, element) in enumerate(pattern_elements.items()):
+ if "title" not in element:
+ name_suffix = f"_{ii+1}" if num_patterns > 1 else ""
+ name = name_prefix + "Entry" + name_suffix
+ else:
+ name = element["title"]
+ if element["type"] == "object":
+ # simple, is already an object, so can be treated like any other
+ # record type.
+ pattern_type = self._treat_record_type(element, name)
+ elif element["type"] in JSON_SCHEMA_ATOMIC_TYPES:
+ # create a property that stores the actual value of the pattern
+ # property.
+ propname = f"{name}_value"
+ prop = db.Property(name=propname, datatype=self._get_atomic_datatype(element))
+ self.model[propname] = prop
+ pattern_type = db.RecordType(name=name)
+ pattern_type.add_property(prop)
+ else:
+ raise NotImplementedError(
+ "Pattern properties are currently only supported for types " +
+ ", ".join(JSON_SCHEMA_ATOMIC_TYPES) + ", and object.")
+
+ # Add pattern property and description
+ pattern_type.add_property(pattern_prop, importance=db.OBLIGATORY)
+ if pattern_type.description:
+ pattern_type.description += f"\n\npattern: {key}"
else:
- # List property will be an entity of its own with a name
- # different from the referenced RT
- ref_rt = self._treat_record_type(
- items, self._stringify(items["title"]))
- self.model[ref_rt.name] = ref_rt
- return db.Property(name=name, datatype=db.LIST(ref_rt)), False
+ pattern_type.description = f"pattern: {key}"
+
+ self.model[name] = pattern_type
+ returns.append(pattern_type)
+
+ return returns
if __name__ == "__main__":
diff --git a/src/caosadvancedtools/scifolder/analysis_cfood.py b/src/caosadvancedtools/scifolder/analysis_cfood.py
index 27cb871aed08f41531c367567ea36ea9a3faaf69..adce7225649ddf80852588d1cb78d045c04db1d3 100644
--- a/src/caosadvancedtools/scifolder/analysis_cfood.py
+++ b/src/caosadvancedtools/scifolder/analysis_cfood.py
@@ -16,17 +16,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import os
-from itertools import chain
-
import caosdb as db
-from caosadvancedtools.cfood import (AbstractFileCFood, assure_has_parent,
+from caosadvancedtools.cfood import (AbstractFileCFood,
assure_has_property,
- assure_object_is_in_list, get_entity)
-from caosadvancedtools.read_md_header import get_header
+ assure_object_is_in_list,
+ )
from .generic_pattern import full_pattern
-from .utils import (get_files_referenced_by_field, parse_responsibles,
+from .utils import (parse_responsibles,
reference_records_corresponding_to_files)
from .withreadme import DATAMODEL as dm
from .withreadme import (RESULTS, REVISIONOF, SCRIPTS, SOURCES, WithREADME,
@@ -44,6 +41,9 @@ class AnalysisCFood(AbstractFileCFood, WithREADME):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
WithREADME.__init__(self)
+ self.analysis = None
+ self.people = None
+ self.project = None
def collect_information(self):
self.find_referenced_files([RESULTS, SOURCES, SCRIPTS])
diff --git a/src/caosadvancedtools/scifolder/experiment_cfood.py b/src/caosadvancedtools/scifolder/experiment_cfood.py
index 38606b5f8ffd372d7bf6f507ed96738d9345f16c..83329863433d302e16744a781a3b599fe0bb11f5 100644
--- a/src/caosadvancedtools/scifolder/experiment_cfood.py
+++ b/src/caosadvancedtools/scifolder/experiment_cfood.py
@@ -17,15 +17,15 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import caosdb as db
-from caosadvancedtools.cfood import (AbstractFileCFood, assure_has_description,
- assure_has_parent, assure_has_property,
- assure_object_is_in_list, get_entity)
-from caosadvancedtools.read_md_header import get_header
+from caosadvancedtools.cfood import (AbstractFileCFood,
+ assure_has_property,
+ assure_object_is_in_list,
+ )
from .generic_pattern import full_pattern
from .utils import parse_responsibles, reference_records_corresponding_to_files
from .withreadme import DATAMODEL as dm
-from .withreadme import RESULTS, REVISIONOF, SCRIPTS, WithREADME, get_glob
+from .withreadme import RESULTS, REVISIONOF, WithREADME, get_glob
class ExperimentCFood(AbstractFileCFood, WithREADME):
@@ -42,7 +42,10 @@ class ExperimentCFood(AbstractFileCFood, WithREADME):
super().__init__(*args, **kwargs)
WithREADME.__init__(self)
- self.name_map = {},
+ self.name_map = ({}, )
+ self.experiment = None
+ self.people = None
+ self.project = None
@staticmethod
def get_re():
diff --git a/src/caosadvancedtools/scifolder/publication_cfood.py b/src/caosadvancedtools/scifolder/publication_cfood.py
index fc78e5b759e98e8989c952ccbafeef117e2ed33d..68e345aca1bd650b4da784f4741866683bd4f04a 100644
--- a/src/caosadvancedtools/scifolder/publication_cfood.py
+++ b/src/caosadvancedtools/scifolder/publication_cfood.py
@@ -16,18 +16,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import os
-from itertools import chain
-
import caosdb as db
from caosadvancedtools.cfood import (AbstractFileCFood,
assure_object_is_in_list, fileguide,
- get_entity)
+ )
from caosadvancedtools.read_md_header import get_header
-from caosadvancedtools.utils import find_records_that_reference_ids
from .generic_pattern import date_suffix_pattern, readme_pattern
-from .utils import (get_files_referenced_by_field, parse_responsibles,
+from .utils import (parse_responsibles,
reference_records_corresponding_to_files)
from .withreadme import DATAMODEL as dm
from .withreadme import (RESULTS, REVISIONOF, SCRIPTS, SOURCES, WithREADME,
@@ -37,16 +33,15 @@ from .withreadme import (RESULTS, REVISIONOF, SCRIPTS, SOURCES, WithREADME,
def folder_to_type(name):
if name == "Theses":
return "Thesis"
- elif name == "Articles":
+ if name == "Articles":
return "Article"
- elif name == "Posters":
+ if name == "Posters":
return "Poster"
- elif name == "Presentations":
+ if name == "Presentations":
return "Presentation"
- elif name == "Reports":
+ if name == "Reports":
return "Report"
- else:
- raise ValueError()
+ raise ValueError()
class PublicationCFood(AbstractFileCFood, WithREADME):
@@ -58,6 +53,8 @@ class PublicationCFood(AbstractFileCFood, WithREADME):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
WithREADME.__init__(self)
+ self.people = None
+ self.publication = None
def collect_information(self):
self.find_referenced_files([RESULTS, SOURCES, SCRIPTS])
diff --git a/src/caosadvancedtools/scifolder/result_table_cfood.py b/src/caosadvancedtools/scifolder/result_table_cfood.py
index deaa2d00118659a9b177a05fe40b19a1793a16fb..e32cd0bd1efe77d5be4583c8bae764a677e33fc4 100644
--- a/src/caosadvancedtools/scifolder/result_table_cfood.py
+++ b/src/caosadvancedtools/scifolder/result_table_cfood.py
@@ -20,17 +20,12 @@ import re
import caosdb as db
import pandas as pd
-from caosadvancedtools.cfood import (AbstractFileCFood, assure_has_description,
- assure_has_parent, assure_has_property,
- assure_object_is_in_list, get_entity)
-from caosadvancedtools.read_md_header import get_header
+from caosadvancedtools.cfood import (AbstractFileCFood,
+ )
from ..cfood import assure_property_is, fileguide
from .experiment_cfood import ExperimentCFood
from .generic_pattern import date_pattern, date_suffix_pattern, project_pattern
-from .utils import parse_responsibles, reference_records_corresponding_to_files
-from .withreadme import DATAMODEL as dm
-from .withreadme import RESULTS, REVISIONOF, SCRIPTS, WithREADME, get_glob
# TODO similarities with TableCrawler
@@ -49,6 +44,9 @@ class ResultTableCFood(AbstractFileCFood):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.table = pd.read_csv(fileguide.access(self.crawled_path))
+ self.recs = []
+ self.experiment = None
+ self.project = None
@staticmethod
def get_re():
@@ -60,7 +58,7 @@ class ResultTableCFood(AbstractFileCFood):
self.experiment, self.project = (
ExperimentCFood.create_identifiable_experiment(self.match))
- for idx, row in self.table.iterrows():
+ for _, row in self.table.iterrows():
rec = db.Record()
rec.add_parent(self.match.group("recordtype"))
@@ -77,10 +75,11 @@ class ResultTableCFood(AbstractFileCFood):
self.identifiables.extend([self.project, self.experiment])
def update_identifiables(self):
- for ii, (idx, row) in enumerate(self.table.iterrows()):
+ for ii, (_, row) in enumerate(self.table.iterrows()):
for col in row.index:
match = re.match(ResultTableCFood.property_name_re, col)
- assure_property_is(self.recs[ii], match.group("pname"), row.loc[col], to_be_updated=self.to_be_updated)
+ assure_property_is(self.recs[ii], match.group("pname"), row.loc[col],
+ to_be_updated=self.to_be_updated)
assure_property_is(self.experiment, self.match.group("recordtype"),
self.recs, to_be_updated=self.to_be_updated,
datatype=db.LIST(self.match.group("recordtype")))
diff --git a/src/caosadvancedtools/scifolder/simulation_cfood.py b/src/caosadvancedtools/scifolder/simulation_cfood.py
index c8f23f1485d7a1f64dcd940552051d2e1ec5bb07..f8f3d07e30b81e42591ce0c4698c0f47164f2b90 100644
--- a/src/caosadvancedtools/scifolder/simulation_cfood.py
+++ b/src/caosadvancedtools/scifolder/simulation_cfood.py
@@ -16,17 +16,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import os
-from itertools import chain
-
import caosdb as db
-from caosadvancedtools.cfood import (AbstractFileCFood, assure_has_parent,
+from caosadvancedtools.cfood import (AbstractFileCFood,
assure_has_property,
- assure_object_is_in_list, get_entity)
-from caosadvancedtools.read_md_header import get_header
+ assure_object_is_in_list,
+ )
from .generic_pattern import full_pattern
-from .utils import (get_files_referenced_by_field, parse_responsibles,
+from .utils import (parse_responsibles,
reference_records_corresponding_to_files)
from .withreadme import DATAMODEL as dm
from .withreadme import (RESULTS, REVISIONOF, SCRIPTS, SOURCES, WithREADME,
@@ -42,6 +39,9 @@ class SimulationCFood(AbstractFileCFood, WithREADME):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
WithREADME.__init__(self)
+ self.people = None
+ self.project = None
+ self.simulation = None
def collect_information(self):
self.find_referenced_files([RESULTS, SOURCES, SCRIPTS])
diff --git a/src/caosadvancedtools/scifolder/software_cfood.py b/src/caosadvancedtools/scifolder/software_cfood.py
index 77fb46521e9aab875b6f99d0a1ee4ac44177e09c..d91817f10a278b91f7c52aa1a0674b1a2daa0394 100644
--- a/src/caosadvancedtools/scifolder/software_cfood.py
+++ b/src/caosadvancedtools/scifolder/software_cfood.py
@@ -17,20 +17,16 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import os
-from itertools import chain
-
import caosdb as db
-from caosadvancedtools.cfood import (AbstractFileCFood, assure_has_parent,
+from caosadvancedtools.cfood import (AbstractFileCFood,
assure_has_property, assure_name_is,
- assure_object_is_in_list, get_entity)
+ assure_object_is_in_list,
+ )
from caosadvancedtools.guard import global_guard as guard
-from caosadvancedtools.read_md_header import get_header
from .generic_pattern import full_pattern
-from .utils import get_files_referenced_by_field, parse_responsibles
+from .utils import parse_responsibles
from .withreadme import BINARIES
-from .withreadme import DATAMODEL as dm
from .withreadme import SOURCECODE, WithREADME
@@ -44,6 +40,9 @@ class SoftwareCFood(AbstractFileCFood, WithREADME):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
WithREADME.__init__(self)
+ self.people = None
+ self.software = None
+ self.softwareversion = None
def collect_information(self):
self.find_referenced_files([BINARIES, SOURCECODE])
diff --git a/src/caosadvancedtools/scifolder/utils.py b/src/caosadvancedtools/scifolder/utils.py
index afa671af85506a57a06ad5198bec4495823c76f1..cbf87c4b802c829f34f4368c3605ce05fa42cfb2 100644
--- a/src/caosadvancedtools/scifolder/utils.py
+++ b/src/caosadvancedtools/scifolder/utils.py
@@ -26,7 +26,6 @@ import pandas as pd
from caosadvancedtools.cfood import assure_object_is_in_list, fileguide
from caosadvancedtools.utils import (find_records_that_reference_ids,
read_field_as_list,
- return_field_or_property,
string_to_person)
logger = logging.getLogger("caosadvancedtools")
@@ -154,7 +153,7 @@ def create_files_list(df, ftype):
files = []
for indx, src in df.loc[ftype,
- pd.notnull(df.loc[ftype])].iteritems():
+ pd.notnull(df.loc[ftype])].items():
desc = df.loc[ftype+" description", indx]
if pd.notnull(desc):
diff --git a/src/caosadvancedtools/scifolder/withreadme.py b/src/caosadvancedtools/scifolder/withreadme.py
index e1968ba49799827467c7ef93a7070b7f090010fb..d8388e1d28bd23e1804c2f747a47f4ea97d265b0 100644
--- a/src/caosadvancedtools/scifolder/withreadme.py
+++ b/src/caosadvancedtools/scifolder/withreadme.py
@@ -26,8 +26,7 @@ import caosdb as db
from caosadvancedtools.cfood import (assure_has_description, assure_has_parent,
assure_object_is_in_list, fileguide)
from caosadvancedtools.read_md_header import get_header as get_md_header
-from caosadvancedtools.table_importer import (win_path_converter,
- win_path_list_converter)
+from caosadvancedtools.table_importer import (win_path_converter)
from caosadvancedtools.utils import return_field_or_property
from .utils import (get_entity_ids_from_include_file,
@@ -236,7 +235,7 @@ class WithREADME(object):
generic_references,
record,
prop_name,
- to_be_updated=self.to_be_updated,
+ to_be_updated=self.to_be_updated, # pylint: disable=no-member
datatype=db.LIST(db.REFERENCE),
)
diff --git a/src/caosadvancedtools/serverside/generic_analysis.py b/src/caosadvancedtools/serverside/generic_analysis.py
index 85d0c860df75fce205c5eaad77731fc04eee9e40..7c7b26cc3ddb19ff54710f04debe3c0a48f35b82 100644
--- a/src/caosadvancedtools/serverside/generic_analysis.py
+++ b/src/caosadvancedtools/serverside/generic_analysis.py
@@ -38,7 +38,7 @@ Das aufgerufene Skript kann beliebige Eigenschaften benutzen und erstellen.
ABER wenn die Standardeigenschaften (InputDataSet, etc) verwendet werden, kann
der Record leicht erzeugt werden.
-
+.. code-block::
"Analyze" "Perform Anlysis"
Knopf an Record Form im WebUI
diff --git a/src/caosadvancedtools/serverside/helper.py b/src/caosadvancedtools/serverside/helper.py
index ba75739e0fdc0a83f235db6920471afb196f4246..b7289c7ffd1d67ebd9862aa5f92cd41ccc62d706 100644
--- a/src/caosadvancedtools/serverside/helper.py
+++ b/src/caosadvancedtools/serverside/helper.py
@@ -223,6 +223,7 @@ def init_data_model(entities):
"be a {}.".format(e.role, local_role))
raise DataModelError(e.name, info)
except db.exceptions.EntityDoesNotExistError:
+ # pylint: disable=raise-missing-from
raise DataModelError(e.name, "This entity does not exist.")
return True
@@ -246,7 +247,7 @@ def get_data(filename, default=None):
Data from the given file.
"""
result = default.copy() if default is not None else {}
- with open(filename, 'r') as fi:
+ with open(filename, mode="r", encoding="utf-8") as fi:
data = json.load(fi)
result.update(data)
diff --git a/src/caosadvancedtools/suppressKnown.py b/src/caosadvancedtools/suppressKnown.py
index c4b57039c5184f2443e4dbb91cf11f5e59ae6790..1b31de7e9d8f1fdce35a135d558dd5ceea3bca2a 100644
--- a/src/caosadvancedtools/suppressKnown.py
+++ b/src/caosadvancedtools/suppressKnown.py
@@ -16,8 +16,11 @@ class SuppressKnown(logging.Filter):
added to the appropriate Logger and logging calls (e.g. to warning, info
etc.) need to have an additional `extra` argument.
This argument should be a dict that contains an identifier and a category.
- Example: `extra={"identifier":"<Record>something</Record>",
- category="entities"}
+
+ Example::
+
+ extra={"identifier":"<Record>something</Record>", category="entities"}
+
The identifier is used to check whether a message was shown before and
should be a string. The category can be used to remove a specific group of
messages from memory and the logger would show those messages again even
@@ -74,9 +77,7 @@ class SuppressKnown(logging.Filter):
return sha256((txt+str(identifier)).encode("utf-8")).hexdigest()
def filter(self, record):
- """
- Return whether the record shall be logged.
-
+ """Return whether the record shall be logged.
If either identifier of category is missing 1 is returned (logging
enabled). If the record has both attributes, it is checked whether the
diff --git a/src/caosadvancedtools/table_converter.py b/src/caosadvancedtools/table_converter.py
index 76f4dfcdb5f040d81d923289a7a730806ad8681b..7d1097e13a0780a7656a18c71bf7f408df5c69c5 100644
--- a/src/caosadvancedtools/table_converter.py
+++ b/src/caosadvancedtools/table_converter.py
@@ -25,6 +25,7 @@ import re
import sys
import caosdb as db
+import numpy as np
import pandas as pd
@@ -48,27 +49,25 @@ def generate_property_name(prop):
def to_table(container):
- """ creates a table from the records in a container """
+ """Create a table from the records in a container."""
if len(container) == 0:
- raise ValueError("Container is empty")
- properties = set()
+ return pd.DataFrame()
+ rts = {p.name for p in container[0].parents}
+ data = []
for rec in container:
- properties.update([generate_property_name(p)
- for p in container[0].get_properties()])
- df = pd.DataFrame(columns=list(properties))
- rts = set([p.name for p in container[0].parents])
-
- for ii, rec in enumerate(container):
- if set([p.name for p in rec.parents]) != rts:
+ if {p.name for p in rec.parents} != rts:
raise ValueError("Parents differ")
- for p in rec.get_properties():
-
- df.loc[ii, generate_property_name(p)] = p.value
+ row_dict = {}
+ for prop in rec.get_properties():
+ propname = generate_property_name(prop)
+ row_dict[propname] = prop.value
+ data.append(row_dict)
+ result = pd.DataFrame(data=data)
- return df
+ return result
def from_table(spreadsheet, recordtype):
@@ -79,7 +78,7 @@ def from_table(spreadsheet, recordtype):
rec = db.Record()
rec.add_parent(name=recordtype)
- for key, value in row.iteritems():
+ for key, value in row.items():
if key.lower() == "description":
rec.description = value
continue
diff --git a/src/caosadvancedtools/table_export.py b/src/caosadvancedtools/table_export.py
index 056207a76fa01357e2269cd4cb8e9a09905d5d90..eabb10754bdb93859dcc6ef3d3ff0838fa6ff6d4 100644
--- a/src/caosadvancedtools/table_export.py
+++ b/src/caosadvancedtools/table_export.py
@@ -27,6 +27,7 @@ them for an export as a table, e.g., for the export to metadata
repositories.
"""
+from inspect import signature
import json
import logging
@@ -83,7 +84,7 @@ class BaseTableExporter(object):
```
{"entry_to_be_exported: {
"optional": True/False
- "find_func": name of member function
+ "find_func": callable or name of member function
"query": query string
"selector": selector for the query
"error": error explanation
@@ -97,8 +98,8 @@ class BaseTableExporter(object):
- optional: True or False, if not present, the entry is
assumed to be mandatory.
- find_func: name of the member function that returns the
- value for this entry. Must not exist together with
- `query`
+ value for this entry or callable object. Must not exist
+ together with `query`
- query: Query string for finding the value for this
entry. If this is given, a record must be given to the
constructor of this class. The query is then executed as
@@ -132,6 +133,7 @@ class BaseTableExporter(object):
self._check_sanity_of_export_dict()
self.raise_error_if_missing = raise_error_if_missing
self.info = {}
+ self.all_keys = [key for key in self.export_dict]
def collect_information(self):
"""Use the items of `export_dict` to collect the information for the
@@ -139,7 +141,8 @@ class BaseTableExporter(object):
"""
- for e, d in self.export_dict.items():
+ for e in self.all_keys:
+ d = self.export_dict[e]
if QUERY in d:
# TODO: How do we make this more general? There might
# be queries that don't need the record or work with
@@ -163,12 +166,15 @@ class BaseTableExporter(object):
else:
self._append_missing(e, d)
elif FIND_FUNCTION in d:
- find_fun = getattr(self, d[FIND_FUNCTION])
try:
- self.info[e] = find_fun()
+ val = self._call_find_function(d[FIND_FUNCTION], e)
+ if val is not None:
+ self.info[e] = val
+ else:
+ self._append_missing(e, d)
except Exception as exc:
self._append_missing(e, d)
- logger.debug(exc)
+ logger.error(exc)
# last resort: check if record has e as property:
else:
try:
@@ -200,6 +206,20 @@ class BaseTableExporter(object):
else:
logger.error(errmssg)
+ def _call_find_function(self, find_function, e):
+ if callable(find_function):
+ find_fun = find_function
+ else:
+ find_fun = getattr(self, find_function)
+
+ sig = signature(find_fun)
+ params = sig.parameters
+ if len(params) > 1:
+ return find_fun(self.record, e)
+ elif len(params) > 0:
+ return find_fun(self.record)
+ return find_fun()
+
def prepare_csv_export(self, delimiter=',', print_header=False,
skip_empty_optionals=False):
"""Return the values in self.info as a single-line string, separated
@@ -238,7 +258,8 @@ class BaseTableExporter(object):
if print_header:
header = ""
- for e, d in self.export_dict.items():
+ for e in self.all_keys:
+ d = self.export_dict[e]
if e in self.info:
body += str(self.info[e]) + delimiter
@@ -287,7 +308,9 @@ class BaseTableExporter(object):
# check find function if present
if FIND_FUNCTION in d:
- if not hasattr(self, d[FIND_FUNCTION]):
+ if callable(d[FIND_FUNCTION]):
+ pass
+ elif not hasattr(self, d[FIND_FUNCTION]):
raise TableExportError(
"Find function " + d[FIND_FUNCTION] +
" was specified for entry " + e +
diff --git a/src/caosadvancedtools/table_importer.py b/src/caosadvancedtools/table_importer.py
index 1f515e78e3ddbd198fa0336589a359ba9154f038..bae813b23195c93ccfd369a626424dd069164fb0 100755
--- a/src/caosadvancedtools/table_importer.py
+++ b/src/caosadvancedtools/table_importer.py
@@ -210,7 +210,7 @@ class TableImporter():
"""
def __init__(self, converters, obligatory_columns=None, unique_keys=None,
- datatypes=None):
+ datatypes=None, existing_columns=None):
"""
Parameters
----------
@@ -221,7 +221,7 @@ class TableImporter():
value check is not necessary.
obligatory_columns : list, optional
- List of column names, each listed column must not have missing values.
+ List of column names that (if they exist) must not have missing values.
unique_keys : list, optional
List of column names that in combination must be unique: each row has a unique
@@ -232,22 +232,31 @@ class TableImporter():
checked whether they have the provided datatype. This dict also defines what columns are
required to exist throught the existing keys.
+ existing_columns : list, optional
+ List of column names that must exist but may have missing (NULL) values
"""
if converters is None:
converters = {}
+ self.converters = converters
+
+ if obligatory_columns is None:
+ obligatory_columns = []
+ self.obligatory_columns = obligatory_columns
+
+ if unique_keys is None:
+ unique_keys = []
+ self.unique_keys = unique_keys
if datatypes is None:
datatypes = {}
+ self.datatypes = datatypes
+
+ if existing_columns is None:
+ existing_columns = []
+ self.existing_columns = existing_columns
self.sup = SuppressKnown()
- self.required_columns = list(converters.keys())+list(datatypes.keys())
- self.obligatory_columns = ([]
- if obligatory_columns is None
- else obligatory_columns)
- self.unique_keys = [] if unique_keys is None else unique_keys
- self.converters = converters
- self.datatypes = datatypes
def read_file(self, filename, **kwargs):
raise NotImplementedError()
@@ -263,7 +272,7 @@ class TableImporter():
"""
- for col in self.required_columns:
+ for col in self.existing_columns:
if col not in df.columns:
errmsg = "Column '{}' missing in ".format(col)
errmsg += ("\n{}.\n".format(filename) if filename
@@ -313,7 +322,7 @@ class TableImporter():
.. note::
If columns are integer, but should be float, this method converts the respective columns
- in place.
+ in place. The same for columns that should have string value but have numeric value.
Parameters
----------
@@ -323,18 +332,21 @@ class TableImporter():
"""
for key, datatype in self.datatypes.items():
+ if key not in df.columns:
+ continue
# Check for castable numeric types first: We unconditionally cast int to the default
# float, because CaosDB does not have different sizes anyway.
col_dtype = df.dtypes[key]
if not strict and not np.issubdtype(col_dtype, datatype):
- issub = np.issubdtype
# These special cases should be fine.
- if issub(col_dtype, np.integer) and issub(datatype, np.floating):
+ if ((datatype == str)
+ or (np.issubdtype(col_dtype, np.integer)
+ and np.issubdtype(datatype, np.floating))
+ ): # NOQA
df[key] = df[key].astype(datatype)
# Now check each element
- for idx, val in df.loc[
- pd.notnull(df.loc[:, key]), key].iteritems():
+ for idx, val in df.loc[pd.notnull(df.loc[:, key]), key].items():
if not isinstance(val, datatype):
msg = (
@@ -363,24 +375,23 @@ class TableImporter():
for index, row in df.iterrows():
# if none of the relevant information is given, skip
-
- if np.array([pd.isnull(row.loc[key]) for key in
- self.obligatory_columns]).all():
-
+ if pd.isnull(row.loc[[key for key in self.obligatory_columns if key in df.columns]]).all():
df = df.drop(index)
continue
# if any of the relevant information is missing, report it
-
i = 0
okay = True
while okay and i < len(self.obligatory_columns):
key = self.obligatory_columns[i]
i += 1
+ if key not in df.columns:
+ continue
- if pd.isnull(row.loc[key]):
+ null_check = pd.isnull(row.loc[key])
+ if (isinstance(null_check, np.ndarray) and null_check.any()) or (not isinstance(null_check, np.ndarray) and null_check):
errmsg = (
"Required information is missing ({}) in {}. row"
" (without header) of "
@@ -449,7 +460,10 @@ class XLSImporter(TableImporter):
"All but the first are being ignored.".format(filename))
try:
- df = xls_file.parse(converters=self.converters, **kwargs)
+ tmpdf = xls_file.parse(**kwargs)
+ applicable_converters = {k: v for k, v in self.converters.items()
+ if k in tmpdf.columns}
+ df = xls_file.parse(converters=applicable_converters, **kwargs)
except Exception as e:
logger.warning(
"Cannot parse {}.\n{}".format(filename, e),
@@ -465,7 +479,11 @@ class XLSImporter(TableImporter):
class CSVImporter(TableImporter):
def read_file(self, filename, sep=",", **kwargs):
try:
- df = pd.read_csv(filename, sep=sep, converters=self.converters,
+ tmpdf = pd.read_csv(filename, sep=sep, converters=self.converters,
+ **kwargs)
+ applicable_converters = {k: v for k, v in self.converters.items()
+ if k in tmpdf.columns}
+ df = pd.read_csv(filename, sep=sep, converters=applicable_converters,
**kwargs)
except ValueError as ve:
logger.warning(
@@ -482,6 +500,10 @@ class CSVImporter(TableImporter):
class TSVImporter(TableImporter):
def read_file(self, filename, **kwargs):
try:
+ tmpdf = pd.read_csv(filename, sep="\t", converters=self.converters,
+ **kwargs)
+ applicable_converters = {k: v for k, v in self.converters.items()
+ if k in tmpdf.columns}
df = pd.read_csv(filename, sep="\t", converters=self.converters,
**kwargs)
except ValueError as ve:
diff --git a/src/caosadvancedtools/table_json_conversion/__init__.py b/src/caosadvancedtools/table_json_conversion/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/src/caosadvancedtools/table_json_conversion/convert.py b/src/caosadvancedtools/table_json_conversion/convert.py
new file mode 100644
index 0000000000000000000000000000000000000000..09882f963fd976583d4acbc8f2dd3b67ef510ac8
--- /dev/null
+++ b/src/caosadvancedtools/table_json_conversion/convert.py
@@ -0,0 +1,497 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Convert XLSX files to JSON dictionaries."""
+
+from __future__ import annotations
+
+import datetime
+import itertools
+import sys
+from functools import reduce
+from operator import getitem
+from types import SimpleNamespace
+from typing import Any, BinaryIO, Callable, TextIO, Union
+from warnings import warn
+
+import jsonschema
+from openpyxl import load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+
+from caosadvancedtools.table_json_conversion import xlsx_utils
+from caosadvancedtools.table_json_conversion.fill_xlsx import read_or_dict
+
+
+def _strict_bool(value: Any) -> bool:
+ """Convert value to bool, but only if it really is a valid XLSX bool."""
+ if isinstance(value, bool):
+ return value
+ raise TypeError(f"Not a good boolean: {repr(value)}")
+
+
+class ForeignError(KeyError):
+ def __init__(self, *args, definitions: list, message: str = ""):
+ super().__init__(message, *args)
+ self.definitions = definitions
+
+
+class XLSXConverter:
+ """Class for conversion from XLSX to JSON.
+
+For a detailed description of the required formatting of the XLSX files, see ``specs.md`` in the
+documentation.
+ """
+
+ PARSER: dict[str, Callable] = {
+ "string": str,
+ "number": float,
+ "integer": int,
+ "boolean": _strict_bool,
+ }
+
+ def __init__(self, xlsx: Union[str, BinaryIO], schema: Union[dict, str, TextIO],
+ strict: bool = False):
+ """
+Parameters
+----------
+xlsx: Union[str, BinaryIO]
+ Path to the XLSX file or opened file object.
+
+schema: Union[dict, str, TextIO]
+ Schema for validation of XLSX content.
+
+strict: bool, optional
+ If True, fail faster.
+"""
+ self._workbook = load_workbook(xlsx)
+ self._schema = read_or_dict(schema)
+ self._defining_path_index = xlsx_utils.get_defining_paths(self._workbook)
+ self._check_columns(fail_fast=strict)
+ self._handled_sheets: set[str] = set()
+ self._result: dict = {}
+ self._errors: dict = {}
+
+ def to_dict(self, validate: bool = False, collect_errors: bool = True) -> dict:
+ """Convert the xlsx contents to a dict.
+
+Parameters
+----------
+validate: bool, optional
+ If True, validate the result against the schema.
+
+collect_errors: bool, optional
+ If True, do not fail at the first error, but try to collect as many errors as possible. After an
+ Exception is raised, the errors can be collected with ``get_errors()`` and printed with
+ ``get_error_str()``.
+
+Returns
+-------
+out: dict
+ A dict representing the JSON with the extracted data.
+ """
+ self._handled_sheets = set()
+ self._result = {}
+ self._errors = {}
+ for sheetname in self._workbook.sheetnames:
+ if sheetname not in self._handled_sheets:
+ self._handle_sheet(self._workbook[sheetname], fail_later=collect_errors)
+ if validate:
+ jsonschema.validate(self._result, self._schema)
+ if self._errors:
+ raise RuntimeError("There were error while handling the XLSX file.")
+ return self._result
+
+ def get_errors(self) -> dict:
+ """Return a dict with collected errors."""
+ return self._errors
+
+ def get_error_str(self) -> str:
+ """Return a beautiful string with the collected errors."""
+ result = ""
+ for loc, value in self._errors.items():
+ result += f"Sheet: {loc[0]}\tRow: {loc[1] + 1}\n"
+ for item in value:
+ result += f"\t\t{item[:-1]}:\t{item[-1]}\n"
+ return result
+
+ def _check_columns(self, fail_fast: bool = False):
+ """Check if the columns correspond to the schema."""
+ def missing(path):
+ message = f"Missing column: {xlsx_utils.p2s(path)}"
+ if fail_fast:
+ raise ValueError(message)
+ else:
+ warn(message)
+ for sheetname in self._workbook.sheetnames:
+ sheet = self._workbook[sheetname]
+ parents: dict = {}
+ col_paths = []
+ for col in xlsx_utils.get_data_columns(sheet).values():
+ parents[xlsx_utils.p2s(col.path[:-1])] = col.path[:-1]
+ col_paths.append(col.path)
+ for path in parents.values():
+ subschema = xlsx_utils.get_subschema(path, self._schema)
+
+ # Unfortunately, there are a lot of special cases to handle here.
+ if subschema.get("type") == "array":
+ subschema = subschema["items"]
+ if "enum" in subschema: # Was handled in parent level already
+ continue
+ for child, content in subschema["properties"].items():
+ child_path = path + [child]
+ if content == {'type': 'string', 'format': 'data-url'}:
+ continue # skip files
+ if content.get("type") == "array" and (
+ content.get("items").get("type") == "object"):
+ if child_path not in itertools.chain(*self._defining_path_index.values()):
+ missing(child_path)
+ elif content.get("type") == "array" and "enum" in content.get("items", []) and (
+ content.get("uniqueItems") is True):
+ # multiple choice
+ for choice in content["items"]["enum"]:
+ if child_path + [choice] not in col_paths:
+ missing(child_path + [choice])
+ elif content.get("type") == "object":
+ pass
+ else:
+ if child_path not in col_paths:
+ missing(child_path)
+
+ def _handle_sheet(self, sheet: Worksheet, fail_later: bool = False) -> None:
+ """Add the contents of the sheet to the result (stored in ``self._result``).
+
+Each row in the sheet corresponds to one entry in an array in the result. Which array exactly is
+defined by the sheet's "proper name" and the content of the foreign columns.
+
+Look at ``xlsx_utils.get_path_position`` for the specification of the "proper name".
+
+
+Parameters
+----------
+fail_later: bool, optional
+ If True, do not fail with unresolvable foreign definitions, but collect all errors.
+"""
+ row_type_column = xlsx_utils.get_row_type_column_index(sheet)
+ foreign_columns = xlsx_utils.get_foreign_key_columns(sheet)
+ foreign_column_paths = {col.index: col.path for col in foreign_columns.values()}
+ data_columns = xlsx_utils.get_data_columns(sheet)
+ data_column_paths = {col.index: col.path for col in data_columns.values()}
+ # Parent path, insert in correct order.
+ parent, proper_name = xlsx_utils.get_path_position(sheet)
+ if parent:
+ parent_sheetname = xlsx_utils.get_worksheet_for_path(parent, self._defining_path_index)
+ if parent_sheetname not in self._handled_sheets:
+ self._handle_sheet(self._workbook[parent_sheetname], fail_later=fail_later)
+
+ # # We save single entries in lists, indexed by their foreign key contents. Each entry
+ # # consists of:
+ # # - foreign: Dict with path -> value for the foreign columns
+ # # - data: The actual data of this entry, a dict.
+ # entries: dict[str, list[SimpleNamespace]] = {}
+
+ for row_idx, row in enumerate(sheet.iter_rows(values_only=True)):
+ # Skip non-data rows.
+ if row[row_type_column] is not None:
+ continue
+ foreign_repr = ""
+ foreign = [] # A list of lists, each of which is: [path1, path2, ..., leaf, value]
+ data: dict = {} # Local data dict
+ # Collect data (in dict relative to current level) and foreign data information
+ for col_idx, value in enumerate(row):
+ if col_idx in foreign_column_paths:
+ foreign_repr += str(value)
+ foreign.append(foreign_column_paths[col_idx] + [value])
+ continue
+
+ if col_idx in data_column_paths:
+ path = data_column_paths[col_idx]
+ if self._is_multiple_choice(path):
+ real_value = path.pop() # Last component is the enum value, insert above
+ # set up list
+ try:
+ _set_in_nested(mydict=data, path=path, value=[], prefix=parent, skip=1)
+ except ValueError as err:
+ if not str(err).startswith("There is already some value at"):
+ raise
+ if not xlsx_utils.parse_multiple_choice(value):
+ continue
+ _set_in_nested(mydict=data, path=path, value=real_value, prefix=parent,
+ skip=1, append_to_list=True)
+ else:
+ value = self._validate_and_convert(value, path)
+ _set_in_nested(mydict=data, path=path, value=value, prefix=parent, skip=1)
+ continue
+
+ try:
+ # Find current position in tree
+ parent_dict = self._get_parent_dict(parent_path=parent, foreign=foreign)
+
+ # Append data to current position's list
+ if proper_name not in parent_dict:
+ parent_dict[proper_name] = []
+ parent_dict[proper_name].append(data)
+ except ForeignError as kerr:
+ if not fail_later:
+ raise
+ self._errors[(sheet.title, row_idx)] = kerr.definitions
+ self._handled_sheets.add(sheet.title)
+
+ def _is_multiple_choice(self, path: list[str]) -> bool:
+ """Test if the path belongs to a multiple choice section."""
+ if not path:
+ return False
+ subschema = self._get_subschema(path[:-1])
+ if (subschema["type"] == "array"
+ and subschema.get("uniqueItems") is True
+ and "enum" in subschema["items"]):
+ return True
+ return False
+
+ def _get_parent_dict(self, parent_path: list[str], foreign: list[list]) -> dict:
+ """Return the dict into which values can be inserted.
+
+This method returns, from the current result-in-making, the entry at ``parent_path`` which matches
+the values given in the ``foreign`` specification.
+"""
+ foreign_groups = _group_foreign_paths(foreign, common=parent_path)
+
+ current_object = self._result
+ for group in foreign_groups:
+ # Find list for which foreign definitions are relevant.
+ current_object = reduce(getitem, group.subpath, current_object)
+ assert isinstance(current_object, list)
+ # Test all candidates.
+ for cand in current_object:
+ if all(reduce(getitem, definition[:-1], cand) == definition[-1]
+ for definition in group.definitions):
+ current_object = cand
+ break
+ else:
+ message = f"Cannot find an element at {parent_path} for these foreign defs:\n"
+ for name, value in group.definitions:
+ message += f" {name}: {value}\n"
+ print(message, file=sys.stderr)
+ error = ForeignError(definitions=group.definitions, message=message)
+ raise error
+ assert isinstance(current_object, dict)
+ return current_object
+
+ def _validate_and_convert(self, value: Any, path: list[str]):
+ """Apply some basic validation and conversion steps.
+
+This includes:
+- Validation against the type given in the schema
+- List typed values are split at semicolons and validated individually
+ """
+ if value is None:
+ return value
+ subschema = self._get_subschema(path)
+ # Array handling only if schema says it's an array.
+ if subschema.get("type") == "array":
+ array_type = subschema["items"]["type"]
+ if isinstance(value, str) and ";" in value:
+ values = [self.PARSER[array_type](v) for v in value.split(";")]
+ return values
+ try:
+ # special case: datetime or date
+ if ("anyOf" in subschema):
+ if isinstance(value, datetime.datetime) and (
+ {'type': 'string', 'format': 'date-time'} in subschema["anyOf"]):
+ return value
+ if isinstance(value, datetime.date) and (
+ {'type': 'string', 'format': 'date'} in subschema["anyOf"]):
+ return value
+ jsonschema.validate(value, subschema)
+ except jsonschema.ValidationError as verr:
+ print(verr)
+ print(path)
+ raise
+
+ # Finally: convert to target type
+ return self.PARSER[subschema.get("type", "string")](value)
+
+ def _get_subschema(self, path: list[str], schema: dict = None) -> dict:
+ """Return the sub schema at ``path``."""
+ if schema is None:
+ schema = self._schema
+ assert schema is not None
+ assert isinstance(schema, dict)
+
+ return xlsx_utils.get_subschema(path, schema)
+
+
+def _group_foreign_paths(foreign: list[list], common: list[str]) -> list[SimpleNamespace]:
+ """Group the foreign keys by their base paths.
+
+Parameters
+----------
+foreign: list[list]
+ A list of foreign definitions, consisting of path components, property and possibly value.
+
+common: list[list[str]]
+ A common path which defines the final target of the foreign definitions. This helps to understand
+ where the ``foreign`` paths shall be split.
+
+Returns
+-------
+out: list[dict[str, list[list]]]
+
+ A list of foreign path segments, grouped by their common segments. Each element is a namespace
+ with detailed information of all those elements which form the group. The namespace has the
+ following attributes:
+
+ - ``path``: The full path to this path segment. This is always the previous segment's ``path``
+ plus this segment's ``subpath``.
+ - ``stringpath``: The stringified ``path``, might be useful for comparison or sorting.
+ - ``subpath``: The path, relative from the previous segment.
+ - ``definitions``: A list of the foreign definitions for this segment, but stripped of the
+ ``path`` components.
+ """
+ # Build a simple dict first, without subpath.
+ results = {}
+ for f_path in foreign:
+ path = []
+ for component in f_path:
+ path.append(component)
+ if path != common[:len(path)]:
+ break
+ path.pop()
+ definition = f_path[len(path):]
+ stringpath = xlsx_utils.p2s(path)
+ if stringpath not in results:
+ results[stringpath] = SimpleNamespace(stringpath=stringpath, path=path,
+ definitions=[definition])
+ else:
+ results[stringpath].definitions.append(definition)
+
+ # Then sort by stringpath and calculate subpath.
+ stringpaths = sorted(results.keys())
+
+ resultlist = []
+ last_level = 0
+ for stringpath in stringpaths:
+ elem = results[stringpath]
+ elem.subpath = elem.path[last_level:]
+ last_level = len(elem.path)
+ resultlist.append(elem)
+
+ # from IPython import embed
+ # embed()
+
+ if last_level != len(common):
+ raise ValueError("Foreign keys must cover the complete `common` depth.")
+ return resultlist
+
+
+# pylint: disable-next=dangerous-default-value,too-many-arguments
+def _set_in_nested(mydict: dict, path: list, value: Any, prefix: list = [], skip: int = 0,
+ overwrite: bool = False, append_to_list: bool = False) -> dict:
+ """Set a value in a nested dict.
+
+Parameters
+----------
+mydict: dict
+ The dict into which the ``value`` shall be inserted.
+path: list
+ A list of keys, denoting the location of the value.
+value
+ The value which shall be set inside the dict.
+prefix: list
+ A list of keys which shall be removed from ``path``. A KeyError is raised if ``path`` does not
+ start with the elements of ``prefix``.
+skip: int = 0
+ Remove this many additional levels from the path, *after* removing the prefix.
+overwrite: bool = False
+ If True, allow overwriting existing content. Otherwise, attempting to overwrite existing values
+ leads to an exception.
+append_to_list: bool = False
+ If True, assume that the element at ``path`` is a list and append the value to it. If the list
+ does not exist, create it. If there is a non-list at ``path`` already, overwrite it with a new
+ list, if ``overwrite`` is True, otherwise raise a ValueError.
+
+Returns
+-------
+mydict: dict
+ The same dictionary that was given as a parameter, but modified.
+ """
+ for idx, el in enumerate(prefix):
+ if path[idx] != el:
+ raise KeyError(f"Path does not start with prefix: {prefix} not in {path}")
+ path = path[len(prefix):]
+ if skip:
+ assert len(path) > skip, f"Path must be long enoug to remove skip={skip} elements."
+ path = path[skip:]
+
+ tmp_dict = mydict
+ while len(path) > 1:
+ key = path.pop(0)
+ if key not in tmp_dict:
+ tmp_dict[key] = {}
+ if not isinstance(tmp_dict[key], dict):
+ if overwrite:
+ tmp_dict[key] = {}
+ else:
+ raise ValueError(f"There is already some value at {path}")
+ tmp_dict = tmp_dict[key]
+ key = path.pop()
+ if append_to_list:
+ if key not in tmp_dict:
+ tmp_dict[key] = []
+ if not isinstance(tmp_dict[key], list):
+ if overwrite:
+ tmp_dict[key] = []
+ else:
+ raise ValueError(f"There is already some non-list value at [{key}]")
+ tmp_dict[key].append(value)
+ else:
+ if key in tmp_dict and not overwrite:
+ raise ValueError(f"There is already some value at [{key}]")
+ if key not in tmp_dict:
+ tmp_dict[key] = {}
+ tmp_dict[key] = value
+ return mydict
+
+
+def to_dict(xlsx: Union[str, BinaryIO], schema: Union[dict, str, TextIO],
+ validate: bool = None, strict: bool = False) -> dict:
+ """Convert the xlsx contents to a dict, it must follow a schema.
+
+Parameters
+----------
+xlsx: Union[str, BinaryIO]
+ Path to the XLSX file or opened file object.
+
+schema: Union[dict, str, TextIO]
+ Schema for validation of XLSX content.
+
+validate: bool, optional
+ If True, validate the result against the schema.
+
+strict: bool, optional
+ If True, fail faster.
+
+
+Returns
+-------
+out: dict
+ A dict representing the JSON with the extracted data.
+ """
+ converter = XLSXConverter(xlsx, schema, strict=strict)
+ return converter.to_dict()
diff --git a/src/caosadvancedtools/table_json_conversion/fill_xlsx.py b/src/caosadvancedtools/table_json_conversion/fill_xlsx.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2e0abc3fc684172065d683c99c1c4309c80d6c0
--- /dev/null
+++ b/src/caosadvancedtools/table_json_conversion/fill_xlsx.py
@@ -0,0 +1,374 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Class and function to fill an XLSX template from actual data."""
+
+from __future__ import annotations
+
+import datetime
+import pathlib
+from types import SimpleNamespace
+from typing import Any, Optional, TextIO, Union
+from warnings import warn
+
+from jsonschema import FormatChecker, validate
+from jsonschema.exceptions import ValidationError
+from openpyxl import load_workbook, Workbook
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
+
+from .xlsx_utils import (
+ array_schema_from_model_schema,
+ get_foreign_key_columns,
+ get_row_type_column_index,
+ is_exploded_sheet,
+ next_row_index,
+ p2s,
+ read_or_dict,
+ ColumnType,
+ RowType
+)
+
+
+class TemplateFiller:
+ """Class to fill XLSX templates. Has an index for all relevant columns."""
+
+ def __init__(self, workbook: Workbook, graceful: bool = False):
+ self._workbook = workbook
+ self._graceful = graceful
+ self._create_index()
+
+ @property
+ def workbook(self):
+ """Return the workbook of this TemplateFiller."""
+ return self._workbook
+
+ def fill_data(self, data: dict):
+ """Fill the data into the workbook."""
+ self._handle_data(data=data)
+
+ class Context:
+ """Context for an entry: simple properties of all ancestors, organized in a dict.
+
+ This is similar to a dictionary with all scalar element properties at the tree nodes up to
+ the root. Siblings in lists and dicts are ignored. Additionally the context knows where
+ its current position is.
+
+ Lookup of elements can easily be achieved by giving the path (as ``list[str]`` or
+ stringified path).
+
+ """
+
+ def __init__(self, current_path: list[str] = None, props: dict[str, Any] = None):
+ self._current_path = current_path if current_path is not None else []
+ self._props = props if props is not None else {} # this is flat
+
+ def copy(self) -> TemplateFiller.Context:
+ """Deep copy."""
+ result = TemplateFiller.Context(current_path=self._current_path.copy(),
+ props=self._props.copy())
+ return result
+
+ def next_level(self, next_level: str) -> TemplateFiller.Context:
+ """Return a copy of this Context, with the path appended by ``next_level``."""
+ result = self.copy()
+ result._current_path.append(next_level) # pylint: disable=protected-access
+ return result
+
+ def __getitem__(self, path: Union[list[str], str], owner=None) -> Any:
+ if isinstance(path, list):
+ path = p2s(path)
+ return self._props[path]
+
+ def __setitem__(self, propname: str, value):
+ fullpath = p2s(self._current_path + [propname])
+ self._props[fullpath] = value
+
+ def fill_from_data(self, data: dict[str, Any]):
+ # TODO recursive for dicts and list?
+ """Fill current level with all scalar elements of ``data``."""
+ for name, value in data.items():
+ if not isinstance(value, (dict, list)):
+ self[name] = value
+ elif isinstance(value, dict):
+ if not value or isinstance(list(value.items())[0], list):
+ continue
+ old_path = self._current_path
+ new_path = self._current_path.copy() + [name]
+ self._current_path = new_path
+ self.fill_from_data(data=value)
+ self._current_path = old_path
+
+ def _create_index(self):
+ """Create a sheet index for the workbook.
+
+ Index the sheets by all path arrays leading to them. Also create a simple column index by
+ column type and path.
+
+ This method creates and populates the dict ``self._sheet_index``.
+ """
+ self._sheet_index = {}
+ for sheetname in self._workbook.sheetnames:
+ sheet = self._workbook[sheetname]
+ type_column = [x.value for x in list(sheet.columns)[
+ get_row_type_column_index(sheet)]]
+ # 0-indexed, as everything outside of sheet.cell(...):
+ coltype_idx = type_column.index(RowType.COL_TYPE.name)
+ path_indices = [i for i, typ in enumerate(type_column) if typ == RowType.PATH.name]
+
+ # Get the paths, use without the leaf component for sheet indexing, with type prefix and
+ # leaf for column indexing.
+ for col_idx, col in enumerate(sheet.columns):
+ if col[coltype_idx].value == RowType.COL_TYPE.name:
+ continue
+ path = []
+ for path_idx in path_indices:
+ if col[path_idx].value is not None:
+ path.append(col[path_idx].value)
+ # col_key = p2s([col[coltype_idx].value] + path)
+ # col_index[col_key] = SimpleNamespace(column=col, col_index=col_idx)
+ if col[coltype_idx].value not in [ColumnType.SCALAR.name, ColumnType.LIST.name,
+ ColumnType.MULTIPLE_CHOICE.name]:
+ continue
+
+ path_str = p2s(path)
+ assert path_str not in self._sheet_index
+ self._sheet_index[path_str] = SimpleNamespace(
+ sheetname=sheetname, sheet=sheet, col_index=col_idx,
+ col_type=col[coltype_idx].value)
+
+ def _handle_data(self, data: dict, current_path: list[str] = None,
+ context: TemplateFiller.Context = None,
+ only_collect_insertables: bool = False,
+ utc: bool = False,
+ ) -> Optional[dict[str, Any]]:
+ """Handle the data and write it into ``workbook``.
+
+Parameters
+----------
+data: dict
+ The data at the current path position. Elements may be dicts, lists or simple scalar values.
+
+current_path: list[str], optional
+ If this is None or empty, we are at the top level. This means that all children shall be entered
+ into their respective sheets and not into a sheet at this level. ``current_path`` and ``context``
+ must either both be given, or none of them.
+
+context: TemplateFiller.Context, optional
+ Directopry of scalar element properties at the tree nodes up to the root. Siblings in lists
+ and dicts are ignored. ``context`` and ``current_path`` must either both be given, or none of
+ them.
+
+only_collect_insertables: bool, optional
+ If True, do not insert anything on this level, but return a dict with entries to be inserted.
+
+utc: bool, optional
+ If True, store times as UTC. Else store as local time on a best-effort base.
+
+Returns
+-------
+out: union[dict, None]
+ If ``only_collect_insertables`` is True, return a dict (path string -> value)
+ """
+ assert (current_path is None) is (context is None), (
+ "`current_path` and `context` must either both be given, or none of them.")
+ if current_path is None:
+ current_path = []
+ if context is None:
+ context = TemplateFiller.Context()
+ context.fill_from_data(data)
+
+ insertables: dict[str, Any] = {}
+ for name, content in data.items():
+ # TODO is this the best way to do it????
+ if name == "file":
+ continue
+ path = current_path + [name]
+ next_context = context.next_level(name)
+ # preprocessing
+ if isinstance(content, list):
+ if not content: # empty list
+ continue
+ # List elements must be all of the same type.
+ assert len(set(type(entry) for entry in content)) == 1
+
+ if isinstance(content[0], dict): # all elements are dicts
+ # Heuristic to detect enum entries (only id and name):
+ if all(set(entry.keys()) == {"id", "name"} for entry in content):
+ # Convert to list of names, do not recurse
+ content = [entry["name"] for entry in content]
+ else:
+ # An array of objects: must go into exploded sheet
+ for entry in content:
+ self._handle_data(data=entry, current_path=path, context=next_context)
+ continue
+ # Heuristic to detect enum entries (dict with only id and name):
+ elif isinstance(content, dict) and set(content.keys()) == {"id", "name"}:
+ content = [content["name"]]
+ # "Normal" dicts
+ elif isinstance(content, dict): # we recurse and simply use the result
+ if not current_path: # Special handling for top level
+ self._handle_data(content, current_path=path, context=next_context)
+ continue
+ insert = self._handle_data(content, current_path=path, context=next_context.copy(),
+ only_collect_insertables=True)
+ assert isinstance(insert, dict)
+ assert not any(key in insertables for key in insert)
+ insertables.update(insert)
+ continue
+ else: # scalars
+ content = [content] # make list for unified treatment below
+
+ # collecting the data
+ assert isinstance(content, list)
+ to_insert = self._try_multiple_choice(path, values=content)
+ if not to_insert:
+ if len(content) > 1:
+ content = [ILLEGAL_CHARACTERS_RE.sub("", str(x)) for x in content]
+ value = ";".join(content) # TODO we need escaping of values
+ else:
+ value = content[0]
+ if isinstance(value, str):
+ value = ILLEGAL_CHARACTERS_RE.sub("", value)
+ if isinstance(value, datetime.datetime):
+ if value.tzinfo is not None:
+ if utc:
+ value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+ else:
+ # Remove timezone, store in local timezone.
+ value = value.astimezone().replace(tzinfo=None)
+
+ path_str = p2s(path)
+ assert path_str not in insertables
+ to_insert = {path_str: value}
+ insertables.update(to_insert)
+ if only_collect_insertables:
+ return insertables
+ if not current_path: # Top level returns, because there are only sheets for the children.
+ return None
+
+ # actual data insertion
+ insert_row = None
+ sheet = None
+ for path_str, value in insertables.items():
+ if self._graceful and path_str not in self._sheet_index:
+ if not (value is None or path_str.endswith(".id") or path_str.endswith(".name")):
+ warn(f"Ignoring path with missing sheet index: {path_str}")
+ continue
+ sheet_meta = self._sheet_index[path_str]
+ if sheet is None:
+ sheet = sheet_meta.sheet
+ assert sheet is sheet_meta.sheet, "All entries must be in the same sheet."
+ col_index = sheet_meta.col_index
+ if insert_row is None:
+ insert_row = next_row_index(sheet)
+
+ sheet.cell(row=insert_row+1, column=col_index+1, value=value)
+
+ # Insert foreign keys
+ if insert_row is not None and sheet is not None and is_exploded_sheet(sheet):
+ try:
+ foreigns = get_foreign_key_columns(sheet)
+ except ValueError:
+ print(f"Sheet: {sheet}")
+ raise
+ for index, path in ((f.index, f.path) for f in foreigns.values()):
+ value = context[path]
+ sheet.cell(row=insert_row+1, column=index+1, value=value)
+
+ return None
+
+ def _try_multiple_choice(self, path: list[str], values: list[str]) -> Optional[dict[str, str]]:
+ """Try to create sheet content for a multiple choice property.
+
+Parameters
+----------
+path: list[str]
+ The Path to this property.
+values: list[str]
+ A list of possible choices, should be unique.
+
+Returns
+-------
+to_insert: Optional[dict[str, str]]
+ A path-value dict. None if this doesn't seem to be a multiple choice data set.
+ """
+ try:
+ assert len(set(values)) == len(values)
+ to_insert = {}
+ found_sheet = None
+ for value in values:
+ assert isinstance(value, str)
+ path_str = p2s(path + [value])
+ assert path_str in self._sheet_index
+ sheet_meta = self._sheet_index[path_str]
+ # All matches shall be on the same sheet
+ assert found_sheet is None or found_sheet == sheet_meta.sheetname
+ found_sheet = sheet_meta.sheetname
+ # Correct type
+ assert sheet_meta.col_type == ColumnType.MULTIPLE_CHOICE.name
+ to_insert[path_str] = "x"
+ except AssertionError:
+ return None
+ return to_insert
+
+
+def fill_template(data: Union[dict, str, TextIO], template: str, result: str,
+ validation_schema: Union[dict, str, TextIO] = None) -> None:
+ """Insert json data into an xlsx file, according to a template.
+
+This function fills the json data into the template stored at ``template`` and stores the result as
+``result``.
+
+Parameters
+----------
+data: Union[dict, str, TextIO]
+ The data, given as Python dict, path to a file or a file-like object.
+template: str
+ Path to the XLSX template.
+result: str
+ Path for the result XLSX.
+validation_schema: dict, optional
+ If given, validate the date against this schema first. This raises an exception if the validation
+ fails. If no validation schema is given, try to ignore more errors in the data when filling the
+ XLSX template.
+"""
+ data = read_or_dict(data)
+ assert isinstance(data, dict)
+
+ # Validation
+ if validation_schema is not None:
+ validation_schema = array_schema_from_model_schema(read_or_dict(validation_schema))
+ try:
+ # FIXME redefine checker for datetime
+ validate(data, validation_schema, format_checker=FormatChecker())
+ except ValidationError as verr:
+ print(verr.message)
+ raise verr
+ else:
+ print("No validation schema given, continue at your own risk.")
+
+ # Filling the data
+ result_wb = load_workbook(template)
+ template_filler = TemplateFiller(result_wb, graceful=(validation_schema is None))
+ template_filler.fill_data(data=data)
+
+ parentpath = pathlib.Path(result).parent
+ parentpath.mkdir(parents=True, exist_ok=True)
+ result_wb.save(result)
diff --git a/src/caosadvancedtools/table_json_conversion/table_generator.py b/src/caosadvancedtools/table_json_conversion/table_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8c50e7d8d40775f86c1a01d0934effe570cf20d
--- /dev/null
+++ b/src/caosadvancedtools/table_json_conversion/table_generator.py
@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""
+This module allows to generate template tables from JSON schemas.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import re
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from openpyxl import Workbook
+from openpyxl.styles import PatternFill
+from openpyxl.workbook.child import INVALID_TITLE_REGEX
+
+from .xlsx_utils import p2s, ColumnType, RowType
+
+
+class TableTemplateGenerator(ABC):
+ """ base class for generating tables from json schema """
+
+ def __init__(self):
+ pass
+
+ @abstractmethod
+ def generate(self, schema: dict, foreign_keys: dict, filepath: str):
+ """Generate a sheet definition from a given JSON schema.
+
+ Parameters:
+ -----------
+ schema: dict
+ Given JSON schema.
+
+ foreign_keys: dict
+ A tree-like configuration (nested dict) that defines which attributes shall be used to
+ create additional columns when a list of references exists. The nested dict is
+ structured like the data model, its innermost elements are leaves of the path trees
+ within the JSON, they define the required keys.
+
+ | Suppose we want to distinguish Persons that are referenced by Trainings, then
+ ``foreign_keys`` must at least contain the following:
+ | ``{"Training": {"Person": ["name", "email"]}}``.
+
+ Values within the dicts can be either a list representing the keys (as in the example
+ above) or a dict that allows to set additional foreign keys at higher depths. In the
+ latter case (dict instead of list) if foreign keys exist at that level (e.g. in the
+ above example there might be further levels below "Person"), then the foreign keys can
+ be set using the special ``__this__`` key.
+
+ Example: ``{"Training": {"__this__": ["date"], "Person": ["name", "email"]}}``
+ Here, ``date`` is the sole foreign key for Training.
+
+ | It probably is worth extending the first example, with a case where a "Training" shall
+ be distiguished by the "name" and "email" of a "Person" which it references. The
+ foreign keys for this example are specified like this:
+ | ``{"Training": {"__this__": [["Person", "name"], ["Person", "email"]]}}``
+ """
+
+ def _generate_sheets_from_schema(self, schema: dict, foreign_keys: Optional[dict] = None
+ ) -> dict[str, dict[str,
+ tuple[ColumnType, Optional[str], list]]]:
+ """Generate a sheet definition from a given JSON schema.
+
+ Parameters
+ ----------
+ schema: dict
+ given JSON schema
+ foreign_keys: dict, optional
+ a configuration that defines which attributes shall be used to create
+ additional columns when a list of references exists. See ``foreign_keys``
+ argument of TableTemplateGenerator.generate.
+
+ Returns
+ -------
+ sheets: dict
+ A two-level dict which describes columns of template sheets.
+
+ | The structure of this two-level dict is as follows:
+ | ``sheets[sheetname][colname]= (<col_type>, <description>, [<path>, ...])``
+
+ I.e. the outer dict contains sheet names as keys, the inner dict has column names as
+ keys and tuples as values. These tuples consist of:
+ - the column type
+ - the description of the corresponding property
+ - a list representing the path.
+
+ """
+ if not ("type" in schema or "anyOf" in schema):
+ raise ValueError("Inappropriate JSON schema: The following object must contain the "
+ f"'type' or 'anyOf' key:\n{schema}\n")
+ if "properties" not in schema:
+ raise ValueError("Inappropriate JSON schema: The following object must contain "
+ f"the 'properties' key:\n{schema}\n")
+ if "type" in schema:
+ assert schema["type"] == "object"
+ if foreign_keys is None:
+ foreign_keys = {}
+ # here, we treat the top level
+ # sheets[sheetname][colname]= (COL_TYPE, description, [path])
+ sheets: dict[str, dict[str, tuple[ColumnType, Optional[str], list]]] = {}
+ for rt_name, rt_def in schema["properties"].items():
+ sheets[rt_name] = self._treat_schema_element(schema=rt_def, sheets=sheets,
+ path=[rt_name], foreign_keys=foreign_keys)
+ return sheets
+
+ def _get_foreign_keys(self, keys: dict, path: list) -> list[list[str]]:
+ """Return the foreign keys that are needed at the location to which path points.
+
+Returns
+-------
+foreign_keys: list[list[str]]
+ Contains lists of strings, each element is the path to one foreign key.
+"""
+ msg_missing = f"A foreign key definition is missing for path:\n{path}\nKeys are:\n{keys}"
+ orig_path = path.copy()
+ while path:
+ if keys is None or path[0] not in keys:
+ raise ValueError(msg_missing)
+ keys = keys[path[0]]
+ path = path[1:]
+ if isinstance(keys, dict) and "__this__" in keys:
+ keys = keys["__this__"]
+ if isinstance(keys, str):
+ raise ValueError("Foreign keys must be a list of strings, but a single "
+ "string was given:\n"
+ f"{orig_path} -> {keys}")
+ if not isinstance(keys, list):
+ raise ValueError(msg_missing)
+
+ # Keys must be either all lists or all strings
+ types = {type(key) for key in keys}
+ if len(types) > 1:
+ raise ValueError("The keys of this path must bei either all lists or all strings:"
+ f" {orig_path}")
+ if types.pop() is str:
+ keys = [[key] for key in keys]
+ return keys
+
+ def _treat_schema_element(self, schema: dict, sheets: dict, path: list[str],
+ foreign_keys: Optional[dict] = None, level_in_sheet_name: int = 1,
+ array_paths: Optional[list] = None
+ ) -> dict[str, tuple[ColumnType, Optional[str], list]]:
+ """Recursively transform elements from the schema into column definitions.
+
+ ``sheets`` is modified in place.
+
+ Parameters
+ ----------
+ schema: dict
+ Part of the json schema; it must be the level that contains the type definition
+ (e.g. 'type' or 'oneOf' key)
+ sheets: dict
+ All the sheets, indexed by their name. This is typically modified in place by this
+ method.
+ path: list[str]
+ The relevant (sub) path for this schema part?
+ array_paths: list
+ A list of path along the way to the current object, where the json contains arrays.
+
+ Returns
+ -------
+ columns: dict
+ Describing the columns; see doc string of `_generate_sheets_from_schema`_
+ """
+ if not ("type" in schema or "enum" in schema or "oneOf" in schema or "anyOf" in schema):
+ raise ValueError("Inappropriate JSON schema: The following schema part must contain "
+ f"'type', 'enum', 'oneOf' or 'anyOf':\n{schema}\n")
+
+ if array_paths is None:
+ # if this is not set, we are at top level and the top level element may always be an
+ # array
+ array_paths = [path]
+ if foreign_keys is None:
+ foreign_keys = {}
+
+ ctype = ColumnType.SCALAR
+
+ # if it is an array, value defs are in 'items'
+ if schema.get('type') == 'array':
+ items = schema['items']
+ # list of references; special treatment
+ if (items.get('type') == 'object' and len(path) > 1):
+ # we add a new sheet with columns generated from the subtree of the schema
+ sheetname = p2s(path)
+ if sheetname in sheets:
+ raise ValueError("The schema would lead to two sheets with the same name, "
+ f"which is forbidden: {sheetname}")
+ col_def = self._treat_schema_element(
+ schema=items, sheets=sheets, path=path, foreign_keys=foreign_keys,
+ level_in_sheet_name=len(path),
+ array_paths=array_paths+[path] # since this level is an array extend the list
+ )
+ if col_def:
+ sheets[sheetname] = col_def
+ # and add the foreign keys that are necessary up to this point
+ for array_path in array_paths:
+ foreigns = self._get_foreign_keys(foreign_keys, array_path)
+ for foreign in foreigns:
+ internal_key = p2s(array_path + foreign)
+ if internal_key in sheets[sheetname]:
+ raise ValueError("The schema would lead to two columns with the "
+ "same name, which is forbidden:\n"
+ f"{foreign} -> {internal_key}")
+ ref_sheet = p2s(array_path)
+ sheets[sheetname][internal_key] = (
+ ColumnType.FOREIGN, f"see sheet '{ref_sheet}'",
+ array_path + foreign)
+ # Columns are added to the new sheet, thus we do not return any columns for the
+ # current sheet.
+ return {}
+
+ # List of enums: represent as checkbox columns
+ if (schema.get("uniqueItems") is True and "enum" in items and len(items) == 1):
+ choices = items["enum"]
+ assert len(path) >= 1
+ prop_name = path[-1]
+ result = {}
+ for choice in choices:
+ name = f"{prop_name}.{choice}"
+ result[name] = (
+ ColumnType.MULTIPLE_CHOICE,
+ schema.get('description'),
+ path + [str(choice)],
+ )
+ return result
+
+ # it is a list of primitive types -> semicolon separated list
+ schema = items
+ ctype = ColumnType.LIST
+
+ # This should only be the case for "new or existing reference".
+ for el in schema.get('oneOf', []):
+ if 'type' in el:
+ schema = el
+ break
+
+ if "properties" in schema: # recurse for each property, then return
+ cols = {}
+ for pname in schema["properties"]:
+ col_defs = self._treat_schema_element(
+ schema["properties"][pname], sheets, path+[pname], foreign_keys,
+ level_in_sheet_name, array_paths=array_paths)
+ for k in col_defs:
+ if k in cols:
+ raise ValueError(f"The schema would lead to two columns with the same "
+ f"name which is forbidden: {k}")
+ cols.update(col_defs)
+ return cols
+
+ # The schema is a leaf.
+ # definition of a single column
+ default_return = {p2s(path[level_in_sheet_name:]): (ctype, schema.get('description'), path)}
+ if 'type' not in schema and 'enum' in schema:
+ return default_return
+ if 'type' not in schema and 'anyOf' in schema:
+ for d in schema['anyOf']:
+ # currently the only case where this occurs is date formats
+ assert d['type'] == 'string'
+ assert d['format'] == 'date' or d['format'] == 'date-time'
+ return default_return
+ if schema["type"] in ['string', 'number', 'integer', 'boolean']:
+ if 'format' in schema and schema['format'] == 'data-url':
+ return {} # file; ignore for now
+ return default_return
+ raise ValueError("Inappropriate JSON schema: The following part should define an"
+ f" object with properties or a primitive type:\n{schema}\n")
+
+
+class XLSXTemplateGenerator(TableTemplateGenerator):
+ """Class for generating XLSX tables from json schema definitions."""
+
+ # def __init__(self):
+ # super().__init__()
+
+ def generate(self, schema: dict, foreign_keys: dict, filepath: str) -> None:
+ """Generate a sheet definition from a given JSON schema.
+
+ Parameters:
+ -----------
+ schema: dict
+ Given JSON schema
+ foreign_keys: dict
+ A configuration that defines which attributes shall be used to create
+ additional columns when a list of references exists. See ``foreign_keys``
+ argument of :meth:`TableTemplateGenerator.generate` .
+ filepath: str
+ The XLSX file will be stored under this path.
+ """
+ sheets = self._generate_sheets_from_schema(schema, foreign_keys)
+ wb = self._create_workbook_from_sheets_def(sheets)
+ parentpath = pathlib.Path(filepath).parent
+ parentpath.mkdir(parents=True, exist_ok=True)
+ wb.save(filepath)
+
+ @staticmethod
+ def _get_max_path_length(sheetdef: dict) -> int:
+ """ returns the length of the longest path contained in the sheet definition
+
+ see TableTemplateGenerator._generate_sheets_from_schema for the structure of the sheets
+ definition dict
+ You need to pass the dict of a single sheet to this function.
+ """
+ return max(len(path) for _, _, path in sheetdef.values())
+
+ @staticmethod
+ def _get_ordered_cols(sheetdef: dict) -> list:
+ """
+ creates a list with tuples (colname, column type, path) where the foreign keys are first
+ """
+ ordered_cols = []
+ # first foreign cols
+ for colname, (ct, desc, path) in sheetdef.items():
+ if ct == ColumnType.FOREIGN:
+ ordered_cols.append((colname, ct, desc, path))
+ # now the other
+ for colname, (ct, desc, path) in sheetdef.items():
+ if ct != ColumnType.FOREIGN:
+ ordered_cols.append((colname, ct, desc, path))
+
+ return ordered_cols
+
+ def _create_workbook_from_sheets_def(
+ self, sheets: dict[str, dict[str, tuple[ColumnType, Optional[str], list]]]):
+ """Create and return a nice workbook for the given sheets."""
+ wb = Workbook()
+ yellowfill = PatternFill(fill_type="solid", fgColor='00FFFFAA')
+ # remove initial sheet
+ assert wb.sheetnames == ["Sheet"]
+ del wb['Sheet']
+
+ for sheetname, sheetdef in sheets.items():
+ if not sheetdef:
+ continue
+ ws = wb.create_sheet(re.sub(INVALID_TITLE_REGEX, '_', sheetname))
+ # First row will by the COL_TYPE row.
+ # First column will be the indicator row with values COL_TYPE, PATH, IGNORE.
+ # The COL_TYPE row will be followed by as many PATH rows as needed.
+
+ max_path_length = self._get_max_path_length(sheetdef)
+ header_index = 2 + max_path_length
+ description_index = 3 + max_path_length
+
+ # create first column
+ ws.cell(1, 1, RowType.COL_TYPE.name)
+ for index in range(max_path_length):
+ ws.cell(2 + index, 1, RowType.PATH.name)
+ ws.cell(header_index, 1, RowType.IGNORE.name)
+ ws.cell(description_index, 1, RowType.IGNORE.name)
+
+ ordered_cols = self._get_ordered_cols(sheetdef)
+
+ # create other columns
+ for index, (colname, coltype, desc, path) in enumerate(ordered_cols):
+ ws.cell(1, 2 + index, coltype.name)
+ for path_index, el in enumerate(path):
+ ws.cell(2 + path_index, 2 + index, el)
+ ws.cell(header_index, 2 + index, colname)
+ if coltype == ColumnType.FOREIGN:
+ # Visual highlighting
+ ws.cell(header_index, 2 + index).fill = yellowfill
+ if desc:
+ ws.cell(description_index, 2 + index, desc)
+
+ # hide special rows
+ for index, row in enumerate(ws.rows):
+ if not (row[0].value is None or row[0].value == RowType.IGNORE.name):
+ ws.row_dimensions[index+1].hidden = True
+
+ # hide special column
+ ws.column_dimensions['A'].hidden = True
+
+ # order sheets
+ # for index, sheetname in enumerate(sorted(wb.sheetnames)):
+ # wb.move_sheet(sheetname, index-wb.index(wb[sheetname]))
+ # reverse sheets
+ for index, sheetname in enumerate(wb.sheetnames[::-1]):
+ wb.move_sheet(sheetname, index-wb.index(wb[sheetname]))
+
+ return wb
diff --git a/src/caosadvancedtools/table_json_conversion/xlsx_utils.py b/src/caosadvancedtools/table_json_conversion/xlsx_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..5002f3ac7fe4bd78accffe0697cd7ecc7273dc27
--- /dev/null
+++ b/src/caosadvancedtools/table_json_conversion/xlsx_utils.py
@@ -0,0 +1,378 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""General utilities to work with XLSX files with (hidden) column and row annotations and typing.
+
+The most prominent functions are:
+
+- ``p2s``: Path to string: ``["some", "path"] -> "some.path"``
+- ``read_or_dict``: Load JSON object from path, file or dict.
+
+This module also defines these enums:
+
+- ColumnType
+- RowType
+"""
+
+from __future__ import annotations
+
+import json
+
+from collections import OrderedDict
+from copy import deepcopy
+from enum import Enum
+from types import SimpleNamespace
+from typing import Any, TextIO, Union
+
+from openpyxl import Workbook
+from openpyxl.worksheet.worksheet import Worksheet
+
+
+TRUTHY = {"true", "wahr", "x", "√", "yes", "ja", "y", "j"} # For multiple choice columns
+FALSY = {"false", "falsch", "-", "no", "nein", "n"} # For multiple choice columns
+
+
+class ColumnType(Enum):
+ """ column types enum """
+ SCALAR = 1
+ LIST = 2
+ FOREIGN = 3
+ MULTIPLE_CHOICE = 4
+ IGNORE = 5
+
+
+class RowType(Enum):
+ """ row types enum """
+ COL_TYPE = 1
+ PATH = 2
+ IGNORE = 3
+
+
+def array_schema_from_model_schema(model_schema: dict) -> dict:
+ """Convert a *data model* schema to a *data array* schema.
+
+Practically, this means that the top level properties are converted into lists. In a simplified
+notation, this can be expressed as:
+
+``array_schema = { elem: [elem typed data...] for elem in model_schema }``
+
+Parameters
+----------
+model_schema: dict
+ The schema description of the data model. Must be a json schema *object*, with a number of
+ *object* typed properties.
+
+Returns
+-------
+array_schema: dict
+ A corresponding json schema, where the properties are arrays with the types of the input's
+ top-level properties.
+ """
+ assert model_schema["type"] == "object"
+ result = deepcopy(model_schema)
+ for name, prop in result["properties"].items():
+ assert prop["type"] == "object"
+ new_prop = {
+ "type": "array",
+ "items": prop
+ }
+ result["properties"][name] = new_prop
+ return result
+
+
+def get_defining_paths(workbook: Workbook) -> dict[str, list[list[str]]]:
+ """For all sheets in ``workbook``, list the paths which they define.
+
+A sheet is said to define a path, if it has data columns for properties inside that path. For
+example, consider the following worksheet:
+
+| `COL_TYPE` | `SCALAR` | `SCALAR` | `LIST` | `SCALAR` |
+| `PATH` | `Training` | `Training` | `Training` | `Training` |
+| `PATH` | `url` | `date` | `subjects` | `supervisor` |
+| `PATH` | | | | `email` |
+|------------|----------------|---------------|--------------|--------------------|
+| | example.com/mp | 2024-02-27 | Math;Physics | steve@example.com |
+| | example.com/m | 2024-02-27 | Math | stella@example.com |
+
+This worksheet defines properties for the paths `["Training"]` and `["Training", "supervisor"]`, and
+thus these two path lists would be returned for the key with this sheet's sheetname.
+
+Parameters
+----------
+workbook: Workbook
+ The workbook to analyze.
+
+Returns
+-------
+out: dict[str, list[list[str]]
+ A dict with worksheet names as keys and lists of paths (represented as string lists) as values.
+"""
+ result: dict[str, list[list[str]]] = {}
+ for sheet in workbook.worksheets:
+ paths = []
+ added = set()
+ for col in get_data_columns(sheet).values():
+ rep = p2s(col.path[:-1])
+ if rep not in added:
+ paths.append(col.path[:-1])
+ added.add(rep)
+ result[sheet.title] = paths
+ return result
+
+
+def get_data_columns(sheet: Worksheet) -> dict[str, SimpleNamespace]:
+ """Return the data paths of the worksheet.
+
+Returns
+-------
+out: dict[str, SimpleNamespace]
+ The keys are the stringified paths. The values are SimpleNamespace objects with ``index``,
+ ``path`` and ``column`` attributes.
+ """
+ column_types = _get_column_types(sheet)
+ path_rows = get_path_rows(sheet)
+ result = OrderedDict()
+ for for_idx, name in column_types.items():
+ if name not in (
+ ColumnType.SCALAR.name,
+ ColumnType.LIST.name,
+ ColumnType.MULTIPLE_CHOICE.name,
+ ):
+ continue
+ path = []
+ for row in path_rows:
+ component = sheet.cell(row=row+1, column=for_idx+1).value
+ if component is None:
+ break
+ assert isinstance(component, str), f"Expected string: {component}"
+ path.append(component)
+ result[p2s(path)] = SimpleNamespace(index=for_idx, path=path,
+ column=list(sheet.columns)[for_idx])
+ return result
+
+
+def get_foreign_key_columns(sheet: Worksheet) -> dict[str, SimpleNamespace]:
+ """Return the foreign keys of the worksheet.
+
+Returns
+-------
+out: dict[str, SimpleNamespace]
+ The keys are the stringified paths. The values are SimpleNamespace objects with ``index``,
+ ``path`` and ``column`` attributes.
+ """
+ column_types = _get_column_types(sheet)
+ path_rows = get_path_rows(sheet)
+ result = OrderedDict()
+ for for_idx, name in column_types.items():
+ if name != ColumnType.FOREIGN.name:
+ continue
+ path = []
+ for row in path_rows:
+ component = sheet.cell(row=row+1, column=for_idx+1).value
+ if component is None:
+ break
+ assert isinstance(component, str), f"Expected string: {component}"
+ path.append(component)
+ result[p2s(path)] = SimpleNamespace(index=for_idx, path=path,
+ column=list(sheet.columns)[for_idx])
+ return result
+
+
+def get_path_position(sheet: Worksheet) -> tuple[list[str], str]:
+ """Return a path which represents the parent element, and the sheet's "proper name".
+
+For top-level sheets / entries (those without foreign columns), the path is an empty list.
+
+A sheet's "proper name" is detected from the data column paths: it is the first component after the
+parent components.
+
+Returns
+-------
+parent: list[str]
+ Path to the parent element. Note that there may be list elements on the path which are **not**
+ represented in this return value.
+
+proper_name: str
+ The "proper name" of this sheet. This defines an array where all the data lives, relative to the
+ parent path.
+ """
+ # Parent element: longest common path shared among any foreign column and all the data columns
+ parent: list[str] = []
+
+ # longest common path in data colums
+ data_paths = [el.path for el in get_data_columns(sheet).values()]
+ for ii in range(min(len(path) for path in data_paths)):
+ components_at_index = {path[ii] for path in data_paths}
+ if len(components_at_index) > 1:
+ break
+ longest_data_path = data_paths[0][:ii]
+
+ # longest common overall path
+ foreign_paths = [el.path for el in get_foreign_key_columns(sheet).values()]
+ ii = 0 # If no foreign_paths, proper name is the first element
+ for foreign_path in foreign_paths:
+ for ii in range(min([len(foreign_path), len(longest_data_path)])):
+ components_at_index = {foreign_path[ii], longest_data_path[ii]}
+ if len(components_at_index) > 1:
+ break
+ if ii > len(parent):
+ parent = foreign_path[:ii]
+
+ return parent, data_paths[0][ii]
+
+
+def get_path_rows(sheet: Worksheet):
+ """Return the 0-based indices of the rows which represent paths."""
+ rows = []
+ rt_col = get_row_type_column_index(sheet)
+ for cell in list(sheet.columns)[rt_col]:
+ if cell.value == RowType.PATH.name:
+ rows.append(cell.row-1)
+ return rows
+
+
+def get_row_type_column_index(sheet: Worksheet):
+ """Return the column index (0-indexed) of the column which defines the row types.
+ """
+ for col in sheet.columns:
+ for cell in col:
+ if cell.value == RowType.COL_TYPE.name:
+ return cell.column - 1
+ raise ValueError("The column which defines row types (COL_TYPE, PATH, ...) is missing")
+
+
+def get_subschema(path: list[str], schema: dict) -> dict:
+ """Return the sub schema at ``path``."""
+ if path:
+ if schema["type"] == "object":
+ next_schema = schema["properties"][path[0]]
+ return get_subschema(path=path[1:], schema=next_schema)
+ if schema["type"] == "array":
+ items = schema["items"]
+ if "enum" in items:
+ return schema
+ next_schema = items["properties"][path[0]]
+ return get_subschema(path=path[1:], schema=next_schema)
+ return schema
+
+
+def get_worksheet_for_path(path: list[str], defining_path_index: dict[str, list[list[str]]]) -> str:
+ """Find the sheet name which corresponds to the given path."""
+ for sheetname, paths in defining_path_index.items():
+ if path in paths:
+ return sheetname
+ raise KeyError(f"Could not find defining worksheet for path: {path}")
+
+
+def is_exploded_sheet(sheet: Worksheet) -> bool:
+ """Return True if this is a an "exploded" sheet.
+
+ An exploded sheet is a sheet whose data entries are LIST valued properties of entries in another
+ sheet. A sheet is detected as exploded iff it has FOREIGN columns.
+ """
+ column_types = _get_column_types(sheet)
+ return ColumnType.FOREIGN.name in column_types.values()
+
+
+def next_row_index(sheet: Worksheet) -> int:
+ """Return the index for the next data row.
+
+ This is defined as the first row without any content.
+ """
+ return sheet.max_row
+
+
+def p2s(path: list[str]) -> str:
+ """Path to string: dot-separated.
+ """
+ return ".".join(path)
+
+
+def parse_multiple_choice(value: Any) -> bool:
+ """Interpret ``value`` as a multiple choice input.
+
+*Truthy* values are:
+- The boolean ``True``.
+- The number "1".
+- The (case-insensitive) strings ``true``, ``wahr``, ``x``, ``√``, ``yes``, ``ja``, ``y``, ``j``.
+
+*Falsy* values are:
+- The boolean ``False``.
+- ``None``, empty strings, lists, dicts.
+- The number "0".
+- The (case-insensitive) strings ``false``, ``falsch``, ``-``, ``no``, ``nein``, ``n``.
+- Everything else.
+
+Returns
+-------
+out: bool
+ The interpretation result of ``value``.
+ """
+ # Non-string cases first:
+ # pylint: disable-next=too-many-boolean-expressions
+ if (value is None or value is False or value == 0
+ or value == [] or value == {} or value == ""):
+ return False
+ if (value is True or value == 1):
+ return True
+
+ # String cases follow:
+ if not isinstance(value, str):
+ return False
+ value = value.lower()
+
+ if value in TRUTHY:
+ return True
+
+ # Strictly speaking, this test is not necessary, but I think it's good practice.
+ if value in FALSY:
+ return False
+ return False
+
+
+def read_or_dict(data: Union[dict, str, TextIO]) -> dict:
+ """If data is a json file name or input stream, read data from there.
+If it is a dict already, just return it."""
+ if isinstance(data, dict):
+ return data
+
+ if isinstance(data, str):
+ with open(data, encoding="utf-8") as infile:
+ data = json.load(infile)
+ elif hasattr(data, "read"):
+ data = json.load(data)
+ else:
+ raise ValueError(f"I don't know how to handle the datatype of `data`: {type(data)}")
+ assert isinstance(data, dict)
+ return data
+
+
+def _get_column_types(sheet: Worksheet) -> OrderedDict:
+ """Return an OrderedDict: column index -> column type for the sheet.
+ """
+ result = OrderedDict()
+ type_row_index = get_row_type_column_index(sheet)
+ for idx, col in enumerate(sheet.columns):
+ type_cell = col[type_row_index]
+ result[idx] = type_cell.value if type_cell.value is not None else (
+ ColumnType.IGNORE.name)
+ assert (hasattr(ColumnType, result[idx]) or result[idx] == RowType.COL_TYPE.name), (
+ f"Unexpected column type value ({idx}{type_row_index}): {type_cell.value}")
+ return result
diff --git a/src/caosadvancedtools/utils.py b/src/caosadvancedtools/utils.py
index 2504f56976e2f6122f3e3468db1c7ae807bbb8cd..05000a34fd27162837ecfe316b20af42b1156c45 100644
--- a/src/caosadvancedtools/utils.py
+++ b/src/caosadvancedtools/utils.py
@@ -55,6 +55,26 @@ def replace_path_prefix(path, old_prefix, new_prefix):
return os.path.join(new_prefix, path)
+def create_entity_link(entity: db.Entity, base_url: str = ""):
+ """
+ creates a string that contains the code for an html link to the provided entity.
+
+ The text of the link is the entity name if one exists and the id otherwise.
+
+ Args:
+ entity (db.Entity): the entity object to which the link will point
+ base_url (str): optional, by default, the url starts with '/Entity' and thus is relative.
+ You can provide a base url that will be prefixed.
+ Returns:
+ str: the string containing the html code
+
+ """
+ return "<a href='{}/Entity/{}'>{}</a>".format(
+ base_url,
+ entity.id,
+ entity.name if entity.name is not None else entity.id)
+
+
def string_to_person(person):
"""
Creates a Person Record from a string.
@@ -93,17 +113,22 @@ def read_field_as_list(field):
return [field]
-def get_referenced_files(glob, prefix=None, filename=None, location=None):
+def get_referenced_files(glob: str, prefix: str = None, filename: str = None, location: str = None):
"""
queries the database for files referenced by the provided glob
- Parameters:
- glob: the glob referencing the file(s)
- prefix: the glob can be relative to some path, in that case that path needs
- to be given as prefix
- filename: the file in which the glob is given (used for error messages)
- location: the location in the file in which the glob is given (used for
- error messages)
+ Parameters
+ ----------
+ glob: str
+ the glob referencing the file(s)
+ prefix: str, optional
+ the glob can be relative to some path, in that case that path needs
+ to be given as prefix
+ filename: str, optional
+ the file in which the glob is given (used for error messages)
+ location: str, optional
+ the location in the file in which the glob is given (used for
+ error messages)
"""
orig_glob = glob
@@ -141,16 +166,19 @@ def get_referenced_files(glob, prefix=None, filename=None, location=None):
return files
-def check_win_path(path, filename=None):
+def check_win_path(path: str, filename: str = None):
"""
check whether '/' are in the path but no '\'.
If that is the case, it is likely, that the path is not a Windows path.
- Parameters:
- path: path to be checked
- filename: if the path is located in a file, this parameter can be used to
- direct the user to the file where the path is located.
+ Parameters
+ ----------
+ path: str
+ Path to be checked.
+ filename: str
+ If the path is located in a file, this parameter can be used to
+ direct the user to the file where the path is located.
"""
if r"\\" not in path and "/" in path:
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 5318621687051c1af04c395681aa40ef3ac85b5c..30299a59638e23d1645c85144e1bcbb7663a3051 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -23,13 +23,13 @@ import sphinx_rtd_theme
# -- Project information -----------------------------------------------------
project = 'caosadvancedtools'
-copyright = '2021, IndiScale GmbH'
+copyright = '2023, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.5.1'
+version = '0.10.1'
# The full version, including alpha/beta/rc tags
-release = '0.5.1-dev'
+release = '0.10.1-dev'
# -- General configuration ---------------------------------------------------
@@ -64,7 +64,7 @@ master_doc = 'index'
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
@@ -74,6 +74,7 @@ exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
+default_role = "py:obj"
# -- Options for HTML output -------------------------------------------------
@@ -190,7 +191,7 @@ epub_exclude_files = ['search.html']
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
"python": ("https://docs.python.org/", None),
- "caosdb-pylib": ("https://caosdb.gitlab.io/caosdb-pylib/", None),
+ "linkahead-pylib": ("https://docs.indiscale.com/caosdb-pylib/", None),
}
@@ -199,3 +200,7 @@ autodoc_default_options = {
'members': None,
'undoc-members': None,
}
+autodoc_mock_imports = [
+ "caosadvancedtools.bloxberg",
+ "labfolder",
+]
diff --git a/src/doc/crawler.rst b/src/doc/crawler.rst
index aea023192c0b95c784d5ac91ade12d0c30591d42..d7b351bb3132cefed8920f977b345dc32e4db819 100644
--- a/src/doc/crawler.rst
+++ b/src/doc/crawler.rst
@@ -77,8 +77,8 @@ problems. The exact behavior depends on your setup. However, you can
have a look at the example in the
`tests <https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/main/integrationtests/crawl.py>`__.
-.. Note:: The crawler depends on the CaosDB Python client, so make sure to install :doc:`pycaosdb
- <caosdb-pylib:getting_started>`.
+.. Note:: The crawler depends on the LinkAhead Python client, so make sure to install :doc:`pylinkahead
+ <linkahead-pylib:README_SETUP>`.
Call ``python3 crawl.py --help`` to see what parameters can be provided.
diff --git a/src/doc/index.rst b/src/doc/index.rst
index 5fdb78da4eddfd0145d0357202246d4b5352dcf4..7fa017ec4202f25fe9f94a154ed8762c4581eebc 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -4,8 +4,8 @@ Welcome to caosadvancedtools' documentation!
Welcome to the advanced Python tools for CaosDB!
-This documentation helps you to :doc:`get started<getting_started>`, explains the most important
-:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`.
+This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of deep dives into specific sub-topics.
.. toctree::
:maxdepth: 2
@@ -15,7 +15,12 @@ This documentation helps you to :doc:`get started<getting_started>`, explains th
Concepts <concepts>
The Caosdb Crawler <crawler>
YAML data model specification <yaml_interface>
+ Specifying a datamodel with JSON schema <json_schema_interface>
+ Convert a data model into a json schema <json_schema_exporter>
+ Conversion between XLSX, JSON and LinkAhead Entities <table-json-conversion/specs>
_apidoc/modules
+ Related Projects <related_projects/index>
+ Back to overview <https://docs.indiscale.com/>
diff --git a/src/doc/json_schema_exporter.rst b/src/doc/json_schema_exporter.rst
new file mode 100644
index 0000000000000000000000000000000000000000..da4fa9d273f7cb9295974e9632e8eb1730bd47a3
--- /dev/null
+++ b/src/doc/json_schema_exporter.rst
@@ -0,0 +1,8 @@
+JSON schema from data model
+===========================
+
+Sometimes you may want to have a `json schema <https://json-schema.org>`_ which describes a
+LinkAhead data model, for example for the automatic generation of user interfaces with third-party
+tools like `rjsf <https://rjsf-team.github.io/react-jsonschema-form/docs/>`_.
+
+For this use case, look at the documentation of the `caosadvancedtools.json_schema_exporter` module.
diff --git a/src/doc/json_schema_interface.rst b/src/doc/json_schema_interface.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0e8aebd3a4204f29608212f7ed0c115fd1d4a134
--- /dev/null
+++ b/src/doc/json_schema_interface.rst
@@ -0,0 +1,75 @@
+Defining datamodels with a JSON schema specification
+====================================================
+
+TODO, see https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/42
+
+Further information
+###################
+
+Pattern Properties
+%%%%%%%%%%%%%%%%%%
+
+The JSON-schema parser has rudimentary support for ``patternProperties``. Since
+their names (only the pattern that their names will suffice) are not known a
+priori, we create RecordTypes for all pattern properties. The names of these
+RecordTypes are created from their parent element's name by appending the string
+``"Entry"`` and possibly a number if there are more than one pattern properties
+for one parent.
+
+All the RecordTypes created for pattern properties have at least an obligatory
+``__matched_pattern`` property which will -- as the name suggests -- store the
+matched pattern of an actual data entry.
+
+.. note::
+
+ The ``__matched_pattern`` property is added automatically to your datamodel
+ as soon as there is at least one pattern property in your JSON schema. So be
+ sure that you don't happen to have an entity with exactly this name in your
+ database.
+
+E.g., a json schema with
+
+.. code-block:: json
+
+ "dataset": {
+ "patternProperties": {
+ "^[0-9]{4,4}": {
+ "type": "boolean"
+ },
+ "^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}": {
+ "type": "object",
+ "properties": {
+ "date_id": {
+ "$ref": "#/definitions/uuid"
+ }
+ }
+ }
+ }
+ }
+
+Would result in a ``Dataset`` RecordType that has the two properties
+``DatasetEntry_1`` and ``DatasetEntry_2`` (as always, name can be overwritten
+explicitly by specifying the ``title`` property), referencing corresponding
+``DatasetEntry_1`` and ``DatasetEntry_2`` Records.
+
+Apart from the aforementioned ``__matched_pattern`` property, ``DatasetEntry_1``
+also has the ``DatasetEntry_1_value`` property with datatype ``BOOLEAN``, that
+stores the actual value. In turn, ``DatasetEntry_2`` is of ``type: object`` and
+is treated like any other RecordType. Consequently, it has, appart from the
+``__matched_pattern`` property, a ``date_id`` property as specified in its
+``properties``.
+
+Array entries without ``items`` specification
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+JSON schema allows for properties of ``type: array`` without the ``items``
+specification that consequently can be arrays of any (and of mixed) types. While
+this is in general problematic when specifying a data model, sometimes these
+properties cannot be specified further, e.g., when you're using an external
+schema that you cannot change.
+
+These properties can still be added to your datamodel by specifying their types
+explicitly in a dictionary or, alternatively, they can be ignored. See the
+``types_for_missing_array_items`` and ``ignore_unspecified_array_items``
+parameters of ``models.parser.JsonSchemaParser``, respectively, for more
+information.
diff --git a/src/doc/related_projects/index.rst b/src/doc/related_projects/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..87808b4264dfdd37dfa162025b24d31e63de7083
--- /dev/null
+++ b/src/doc/related_projects/index.rst
@@ -0,0 +1,25 @@
+Related Projects
+++++++++++++++++
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+
+.. container:: projects
+
+ For in-depth documentation for users, administrators and developers, you may want to visit the subproject-specific documentation pages for:
+
+ :`Server <https://docs.indiscale.com/caosdb-server/>`_: The Java part of the LinkAhead server.
+
+ :`MySQL backend <https://docs.indiscale.com/caosdb-mysqlbackend/>`_: The MySQL/MariaDB components of the LinkAhead server.
+
+ :`WebUI <https://docs.indiscale.com/caosdb-webui/>`_: The default web frontend for the LinkAhead server.
+
+ :`PyLinkAhead <https://docs.indiscale.com/caosdb-pylib/>`_: The LinkAhead Python library.
+
+ :`LinkAhead Crawler <https://docs.indiscale.com/caosdb-crawler/>`_: The crawler is the main tool for automatic data integration in LinkAhead.
+
+ :`LinkAhead <https://docs.indiscale.com/caosdb-deploy>`_: Your all inclusive LinkAhead software package.
+
+ :`Back to Overview <https://docs.indiscale.com/>`_: LinkAhead Documentation.
diff --git a/src/doc/table-json-conversion/specs.rst b/src/doc/table-json-conversion/specs.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c98eddc1180f552f1d2389b1bb57979e93550ab8
--- /dev/null
+++ b/src/doc/table-json-conversion/specs.rst
@@ -0,0 +1,527 @@
+Conversion between LinkAhead data models, JSON schema, and XLSX (and vice versa)
+================================================================================
+
+This file describes the conversion between JSON schema files and XLSX
+templates, and between JSON data files following a given schema and XLSX
+files with data. This conversion is handled by the Python modules in the
+``table_json_conversion`` library.
+
+Data models in JSON Schema and JSON data
+----------------------------------------
+
+Let’s start simple! If you would describe a ``Person`` Record with the
+Properties ``family_name`` and ``given_name`` in JSON, it would probably
+look like this:
+
+.. code:: json
+
+ {
+ "Person":
+ {
+ "family_name": "Steve",
+ "given_name": "Stevie"
+ }
+ }
+
+The data model in LinkAhead defines the types of records present in a
+LinkAhead instance and their structure. This data model can also be
+represented in a JSON Schema, which defines the structure of JSON files
+containing records pertaining to the data model.
+
+You can define this kind of structure with the following JSON schema:
+
+.. code:: json
+
+ {
+ "type": "object",
+ "properties": {
+ "Person": {
+ "type": "object",
+ "properties": {
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+
+The above schema (and schemas created by
+``json_schema_exporter.merge_schemas(...)``) is, from a broad view, a
+dict with all the top level recordtypes (the recordtype names are the
+keys). This is sufficient to describe the data model. However, actual
+data often consists of multiple entries of the same type (e.g. multiple
+Persons).
+
+Since the data model schema does not match multiple data sets, there is
+a utility function which creates a *data array* schema out of the *data
+model* schema: It basically replaces the top-level entries of the data
+model by lists which may contain data.
+
+For example, the following JSON describes two “Person” Records:
+
+.. code:: json
+
+ {
+ "Person": [
+ {
+ "family_name": "Steve",
+ "given_name": "Stevie"
+ },
+ {
+ "family_name": "Star",
+ "given_name": "Stella"
+ }
+ ]
+ }
+
+The *JSON Schema* for a JSON like the above one could look like the
+following:
+
+.. code:: json
+
+ {
+ "type": "object",
+ "properties": {
+ "Person": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+
+This would define that the top level object/dict may have a key
+``Person`` which has as value an array of objects that in turn have the
+properties ``family_name`` and ``given_name``.
+
+You can create a data array schema from a data model schema using
+``xlsx_utils.array_schema_from_model_schema``.
+
+From JSON to XLSX: Data Representation
+--------------------------------------
+
+The following describes how JSON files representing LinkAhead records
+are converted into XLSX files, or how JSON files with records are
+created from XLSX files.
+
+The attribute name (e.g., “Person” above) determines the RecordType, and
+the value of this attribute can either be an object or a list. If it is
+an object (as in the example above), a single record is represented. In
+the case of a list, multiple records sharing the same RecordType as the
+parent are represented.
+
+The *Properties* of the record (e.g., ``family_name`` and ``given_name``
+above) become *columns* in the XLSX file. Thus the XLSX file created
+from the above example would have a sheet “Person” with the following
+table:
+
+========== ===========
+given_name family_name
+========== ===========
+Stevie Steve
+Stella Star
+========== ===========
+
+The properties of objects (Records) in the JSON have an attribute name
+and a value. The value can be:
+
+a. A primitive (text, number, boolean, …)
+b. A record
+c. A list of primitive types
+d. A list of unique enums (multiple choice)
+e. A list of records
+
+In cases *a.* and *c.*, a cell is created in the column corresponding to
+the property in the XLSX file. In case *b.*, columns are created for the
+Properties of the record, where for each of the Properties the cases
+*a.* - *e.* are considered recursively. Case *d.* leads to a number of
+columns, one for each of the possible choices.
+
+For case *e.* however, the two-dimensional structure of an XLSX sheet is
+not sufficient. Therefore, for such cases, *new* XLSX sheets/tables are
+created.
+
+In these sheets/tables, the referenced records are treated as described
+above (new columns for the Properties). However, there are now
+additional columns that indicate from which “external” record these
+records are referenced.
+
+Let’s now consider these five cases in detail and with examples:
+
+a. Properties with primitive data types
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: json
+
+ {
+ "Training": [
+ {
+ "date": "2023-01-01",
+ "url": "www.indiscale.com",
+ "duration": 1.0,
+ "participants": 1,
+ "remote": false
+ },
+ {
+ "date": "2023-06-15",
+ "url": "www.indiscale.com/next",
+ "duration": 2.5,
+ "participants": None,
+ "remote": true
+ }
+ ]
+ }
+
+This entry will be represented in an XLSX sheet with the following
+content:
+
++------------+------------------------+----------+--------------+--------+
+| date | url | duration | participants | remote |
++============+========================+==========+==============+========+
+| 2023-01-01 | www.indiscale.com | 1.0 | 1 | false |
++------------+------------------------+----------+--------------+--------+
+| 2023-06-15 | www.indiscale.com/next | 2.5 | | true |
++------------+------------------------+----------+--------------+--------+
+
+b. Property referencing a record
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: json
+
+ {
+ "Training": [
+ {
+ "date": "2023-01-01",
+ "supervisor": {
+ "family_name": "Stevenson",
+ "given_name": "Stevie"
+ }
+ }
+ ]
+ }
+
+This entry will be represented in an XLSX sheet named "Training" with the following
+content:
+
+========== ========================== =========================
+date supervisor.family_name supervisor.given_name
+========== ========================== =========================
+2023-01-01 Stevenson Stevie
+========== ========================== =========================
+
+
+c. Properties containing lists of primitive data types
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: json
+
+ {
+ "Training": [
+ {
+ "url": "www.indiscale.com",
+ "subjects": ["Math", "Physics"],
+ }
+ ]
+ }
+
+This entry would be represented in an XLSX sheet with the following
+content:
+
+================= ============
+url subjects
+================= ============
+www.indiscale.com Math;Physics
+================= ============
+
+The list elements are written into the cell separated by ``;``
+(semicolon). If the elements contain the separator ``;``, it is escaped
+with ``\``.
+
+d. Multiple choice properties
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: json
+
+ {
+ "Training": [
+ {
+ "date": "2024-04-17",
+ "skills": [
+ "Planning",
+ "Evaluation"
+ ]
+ }
+ ]
+ }
+
+If the ``skills`` list is denoted as an ``enum`` array with
+``"uniqueItems": true`` in the json schema, this entry would be
+represented like this in an XLSX:
+
++------------+-----------------+----------------------+-------------------+
+| date | skills.Planning | skills.Communication | skills.Evaluation |
++============+=================+======================+===================+
+| 2024-04-17 | x | | x |
++------------+-----------------+----------------------+-------------------+
+
+Note that this example assumes that the list of possible choices, as
+given in the json schema, was “Planning, Communication, Evaluation”.
+
+e. Properties containing lists with references
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: json
+
+ {
+ "Training": [
+ {
+ "date": "2023-01-01",
+ "coach": [
+ {
+ "family_name": "Sky",
+ "given_name": "Max",
+ },
+ {
+ "family_name": "Sky",
+ "given_name": "Min",
+ }
+ ]
+ }
+ ]
+ }
+
+Since the two coaches cannot be represented properly in a single cell,
+another worksheet is needed to contain the properties of the coaches.
+
+The sheet for the Trainings in this example only contains the “date”
+column
+
++------------+
+| date |
++============+
+| 2023-01-01 |
++------------+
+
+Additionally, there is *another* sheet where the coaches are stored.
+Here, it is crucial to define how the correct element is chosen from
+potentially multiple “Trainings”. In this case, it means that the “date”
+must be unique.
+
+
+The second sheet looks like this:
+
+========== ===================== ====================
+date ``coach.family_name`` ``coach.given_name``
+========== ===================== ====================
+2023-01-01 Sky Max
+2023-01-01 Sky Min
+========== ===================== ====================
+
+Note: This uniqueness requirement is not strictly checked right now, it
+is your responsibility as a user that such “foreign properties” are
+truly unique.
+
+When converting JSON files that contain Records that were exported from LinkAhead
+it might be a good idea to use the LinkAhead ID as a unique identifier for Records. However, if
+your Records do not yet have LinkAhead IDs you need to find some other identifying
+properties/foreign keys. Note, that those properties only need to identify a Record uniquely within
+the list of Records: In the above example the "coach" Record needs to be identified in the list of
+coaches.
+
+
+Data in XLSX: Hidden automation logic
+-------------------------------------
+
+First column: Marker for row types
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first column in each sheet will be hidden and it will contain an
+entry in each row that needs special treatment. The following values are
+used:
+
+- ``IGNORE``: This row is ignored. It can be used for explanatory texts
+ or layout.
+- ``COL_TYPE``: Typically the first row that is not ``IGNORE``. It
+ indicates the row that defines the type of columns (``FOREIGN``,
+ ``SCALAR``, ``LIST``, ``MULTIPLE_CHOICE``, ``IGNORE``). This row must
+ occur exactly once per sheet.
+- ``PATH``: Indicates that the row is used to define the path within
+ the JSON. These rows are typically hidden for users.
+
+An example table could look like this:
+
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| IGNORE | | Welcome | to | this | file |
++==========+=====================================+================+===============+==============+=====================+
+| IGNORE | | Please | enter your | data here: | |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| COL_TYPE | IGNORE | SCALAR | SCALAR | LIST | SCALAR |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| PATH | | Training | Training | Training | Training |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| PATH | | url | date | subjects | supervisor |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| PATH | | | | | email |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| IGNORE | Please enter one training per line. | Training URL | Training date | Subjects | Supervisor's email |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| | | example.com/mp | 2024-02-27 | Math;Physics | steve@example.com |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+| | | example.com/m | 2024-02-28 | Math | stella@example.com |
++----------+-------------------------------------+----------------+---------------+--------------+---------------------+
+
+
+Parsing XLSX data
+~~~~~~~~~~~~~~~~~
+
+To extract the value of a given cell, we traverse all path elements (in
+``PATH`` rows) from top to bottom. The final element of the path is the
+name of the Property to which the value belongs. In the example above,
+``steve@example.com`` is the value of the ``email`` Property in the path
+``["Training", "supervisor", "email"]``.
+
+The path elements are sufficient to identify the object within a JSON,
+at least if the corresponding JSON element is a single object. If the
+JSON element is an array, the appropriate object within the array needs
+to be selected.
+
+For this selection additional ``FOREIGN`` columns are used. The paths in
+these columns must all have the same *base* and one additional *unique
+key* component. For example, two ``FOREIGN`` columns could be
+``["Training", "date"]`` and ``["Training", "url"]``, where
+``["Training"]`` is the *base path* and ``"date"`` and ``"url"`` are the
+*unique keys*.
+
+The base path defines the table (or recordtype) to which the entries
+belong, and the values of the unique keys define the actual rows to
+which data belongs.
+
+For example, this table defines three coaches for the two trainings from
+the last table:
+
++----------+-----------------------+-----------------------+------------------------+
+| COL_TYPE | FOREIGN | FOREIGN | SCALAR |
++----------+-----------------------+-----------------------+------------------------+
+| PATH | Training | Training | Training |
++----------+-----------------------+-----------------------+------------------------+
+| PATH | date | url | coach |
++----------+-----------------------+-----------------------+------------------------+
+| PATH | | | given_name |
++----------+-----------------------+-----------------------+------------------------+
+| IGNORE | Date of training | URL of training | The coach’s given name |
++----------+-----------------------+-----------------------+------------------------+
+| IGNORE | from sheet ‘Training’ | from sheet ‘Training’ | |
++----------+-----------------------+-----------------------+------------------------+
+| | 2024-02-27 | example.com/mp | Ada |
++----------+-----------------------+-----------------------+------------------------+
+| | 2024-02-27 | example.com/mp | Berta |
++----------+-----------------------+-----------------------+------------------------+
+| | 2024-02-28 | example.com/m | Chris |
++----------+-----------------------+-----------------------+------------------------+
+
+Sepcial case: multiple choice “checkboxes”
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As a special case, enum arrays with ``"uniqueItems": true`` can be
+represented as multiple columns, with one column per choice. The choices
+are denoted as the last PATH component, the column type must be
+MULTIPLE_CHOICE.
+
+Stored data is denoted as an “x” character in the respective cell, empty
+cells denote that the item was not selected. Additionally, the
+implementation also allows TRUE or 1 for selected items, and
+FALSE, 0 or cells with only whitespace characters for deselected
+items:
+
++----------+-----------------+----------------------+-------------------+
+| COL_TYPE | MULTIPLE_CHOICE | MULTIPLE_CHOICE | MULTIPLE_CHOICE |
++----------+-----------------+----------------------+-------------------+
+| PATH | skills | skills | skills |
++----------+-----------------+----------------------+-------------------+
+| PATH | Planning | Communication | Evaluation |
++----------+-----------------+----------------------+-------------------+
+| IGNORE | skills.Planning | skills.Communication | skills.Evaluation |
++----------+-----------------+----------------------+-------------------+
+| | x | | X |
++----------+-----------------+----------------------+-------------------+
+| | " " | TRUE | FALSE |
++----------+-----------------+----------------------+-------------------+
+| | 0 | x | 1 |
++----------+-----------------+----------------------+-------------------+
+
+These rows correspond to:
+
+1. Planning, Evaluation
+2. Communication
+3. Communication, Evaluation
+
+
+User Interaction
+----------------
+The primary and most straight forward use case of this utility is to export
+LinkAhead data as JSON and then as XLSX tables. This can be done fully
+automatic.
+
+TODO show how!
+
+The hidden cells for automation are designed such that the XLSX template that
+is created can be customized such that it is a nicely formatted table. The
+hidden content must remain. See below for tips on how to manipulate the table.
+
+The second use case is to use XLSX to collect data and then import it into
+LinkAhead. Here, it may be necessary to define foreign keys in order to
+identify Records in lists.
+
+Table Manipulation
+~~~~~~~~~~~~~~~~~~
+
+- All formatting is ignored
+- Nothing has to be observed when adding new data rows
+- When adding new descriptory rows (for example one for descriptions of the
+ columns), the ``COL_TYPE`` must be set to ``IGNORE``
+- You can freely rename sheets.
+- You can freely rename columns (since the row containing the column names is
+ set to ``IGNROE``; the Property name is taken from the last path element)
+- You can change the order of columns. However, you have to make sure to move
+ the full column including hidden elements. Thus you should not select a range
+ of cells, but click on the column index in your spread sheet program.
+
+Note: Requirements
+------------------
+
+This conversion does not allow arbitrary JSON schema files nor does it
+support arbitrary JSON files since conversion to XLSX files would not
+make sense. Instead, this utility is tailored to supported conversion of
+data (models) that are structured like data (models) in LinkAhead:
+
+- The JSON schema describes a data model of RecordTypes and Properties as it would be generated by the caosadvancedtools.json_schema_exporter module.
+- The JSON files must contain arrays of Records complying with such a data model.
+
+Thus, when converting from a JSON schema, the top level of the JSON
+schema must be a dict. The keys of the dict are RecordType names.
+
+
+
+
+Current limitations
+-------------------
+
+The current implementation still lacks the following:
+
+- Files handling is not implemented yet.
+
diff --git a/src/doc/yaml_interface.rst b/src/doc/yaml_interface.rst
index 84fef21974c7fbb9e094ab34a2f7f87bbbb4759c..6d2ec17867bd2cea408059b93375624c5680ffe5 100644
--- a/src/doc/yaml_interface.rst
+++ b/src/doc/yaml_interface.rst
@@ -19,10 +19,10 @@ in the library sources.
Person:
recommended_properties:
firstName:
- datatype: TEXT
+ datatype: TEXT
description: 'first name'
lastName:
- datatype: TEXT
+ datatype: TEXT
description: 'last name'
LabbookEntry:
recommended_properties:
@@ -52,7 +52,7 @@ This example defines 3 ``RecordTypes``:
- A ``Project`` with one obligatory property ``datatype``
- A Person with a ``firstName`` and a ``lastName`` (as recommended properties)
- A ``LabbookEntry`` with multiple recommended properties of different data types
-- It is assumed that the server knows a RecordType or Property with the name
+- It is assumed that the server knows a RecordType or Property with the name
``Textfile``.
@@ -69,9 +69,22 @@ Note the difference between the three property declarations of ``LabbookEntry``:
- ``responsible``: This defines and adds a property with name "responsible" to ``LabbookEntry`, which has a datatype ``Person``. ``Person`` is defined above.
- ``firstName``: This defines and adds a property with the standard data type ``TEXT`` to record type ``Person``.
-If the data model depends on record types or properties which already exist in CaosDB, those can be
-added using the ``extern`` keyword: ``extern`` takes a list of previously defined names.
+If the data model depends on record types or properties which already exist in
+LinkAhead, those can be added using the ``extern`` keyword: ``extern`` takes a
+list of previously defined names of Properties and/or RecordTypes. Note that if you happen to use an already existing ``REFERENCE`` property that has an already existing RecordType as datatype, you also need to add that RecordType's name to the ``extern`` list, e.g.,
+
+.. code-block:: yaml
+
+ extern:
+ # Let's assume the following is a reference property with datatype Person
+ - Author
+ # We need Person (since it's the datatype of Author) even though we might
+ # not use it explicitly
+ - Person
+ Dataset:
+ recommended_properties:
+ Author:
Reusing Properties
==================
@@ -143,7 +156,6 @@ Keywords
added as parents, and all Properties with at least the importance ``XXX`` are inherited. For
example, ``inherited_from_recommended`` will inherit all Properties of importance ``recommended``
and ``obligatory``, but not ``suggested``.
-- **parent**: Parent of this entity. Same as ``inherit_from_obligatory``. (*Deprecated*)
Usage
=====
@@ -152,18 +164,25 @@ You can use the yaml parser directly in python as follows:
.. code-block:: python
-
+
from caosadvancedtools.models import parser as parser
model = parser.parse_model_from_yaml("model.yml")
This creates a DataModel object containing all entities defined in the yaml file.
-You can then use the functions from caosadvancedtools.models.data_model.DataModel to synchronize
+If the parsed data model shall be appended to a pre-exsting data model, the optional
+``extisting_model`` can be used:
+
+.. code-block:: python
+
+ new_model = parser.parse_model_from_yaml("model.yml", existing_model=old_model)
+
+You can now use the functions from ``DataModel`` to synchronize
the model with a CaosDB instance, e.g.:
.. code-block:: python
-
+
model.sync_data_model()
.. LocalWords: yml projectId UID firstName lastName LabbookEntry entryId textElement labbook
diff --git a/tox.ini b/tox.ini
index ddfb4714f6b86d93e9567fe3c1410f2a1ce965cf..a7e06bf51f1f4cad2a2c695e44d3a4d09020b2a3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,7 @@
[tox]
-envlist=py36, py37, py38, py39, py310
+envlist=py38, py39, py310, py311, py312, py313
skip_missing_interpreters = true
+
[testenv]
deps=nose
pandas
@@ -8,10 +9,15 @@ deps=nose
pytest
pytest-cov
gitignore-parser
- openpyxl
+ openpyxl >= 3.0.7
xlrd == 1.2
h5py
-commands=py.test --cov=caosadvancedtools -vv {posargs}
+commands=py.test --cov=caosadvancedtools --cov-report=html:.tox/cov_html -vv {posargs}
[flake8]
max-line-length=100
+
+[pytest]
+testpaths = unittests
+addopts = -vv
+xfail_strict = True
diff --git a/unittests/json-schema-models/datamodel_atomic_properties.schema.json b/unittests/json-schema-models/datamodel_atomic_properties.schema.json
index 3828f131180a839d5c9b8bc5aa1a1285717da723..7b4a23e5bb48b995d07a261bcae0a8a486b7969a 100644
--- a/unittests/json-schema-models/datamodel_atomic_properties.schema.json
+++ b/unittests/json-schema-models/datamodel_atomic_properties.schema.json
@@ -18,7 +18,8 @@
"date": { "type": "string", "format": "date" },
"integer": { "type": "integer", "description": "Some integer property" },
"boolean": { "type": "boolean" },
- "number_prop": { "type": "number", "description": "Some float property" }
+ "number_prop": { "type": "number", "description": "Some float property" },
+ "null_prop": { "type": "null", "description": "This property will never have a value." }
}
}
]
diff --git a/unittests/json-schema-models/datamodel_missing_array_items.schema.json b/unittests/json-schema-models/datamodel_missing_array_items.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..8ac17ac3162def3dbf070d7027fd318366bb4682
--- /dev/null
+++ b/unittests/json-schema-models/datamodel_missing_array_items.schema.json
@@ -0,0 +1,9 @@
+{
+ "title": "something_with_missing_array_items",
+ "type": "object",
+ "properties": {
+ "missing": {
+ "type": "array"
+ }
+ }
+}
diff --git a/unittests/json-schema-models/datamodel_no_toplevel_entity.schema.json b/unittests/json-schema-models/datamodel_no_toplevel_entity.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..35240d765479b719576e6ee67e387790d3d6d160
--- /dev/null
+++ b/unittests/json-schema-models/datamodel_no_toplevel_entity.schema.json
@@ -0,0 +1,56 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://my-schema-id.net",
+ "type": "object",
+ "definitions": {
+ "uuid": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+ },
+ "datetime": {
+ "type": "string",
+ "format": "date-time"
+ }
+ },
+ "properties": {
+ "Dataset1": {
+ "title": "Dataset1",
+ "description": "Some description",
+ "type": "object",
+ "properties": {
+ "title": {
+ "type": "string",
+ "description": "full dataset title"
+ },
+ "campaign": {
+ "type": "string",
+ "description": "FIXME"
+ },
+ "number_prop": {
+ "type": "number",
+ "description": "Some float property"
+ },
+ "user_id": {
+ "$ref": "#/definitions/uuid"
+ }
+ },
+ "required": ["title", "number_prop"]
+ }
+ },
+ "patternProperties": {
+ "^[0-9]{4,4}": {
+ "type": "boolean"
+ },
+ "^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}": {
+ "type": "object",
+ "properties": {
+ "date_id": {
+ "$ref": "#/definitions/uuid"
+ }
+ }
+ }
+ }
+}
diff --git a/unittests/json-schema-models/datamodel_pattern_properties.schema.json b/unittests/json-schema-models/datamodel_pattern_properties.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..9b85c7b80cf0990713f8f130050c21751e311b42
--- /dev/null
+++ b/unittests/json-schema-models/datamodel_pattern_properties.schema.json
@@ -0,0 +1,39 @@
+[
+ {
+ "title": "Dataset",
+ "type": "object",
+ "patternProperties": {
+ "^[0-9]{4,4}": {
+ "type": "boolean"
+ },
+ "^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}": {
+ "type": "object",
+ "properties": {
+ "date_id": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+ }
+ }
+ }
+ }
+ },
+ {
+ "title": "Dataset2",
+ "type": "object",
+ "properties": {
+ "datetime": {
+ "type": "string",
+ "format": "date-time"
+ }
+ },
+ "patternProperties": {
+ ".*": {
+ "title": "Literally anything",
+ "type": "object"
+ }
+ }
+ }
+]
diff --git a/unittests/model.yml b/unittests/models/model.yml
similarity index 100%
rename from unittests/model.yml
rename to unittests/models/model.yml
diff --git a/unittests/models/model_invalid.yml b/unittests/models/model_invalid.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c8368b9701db9b3461b7e0f1f3514c2411f56b56
--- /dev/null
+++ b/unittests/models/model_invalid.yml
@@ -0,0 +1,2 @@
+Project:
+ ObligatoryProperties:
diff --git a/unittests/table_json_conversion/__init__.py b/unittests/table_json_conversion/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/unittests/table_json_conversion/create_jsonschema.py b/unittests/table_json_conversion/create_jsonschema.py
new file mode 100755
index 0000000000000000000000000000000000000000..8ab4ad2d973b78522e858b3ee866b870ecf187a4
--- /dev/null
+++ b/unittests/table_json_conversion/create_jsonschema.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2023 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2023 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Create JSON-Schema according to configuration.
+
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+
+import caosadvancedtools.json_schema_exporter as jsex
+from caosadvancedtools.models import parser
+# import tomli
+
+
+def prepare_datamodel(modelfile, recordtypes: list[str], outfile: str,
+ do_not_create: list[str] = None):
+ if do_not_create is None:
+ do_not_create = []
+ model = parser.parse_model_from_yaml(modelfile)
+
+ exporter = jsex.JsonSchemaExporter(additional_properties=False,
+ # additional_options_for_text_props=additional_text_options,
+ # name_and_description_in_properties=True,
+ name_property_for_new_records=True,
+ do_not_create=do_not_create,
+ # do_not_retrieve=do_not_retrieve,
+ no_remote=True,
+ use_rt_pool=model,
+ )
+ schemas = []
+ for recordtype in recordtypes:
+ schemas.append(exporter.recordtype_to_json_schema(model.get_deep(recordtype)))
+ merged_schema = jsex.merge_schemas(schemas)
+
+ with open(outfile, mode="w", encoding="utf8") as json_file:
+ json.dump(merged_schema, json_file, ensure_ascii=False, indent=2)
+
+
+def _parse_arguments():
+ """Parse the arguments."""
+ arg_parser = argparse.ArgumentParser(description='')
+
+ return arg_parser.parse_args()
+
+
+def main():
+ """The main function of this script."""
+ _ = _parse_arguments()
+ prepare_datamodel("data/simple_model.yml", ["Training", "Person"], "data/simple_schema.json",
+ do_not_create=["Organisation"])
+ prepare_datamodel("data/multiple_refs_model.yml", ["Training", "Person"],
+ "data/multiple_refs_schema.json")
+ prepare_datamodel("data/indirect_model.yml", ["Wrapper"],
+ "data/indirect_schema.json")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/unittests/table_json_conversion/data/error_simple_data.json b/unittests/table_json_conversion/data/error_simple_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..4d57b0335b4685ea82f1668d50b52a9d30ef1759
--- /dev/null
+++ b/unittests/table_json_conversion/data/error_simple_data.json
@@ -0,0 +1,11 @@
+{
+ "Training": [{
+ "duration": 1.0,
+ "participants": 0.5
+ }],
+ "Person": [{
+ "family_name": "Auric",
+ "given_name": "Goldfinger",
+ "Organisation": "Federal Reserve"
+ }]
+}
diff --git a/unittests/table_json_conversion/data/indirect_data.json b/unittests/table_json_conversion/data/indirect_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..76db75d97e1dafff223ea2b27ecca1086d6bc4af
--- /dev/null
+++ b/unittests/table_json_conversion/data/indirect_data.json
@@ -0,0 +1,18 @@
+{
+ "Wrapper": [{
+ "Results": [
+ {
+ "year": 2022,
+ "avg_score": 2.4
+ },
+ {
+ "year": 2023,
+ "avg_score": 4.2
+ }
+ ],
+ "Training": {
+ "name": "Basic Training",
+ "url": "www.example.com/training/basic"
+ }
+ }]
+}
diff --git a/unittests/table_json_conversion/data/indirect_data.xlsx b/unittests/table_json_conversion/data/indirect_data.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..3d0cf3245a414a4161b99034051424c699b5d453
Binary files /dev/null and b/unittests/table_json_conversion/data/indirect_data.xlsx differ
diff --git a/unittests/table_json_conversion/data/indirect_model.yml b/unittests/table_json_conversion/data/indirect_model.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2a7f4f98ff9a46478eb631e6990deceadc9a498c
--- /dev/null
+++ b/unittests/table_json_conversion/data/indirect_model.yml
@@ -0,0 +1,18 @@
+Training:
+ recommended_properties:
+ url:
+ datatype: TEXT
+ description: 'The URL'
+Results:
+ description: "Results for a training"
+ recommended_properties:
+ year:
+ datatype: INTEGER
+ avg_score:
+ description: The average score for the linked training.
+ datatype: DOUBLE
+Wrapper:
+ recommended_properties:
+ Training:
+ Results:
+ datatype: LIST<Results>
diff --git a/unittests/table_json_conversion/data/indirect_schema.json b/unittests/table_json_conversion/data/indirect_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..64b6ff279c584456fe1f90454225d144c90e014d
--- /dev/null
+++ b/unittests/table_json_conversion/data/indirect_schema.json
@@ -0,0 +1,63 @@
+{
+ "type": "object",
+ "properties": {
+ "Wrapper": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Wrapper",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "Training": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Training",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "url": {
+ "type": "string",
+ "description": "The URL"
+ }
+ }
+ },
+ "Results": {
+ "description": "Results for a training",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "description": "Results for a training",
+ "title": "Results",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "year": {
+ "type": "integer"
+ },
+ "avg_score": {
+ "description": "The average score for the linked training.",
+ "type": "number"
+ }
+ }
+ }
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+ },
+ "required": [
+ "Wrapper"
+ ],
+ "additionalProperties": false,
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}
diff --git a/unittests/table_json_conversion/data/indirect_template.xlsx b/unittests/table_json_conversion/data/indirect_template.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..0c521e554027f565ecae6fe27783034361b8ff41
Binary files /dev/null and b/unittests/table_json_conversion/data/indirect_template.xlsx differ
diff --git a/unittests/table_json_conversion/data/multiple_choice_data.json b/unittests/table_json_conversion/data/multiple_choice_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee24ef7adbd61abf22d47bb3d49f43f3e1e26501
--- /dev/null
+++ b/unittests/table_json_conversion/data/multiple_choice_data.json
@@ -0,0 +1,11 @@
+{
+ "Training": [{
+ "name": "Super Skill Training",
+ "date": "2024-04-17",
+ "skills": [
+ "Planning",
+ "Evaluation"
+ ],
+ "exam_types": []
+ }]
+}
diff --git a/unittests/table_json_conversion/data/multiple_choice_data.xlsx b/unittests/table_json_conversion/data/multiple_choice_data.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..28cf4007d8a1a061235863d12e5bdc5b5747f386
Binary files /dev/null and b/unittests/table_json_conversion/data/multiple_choice_data.xlsx differ
diff --git a/unittests/table_json_conversion/data/multiple_choice_data_missing.xlsx b/unittests/table_json_conversion/data/multiple_choice_data_missing.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..10d76529ab11d2e79ca0651313dd50f0cc326341
Binary files /dev/null and b/unittests/table_json_conversion/data/multiple_choice_data_missing.xlsx differ
diff --git a/unittests/table_json_conversion/data/multiple_choice_schema.json b/unittests/table_json_conversion/data/multiple_choice_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..71bf0379aba4ad6f8510581ba0defadb81a66609
--- /dev/null
+++ b/unittests/table_json_conversion/data/multiple_choice_schema.json
@@ -0,0 +1,57 @@
+{
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Training",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "date": {
+ "description": "The date of the training.",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ },
+ "skills": {
+ "description": "Skills that are trained.",
+ "type": "array",
+ "items": {
+ "enum": [
+ "Planning",
+ "Communication",
+ "Evaluation"
+ ]
+ },
+ "uniqueItems": true
+ },
+ "exam_types": {
+ "type": "array",
+ "items": {
+ "enum": [
+ "Oral",
+ "Written"
+ ]
+ },
+ "uniqueItems": true
+ }
+ }
+ }
+ },
+ "required": [
+ "Training"
+ ],
+ "additionalProperties": false,
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}
diff --git a/unittests/table_json_conversion/data/multiple_choice_template.xlsx b/unittests/table_json_conversion/data/multiple_choice_template.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..e523506201ee7301dfa6f814e0315c01b95b08ee
Binary files /dev/null and b/unittests/table_json_conversion/data/multiple_choice_template.xlsx differ
diff --git a/unittests/table_json_conversion/data/multiple_refs_data.json b/unittests/table_json_conversion/data/multiple_refs_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa7c7af8e25096d15683bc924a41fb9572db3eb5
--- /dev/null
+++ b/unittests/table_json_conversion/data/multiple_refs_data.json
@@ -0,0 +1,48 @@
+{
+ "Training": [{
+ "trainer": [],
+ "participant": [
+ {
+ "full_name": "Petra Participant",
+ "email": "petra@indiscale.com"
+ },
+ {
+ "full_name": "Peter",
+ "email": "peter@getlinkahead.com"
+ }
+ ],
+ "Organisation": [
+ {
+ "Person": [
+ {
+ "full_name": "Henry Henderson",
+ "email": "henry@organization.org"
+ },
+ {
+ "full_name": "Harry Hamburg",
+ "email": "harry@organization.org"
+ }
+ ],
+ "name": "World Training Organization",
+ "Country": "US"
+ },
+ {
+ "Person": [
+ {
+ "full_name": "Hermione Harvard",
+ "email": "hermione@organisation.org.uk"
+ },
+ {
+ "full_name": "Hazel Harper",
+ "email": "hazel@organisation.org.uk"
+ }
+ ],
+ "name": "European Training Organisation",
+ "Country": "UK"
+ }
+ ],
+ "date": "2024-03-21T14:12:00.000Z",
+ "url": "www.indiscale.com",
+ "name": "Example training with multiple organizations."
+ }]
+}
diff --git a/unittests/table_json_conversion/data/multiple_refs_data.xlsx b/unittests/table_json_conversion/data/multiple_refs_data.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..21622ede9515b0cfa9f965f8c2ee89f782c4bf0c
Binary files /dev/null and b/unittests/table_json_conversion/data/multiple_refs_data.xlsx differ
diff --git a/unittests/table_json_conversion/data/multiple_refs_data_wrong_foreign.xlsx b/unittests/table_json_conversion/data/multiple_refs_data_wrong_foreign.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..f6e9d9f9f2024708a0b70d8ed4660cc97e04ff27
Binary files /dev/null and b/unittests/table_json_conversion/data/multiple_refs_data_wrong_foreign.xlsx differ
diff --git a/unittests/table_json_conversion/data/multiple_refs_model.yml b/unittests/table_json_conversion/data/multiple_refs_model.yml
new file mode 100644
index 0000000000000000000000000000000000000000..576ff1d59215f893cb5dec1fd5a90c5975713c60
--- /dev/null
+++ b/unittests/table_json_conversion/data/multiple_refs_model.yml
@@ -0,0 +1,36 @@
+Person:
+ recommended_properties:
+ full_name:
+ datatype: TEXT
+ email:
+ datatype: TEXT
+Training:
+ recommended_properties:
+ date:
+ datatype: DATETIME
+ description: 'The date of the training.'
+ url:
+ datatype: TEXT
+ description: 'The URL'
+ trainer:
+ datatype: LIST<Person>
+ participant:
+ datatype: LIST<Person>
+ supervisor:
+ datatype: Person
+ responsible:
+ datatype: Person
+ Organisation:
+ datatype: LIST<Organisation>
+ supervisor_inherit:
+ inherit_from_suggested:
+ - Person
+ responsible_inherit:
+ inherit_from_suggested:
+ - Person
+Organisation:
+ recommended_properties:
+ Country:
+ datatype: TEXT
+ Person:
+ datatype: LIST<Person>
diff --git a/unittests/table_json_conversion/data/multiple_refs_schema.json b/unittests/table_json_conversion/data/multiple_refs_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..2adec8de92b548ef97129074c7a24e9378118a4f
--- /dev/null
+++ b/unittests/table_json_conversion/data/multiple_refs_schema.json
@@ -0,0 +1,210 @@
+{
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Training",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "date": {
+ "description": "The date of the training.",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "The URL"
+ },
+ "trainer": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "trainer",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "participant": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "participant",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "supervisor": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "supervisor",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ },
+ "responsible": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "responsible",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ },
+ "Organisation": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Organisation",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "Country": {
+ "type": "string"
+ },
+ "Person": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Person",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "supervisor_inherit": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "supervisor_inherit",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ },
+ "responsible_inherit": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "responsible_inherit",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ },
+ "Person": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Person",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "full_name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+ },
+ "required": [],
+ "additionalProperties": false,
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}
diff --git a/unittests/table_json_conversion/data/multiple_refs_template.xlsx b/unittests/table_json_conversion/data/multiple_refs_template.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..cff3dad99a3c296e360d660ed5178a0eee48cd40
Binary files /dev/null and b/unittests/table_json_conversion/data/multiple_refs_template.xlsx differ
diff --git a/unittests/table_json_conversion/data/simple_data.json b/unittests/table_json_conversion/data/simple_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..92a1661a7e975747fa346997c0a3309e740c7324
--- /dev/null
+++ b/unittests/table_json_conversion/data/simple_data.json
@@ -0,0 +1,32 @@
+{
+ "Training": [{
+ "date": "2023-01-01",
+ "url": "www.indiscale.com",
+ "coach": [
+ {
+ "family_name": "Sky",
+ "given_name": "Max",
+ "Organisation": "ECB"
+ },
+ {
+ "family_name": "Sky",
+ "given_name": "Min",
+ "Organisation": "ECB"
+ }
+ ],
+ "supervisor": {
+ "family_name": "Steve",
+ "given_name": "Stevie",
+ "Organisation": "IMF"
+ },
+ "duration": 1.0,
+ "participants": 1,
+ "subjects": ["Math", "Physics"],
+ "remote": false
+ }],
+ "Person": [{
+ "family_name": "Steve",
+ "given_name": "Stevie",
+ "Organisation": "IMF"
+ }]
+}
diff --git a/unittests/table_json_conversion/data/simple_data.xlsx b/unittests/table_json_conversion/data/simple_data.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..662a636603944a91cbaeaea8cc0c3bbab12f2f50
Binary files /dev/null and b/unittests/table_json_conversion/data/simple_data.xlsx differ
diff --git a/unittests/table_json_conversion/data/simple_data_ascii_chars.json b/unittests/table_json_conversion/data/simple_data_ascii_chars.json
new file mode 100644
index 0000000000000000000000000000000000000000..84e22b9bcbf3b5c053d955ed398b442379a99395
--- /dev/null
+++ b/unittests/table_json_conversion/data/simple_data_ascii_chars.json
@@ -0,0 +1,18 @@
+{
+ "Training": [{
+ "date": "2023-01-01",
+ "url": "char: >\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009<",
+ "subjects": [
+ ">\u000a\u000b\u000c\u000e\u000f<",
+ ">\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017<",
+ ">\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f<",
+ ">\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027<",
+ ">\u0028\u0029\u002a\u002b\u002c\u002d\u002e\u002f<"
+ ]
+ }],
+ "Person": [{
+ "family_name": "Steve",
+ "given_name": "Stevie",
+ "Organisation": "IMF"
+ }]
+}
diff --git a/unittests/table_json_conversion/data/simple_data_ascii_chars.xlsx b/unittests/table_json_conversion/data/simple_data_ascii_chars.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..bdf60b568bf51726a0a25e599bb6c7e70988d287
Binary files /dev/null and b/unittests/table_json_conversion/data/simple_data_ascii_chars.xlsx differ
diff --git a/unittests/table_json_conversion/data/simple_data_datetime.xlsx b/unittests/table_json_conversion/data/simple_data_datetime.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..c6752614e700ecfd6040c087ff3254a68fd5e158
Binary files /dev/null and b/unittests/table_json_conversion/data/simple_data_datetime.xlsx differ
diff --git a/unittests/table_json_conversion/data/simple_data_missing.xlsx b/unittests/table_json_conversion/data/simple_data_missing.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..6e6d3a39d965de81236f2ad14bd2116ac4d7669b
Binary files /dev/null and b/unittests/table_json_conversion/data/simple_data_missing.xlsx differ
diff --git a/unittests/table_json_conversion/data/simple_data_schema.json b/unittests/table_json_conversion/data/simple_data_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..0a4d44f733b3a8301e2d053cd570c904ef02750f
--- /dev/null
+++ b/unittests/table_json_conversion/data/simple_data_schema.json
@@ -0,0 +1,145 @@
+{
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Training",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "date": {
+ "description": "The date of the training.",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "The URL"
+ },
+ "subjects": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "coach": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "coach",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ },
+ "Organisation": {
+ "enum": [
+ "Federal Reserve",
+ "IMF",
+ "ECB"
+ ]
+ }
+ }
+ }
+ },
+ "supervisor": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "supervisor",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ },
+ "Organisation": {
+ "enum": [
+ "Federal Reserve",
+ "IMF",
+ "ECB"
+ ]
+ }
+ }
+ },
+ "duration": {
+ "type": "number"
+ },
+ "participants": {
+ "type": "integer"
+ },
+ "remote": {
+ "type": "boolean"
+ },
+ "slides": {
+ "type": "string",
+ "format": "data-url"
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+ },
+ "Person": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Person",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ },
+ "Organisation": {
+ "enum": [
+ "Federal Reserve",
+ "IMF",
+ "ECB"
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+ }
+ },
+ "required": [
+ "Training",
+ "Person"
+ ],
+ "additionalProperties": false,
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}
diff --git a/unittests/table_json_conversion/data/simple_data_wrong_foreign.xlsx b/unittests/table_json_conversion/data/simple_data_wrong_foreign.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..dbd91d29947ec673a704a762f474111223484e56
Binary files /dev/null and b/unittests/table_json_conversion/data/simple_data_wrong_foreign.xlsx differ
diff --git a/unittests/table_json_conversion/data/simple_model.yml b/unittests/table_json_conversion/data/simple_model.yml
new file mode 100644
index 0000000000000000000000000000000000000000..74fb5bc5dc4251bb3834ea2f6201f991cab510d1
--- /dev/null
+++ b/unittests/table_json_conversion/data/simple_model.yml
@@ -0,0 +1,36 @@
+Person:
+ recommended_properties:
+ family_name:
+ datatype: TEXT
+ given_name:
+ datatype: TEXT
+ Organisation:
+Training:
+ recommended_properties:
+ date:
+ datatype: DATETIME
+ description: 'The date of the training.'
+ url:
+ datatype: TEXT
+ description: 'The URL'
+ subjects:
+ datatype: LIST<TEXT>
+ coach:
+ datatype: LIST<Person>
+ supervisor:
+ datatype: Person
+ duration:
+ datatype: DOUBLE
+ participants:
+ datatype: INTEGER
+ remote:
+ datatype: BOOLEAN
+ slides:
+ datatype: FILE
+ProgrammingCourse:
+ inherit_from_suggested:
+ - Training
+Organisation:
+ recommended_properties:
+ Country:
+ datatype: TEXT
diff --git a/unittests/table_json_conversion/data/simple_schema.json b/unittests/table_json_conversion/data/simple_schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..01a732d67758ed52a334ff7076778a45f8850f95
--- /dev/null
+++ b/unittests/table_json_conversion/data/simple_schema.json
@@ -0,0 +1,139 @@
+{
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Training",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "date": {
+ "description": "The date of the training.",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "The URL"
+ },
+ "subjects": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "coach": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "coach",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ },
+ "Organisation": {
+ "enum": [
+ "Federal Reserve",
+ "IMF",
+ "ECB"
+ ]
+ }
+ }
+ }
+ },
+ "supervisor": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "supervisor",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ },
+ "Organisation": {
+ "enum": [
+ "Federal Reserve",
+ "IMF",
+ "ECB"
+ ]
+ }
+ }
+ },
+ "duration": {
+ "type": "number"
+ },
+ "participants": {
+ "type": "integer"
+ },
+ "remote": {
+ "type": "boolean"
+ },
+ "slides": {
+ "type": "string",
+ "format": "data-url"
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ },
+ "Person": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": false,
+ "title": "Person",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ "family_name": {
+ "type": "string"
+ },
+ "given_name": {
+ "type": "string"
+ },
+ "Organisation": {
+ "enum": [
+ "Federal Reserve",
+ "IMF",
+ "ECB"
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+ }
+ },
+ "required": [
+ "Training",
+ "Person"
+ ],
+ "additionalProperties": false,
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}
diff --git a/unittests/table_json_conversion/data/simple_template.xlsx b/unittests/table_json_conversion/data/simple_template.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..1162965bf44642a4523123fa52c58dd240b25e5f
Binary files /dev/null and b/unittests/table_json_conversion/data/simple_template.xlsx differ
diff --git a/unittests/table_json_conversion/how_to_schema.md b/unittests/table_json_conversion/how_to_schema.md
new file mode 100644
index 0000000000000000000000000000000000000000..a7b4e3ca35a1fc9e67ebbb29f316825e89596f4a
--- /dev/null
+++ b/unittests/table_json_conversion/how_to_schema.md
@@ -0,0 +1,19 @@
+Insert the data model into a LinkAhead server.
+
+Run the following code:
+```
+ model = parser.parse_model_from_yaml("./model.yml")
+
+ exporter = jsex.JsonSchemaExporter(additional_properties=False,
+ #additional_options_for_text_props=additional_text_options,
+ #name_and_description_in_properties=True,
+ #do_not_create=do_not_create,
+ #do_not_retrieve=do_not_retrieve,
+ )
+ schema_top = exporter.recordtype_to_json_schema(model.get_deep("Training"))
+ schema_pers = exporter.recordtype_to_json_schema(model.get_deep("Person"))
+ merged_schema = jsex.merge_schemas([schema_top, schema_pers])
+
+ with open("model_schema.json", mode="w", encoding="utf8") as json_file:
+ json.dump(merged_schema, json_file, ensure_ascii=False, indent=2)
+```
diff --git a/unittests/table_json_conversion/test_fill_xlsx.py b/unittests/table_json_conversion/test_fill_xlsx.py
new file mode 100644
index 0000000000000000000000000000000000000000..899bb81ef1f91f3326f214f49f135a55b97d299f
--- /dev/null
+++ b/unittests/table_json_conversion/test_fill_xlsx.py
@@ -0,0 +1,198 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+import json
+import os
+import re
+import tempfile
+
+import jsonschema.exceptions as schema_exc
+import pytest
+from openpyxl import load_workbook
+
+from caosadvancedtools.table_json_conversion import xlsx_utils
+from caosadvancedtools.table_json_conversion.fill_xlsx import fill_template
+from caosadvancedtools.table_json_conversion.xlsx_utils import (
+ get_row_type_column_index,
+ get_path_rows,
+ read_or_dict,
+)
+
+from .utils import compare_workbooks
+
+
+def rfp(*pathcomponents):
+ """
+ Return full path.
+ Shorthand convenience function.
+ """
+ return os.path.join(os.path.dirname(__file__), *pathcomponents)
+
+
+def fill_and_compare(json_file: str, template_file: str, known_good: str,
+ schema: str = None, custom_output: str = None):
+ """Fill the data into a template and compare to a known good.
+
+Parameters:
+-----------
+schema: str, optional,
+ Json schema to validate against.
+custom_output: str, optional
+ If given, write to this file and drop into an IPython shell. For development only.
+ """
+ with tempfile.TemporaryDirectory() as tmpdir:
+ outfile = os.path.join(tmpdir, 'test.xlsx')
+ assert not os.path.exists(outfile)
+ if custom_output is not None:
+ outfile = custom_output
+ fill_template(data=json_file, template=template_file, result=outfile,
+ validation_schema=schema)
+ assert os.path.exists(outfile)
+ generated = load_workbook(outfile) # workbook can be read
+ known_good_wb = load_workbook(known_good)
+ compare_workbooks(generated, known_good_wb)
+
+
+def test_detect():
+ example = load_workbook(rfp("data/simple_template.xlsx"))
+ assert 0 == get_row_type_column_index(example['Person'])
+ assert [1, 2] == get_path_rows(example['Person'])
+
+
+def test_temporary():
+ # TODO: remove the following after manual testing
+ di = '/home/henrik/CaosDB/management/external/dimr/eingabemaske/crawler/schemas'
+ dd = '/home/henrik/CaosDB/management/external/dimr/eingabemaske/django/laforms/persistent/'
+ allreadydone = [
+ "Präventionsmaßnahmen",
+ "Beratungsstellen",
+ "Schutzeinrichtungen",
+ "Einzelfallversorgung",
+ "Strategiedokumente",
+ "Kooperationsvereinbarungen",
+ "Gremien",
+ "Verwaltungsvorschriften",
+ "Gewaltschutzkonzepte und -maßnahmen",
+ "Polizeilicher Opferschutz",
+ "Feedback",
+ ]
+ for prefix, _, files in os.walk(dd):
+ for fi in files:
+ match = re.match(r"(?P<teilb>.*)_2024-.*\.json", fi)
+
+ if match:
+ print(match.group('teilb'))
+ tb = match.group('teilb')
+ if tb in allreadydone:
+ continue
+ # allreadydone.append(tb)
+ template = os.path.join(di, "template_"+tb+".xlsx")
+ schema = os.path.join(di, "schema_"+tb+".json")
+ if not os.path.exists(template):
+ print(template)
+ assert False
+ jfi = os.path.join(prefix, fi)
+ print(jfi)
+ if not fi.startswith("Art"):
+ continue
+ # if jfi != "/home/henrik/CaosDB/management/external/dimr/eingabemaske/django/laforms/persistent/data/datenhalterin_gg/he_gg_2/Art__13_Bewusstseinsbildung_2024-01-11T10:22:26.json":
+ # continue
+ with open(jfi, encoding="utf-8") as infile:
+ data = json.load(infile)
+ data = data["form_data"]
+ if "__version__" in data:
+ del data["__version__"]
+ with tempfile.TemporaryDirectory() as tmpdir:
+ outfile = os.path.join(tmpdir, 'test.xlsx')
+ fill_template(data=data, template=template, result=outfile,
+ validation_schema=schema)
+ os.system(f'libreoffice {outfile}')
+
+
+def test_fill_xlsx():
+ fill_and_compare(json_file=rfp("data/simple_data.json"),
+ template_file=rfp("data/simple_template.xlsx"),
+ known_good=rfp("data/simple_data.xlsx"),
+ schema=rfp("data/simple_schema.json"))
+ fill_and_compare(json_file=rfp("data/multiple_refs_data.json"),
+ template_file=rfp("data/multiple_refs_template.xlsx"),
+ known_good=rfp("data/multiple_refs_data.xlsx"),
+ schema=rfp("data/multiple_refs_schema.json"))
+ fill_and_compare(json_file=rfp("data/indirect_data.json"),
+ template_file=rfp("data/indirect_template.xlsx"),
+ known_good=rfp("data/indirect_data.xlsx"),
+ schema=rfp("data/indirect_schema.json"))
+ fill_and_compare(json_file=rfp("data/simple_data_ascii_chars.json"),
+ template_file=rfp("data/simple_template.xlsx"),
+ known_good=rfp("data/simple_data_ascii_chars.xlsx"),
+ schema=rfp("data/simple_schema.json"))
+ fill_and_compare(json_file=rfp("data/multiple_choice_data.json"),
+ template_file=rfp("data/multiple_choice_template.xlsx"),
+ known_good=rfp("data/multiple_choice_data.xlsx"),
+ schema=rfp("data/multiple_choice_schema.json"))
+
+
+def test_datetime():
+ """Datetime values from LinkAhead are not serialized as strings."""
+ json_file = rfp("data/simple_data.json")
+ template_file = rfp("data/simple_template.xlsx")
+ known_good = rfp("data/simple_data_datetime.xlsx")
+ # TODO Implement checker for datetime
+ # schema = rfp("data/simple_schema.json")
+
+ # Set datetime explicitly
+ json_data = read_or_dict(json_file)
+ json_data["Training"][0]["date"] = datetime.datetime(2023, 1, 1)
+
+ # Code copied mostly from `fill_and_compare(...)`
+ with tempfile.TemporaryDirectory() as tmpdir:
+ outfile = os.path.join(tmpdir, 'test.xlsx')
+ assert not os.path.exists(outfile)
+ fill_template(data=json_data, template=template_file, result=outfile,
+ # validation_schema=schema
+ )
+ assert os.path.exists(outfile)
+ generated = load_workbook(outfile) # workbook can be read
+
+ known_good_wb = load_workbook(known_good)
+ compare_workbooks(generated, known_good_wb)
+
+
+def test_errors():
+ with pytest.raises(AssertionError) as exc:
+ fill_and_compare(json_file=rfp("data/error_simple_data.json"),
+ template_file=rfp("data/simple_template.xlsx"),
+ known_good=rfp("data/simple_data.xlsx"))
+ assert "Auric\nSteve" in str(exc.value)
+ with pytest.raises(schema_exc.ValidationError) as exc:
+ fill_and_compare(json_file=rfp("data/error_simple_data.json"),
+ template_file=rfp("data/simple_template.xlsx"),
+ known_good=rfp("data/simple_data.xlsx"),
+ schema=rfp("data/simple_schema.json"))
+ assert exc.value.message == "0.5 is not of type 'integer'"
+
+
+def test_data_schema_generation():
+ model_schema = xlsx_utils.read_or_dict(rfp("data/simple_schema.json"))
+ array_schema = xlsx_utils.array_schema_from_model_schema(model_schema)
+ expected = xlsx_utils.read_or_dict(rfp("data/simple_data_schema.json"))
+ assert array_schema == expected
diff --git a/unittests/table_json_conversion/test_read_xlsx.py b/unittests/table_json_conversion/test_read_xlsx.py
new file mode 100644
index 0000000000000000000000000000000000000000..0eec2e9caa1f800ad86ab43057b8c512dc09881f
--- /dev/null
+++ b/unittests/table_json_conversion/test_read_xlsx.py
@@ -0,0 +1,221 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Testing the conversion from XLSX to JSON"""
+
+
+import datetime
+import json
+import os
+import re
+
+from types import SimpleNamespace
+
+import pytest
+from caosadvancedtools.table_json_conversion import convert
+
+from .utils import assert_equal_jsons
+
+
+def rfp(*pathcomponents):
+ """Return full path, a shorthand convenience function.
+ """
+ return os.path.join(os.path.dirname(__file__), *pathcomponents)
+
+
+def convert_and_compare(xlsx_file: str, schema_file: str, known_good_file: str,
+ known_good_data: dict = None, strict: bool = False,
+ validate: bool = True) -> dict:
+ """Convert an XLSX file and compare to a known result.
+
+Exactly one of ``known_good_file`` and ``known_good_data`` should be non-empty.
+
+Returns
+-------
+json: dict
+ The result of the conversion.
+ """
+ result = convert.to_dict(xlsx=xlsx_file, schema=schema_file, validate=validate)
+ if known_good_file:
+ with open(known_good_file, encoding="utf-8") as myfile:
+ expected = json.load(myfile)
+ else:
+ expected = known_good_data
+ assert_equal_jsons(result, expected, allow_none=not strict, allow_empty=not strict)
+ return result
+
+
+def test_conversions():
+ """Test conversion from XLSX to JSON."""
+ convert_and_compare(xlsx_file=rfp("data/simple_data.xlsx"),
+ schema_file=rfp("data/simple_schema.json"),
+ known_good_file=rfp("data/simple_data.json"))
+ convert_and_compare(xlsx_file=rfp("data/multiple_refs_data.xlsx"),
+ schema_file=rfp("data/multiple_refs_schema.json"),
+ known_good_file=rfp("data/multiple_refs_data.json"))
+ convert_and_compare(xlsx_file=rfp("data/indirect_data.xlsx"),
+ schema_file=rfp("data/indirect_schema.json"),
+ known_good_file=rfp("data/indirect_data.json"))
+ convert_and_compare(xlsx_file=rfp("data/multiple_choice_data.xlsx"),
+ schema_file=rfp("data/multiple_choice_schema.json"),
+ known_good_file=rfp("data/multiple_choice_data.json"),
+ strict=True)
+
+ with open(rfp("data/simple_data.json"), encoding="utf-8") as myfile:
+ expected_datetime = json.load(myfile)
+ expected_datetime["Training"][0]["date"] = datetime.datetime(2023, 1, 1, 0, 0)
+ convert_and_compare(xlsx_file=rfp("data/simple_data_datetime.xlsx"),
+ schema_file=rfp("data/simple_schema.json"),
+ known_good_file="", known_good_data=expected_datetime)
+
+ # Data loss when saving as xlsx
+ with pytest.raises(AssertionError) as err:
+ convert_and_compare(xlsx_file=rfp("data/simple_data_ascii_chars.xlsx"),
+ schema_file=rfp("data/simple_schema.json"),
+ known_good_file=rfp("data/simple_data_ascii_chars.json"))
+ assert str(err.value).startswith("Values at path ['Training', 0, ")
+
+
+def test_missing_columns():
+ with pytest.raises(ValueError) as caught:
+ convert.to_dict(xlsx=rfp("data/simple_data_missing.xlsx"),
+ schema=rfp("data/simple_schema.json"), strict=True)
+ assert str(caught.value) == "Missing column: Training.coach.given_name"
+ with pytest.warns(UserWarning) as caught:
+ convert.to_dict(xlsx=rfp("data/simple_data_missing.xlsx"),
+ schema=rfp("data/simple_schema.json"))
+ assert str(caught.pop().message) == "Missing column: Training.coach.given_name"
+ with pytest.warns(UserWarning) as caught:
+ convert.to_dict(xlsx=rfp("data/multiple_choice_data_missing.xlsx"),
+ schema=rfp("data/multiple_choice_schema.json"))
+ messages = {str(w.message) for w in caught}
+ for expected in [
+ "Missing column: Training.skills.Communication",
+ "Missing column: Training.exam_types.Oral",
+ ]:
+ assert expected in messages
+
+
+def test_faulty_foreign():
+ # Simple wrong foreign key
+ converter = convert.XLSXConverter(xlsx=rfp("data/simple_data_wrong_foreign.xlsx"),
+ schema=rfp("data/simple_schema.json"))
+ with pytest.raises(RuntimeError):
+ converter.to_dict()
+ errors = converter.get_errors()
+ assert errors == {('Training.coach', 6): [['date', datetime.datetime(2023, 1, 2, 0, 0)],
+ ['url', 'www.indiscale.com']]}
+
+ # More extensive example
+ converter = convert.XLSXConverter(xlsx=rfp("data/multiple_refs_data_wrong_foreign.xlsx"),
+ schema=rfp("data/multiple_refs_schema.json"))
+ with pytest.raises(RuntimeError):
+ converter.to_dict()
+ errors = converter.get_errors()
+ assert errors == {
+ ('Training.Organisation.Person', 8): [
+ ['name', 'World Training Organization 2']],
+ ('Training.Organisation.Person', 9): [
+ ['date', '2024-03-21T14:12:00.000Z'],
+ ['url', 'www.getlinkahead.com']],
+ ('Training.participant', 6): [
+ ['date', '2024-03-21T14:12:00.000Z'],
+ ['url', None]],
+ ('Training.participant', 7): [
+ ['date', '2024-03-21T14:12:00.000Z'],
+ ['url', None]],
+ }
+
+ error_str = converter.get_error_str()
+ assert error_str == """Sheet: Training.Organisation.Person\tRow: 9
+\t\t['name']:\tWorld Training Organization 2
+Sheet: Training.Organisation.Person\tRow: 10
+\t\t['date']:\t2024-03-21T14:12:00.000Z
+\t\t['url']:\twww.getlinkahead.com
+Sheet: Training.participant\tRow: 7
+\t\t['date']:\t2024-03-21T14:12:00.000Z
+\t\t['url']:\tNone
+Sheet: Training.participant\tRow: 8
+\t\t['date']:\t2024-03-21T14:12:00.000Z
+\t\t['url']:\tNone
+"""
+
+
+def test_set_in_nested():
+ """Test the ``_set_in_nested`` function."""
+ set_in_nested = convert._set_in_nested # pylint: disable=protected-access
+
+ test_data_in = [
+ {"mydict": {}, "path": ["a", 1], "value": 3},
+ {"mydict": {"a": 1}, "path": ["a"], "value": 3, "overwrite": True},
+ {"mydict": {"a": 1}, "path": ["a", 1], "value": 3, "overwrite": True},
+ {"mydict": {"b": 2}, "path": ["a", 1, 3.141], "value": 3},
+ {"mydict": {}, "path": ["X", "Y", "a", 1], "value": 3, "prefix": ["X", "Y"]},
+ ]
+ test_data_out = [
+ {"a": {1: 3}},
+ {"a": 3},
+ {"a": {1: 3}},
+ {"a": {1: {3.141: 3}}, "b": 2},
+ {"a": {1: 3}},
+ ]
+
+ for data_in, data_out in zip(test_data_in, test_data_out):
+ assert set_in_nested(**data_in) == data_out
+
+ # Testing exceptions
+ test_data_in = [
+ {"mydict": {"a": 1}, "path": ["a"], "value": 3},
+ {"mydict": {"a": 1}, "path": ["a", 1], "value": 3},
+ {"mydict": {}, "path": ["a", 1], "value": 3, "prefix": ["X", "Y", "Z"]},
+ ]
+ exceptions = [
+ [ValueError, r"There is already some value at \[a\]"],
+ [ValueError, r"There is already some value at \[1\]"],
+ [KeyError, r"Path does not start with prefix: \['X', 'Y', 'Z'\] not in \['a', 1\]"],
+ ]
+
+ for data_in, (exc_out, match) in zip(test_data_in, exceptions):
+ with pytest.raises(exc_out, match=match):
+ set_in_nested(**data_in)
+
+
+def test_group_foreign_paths():
+ """Test the ``_group_foreign_paths`` function."""
+ group = convert._group_foreign_paths # pylint: disable=protected-access
+
+ foreign = [
+ ["A", "x", 1.1],
+ ["A", "y", "z", "some text"],
+ ["A", "B", "CC", "x", 42],
+ ]
+ common = ["A", "B", "CC"]
+ common_wrong = ["A", "B", "C"]
+ expected = [
+ SimpleNamespace(stringpath="A", path=["A"], subpath=["A"],
+ definitions=[["x", 1.1], ["y", "z", "some text"]]),
+ SimpleNamespace(stringpath="A.B.CC", path=["A", "B", "CC"], subpath=["B", "CC"],
+ definitions=[["x", 42]]),
+ ]
+
+ with pytest.raises(ValueError, match=re.escape(
+ "Foreign keys must cover the complete `common` depth.")):
+ result = group(foreign=foreign, common=common_wrong)
+ result = group(foreign=foreign, common=common)
+ assert result == expected
diff --git a/unittests/table_json_conversion/test_table_template_generator.py b/unittests/table_json_conversion/test_table_template_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9a84dcf53ec991eec709aab406a7652881e6ea8
--- /dev/null
+++ b/unittests/table_json_conversion/test_table_template_generator.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import json
+import os
+import tempfile
+
+import pytest
+from caosadvancedtools.table_json_conversion.table_generator import XLSXTemplateGenerator
+from caosadvancedtools.table_json_conversion.xlsx_utils import ColumnType
+from openpyxl import load_workbook
+
+from .utils import compare_workbooks
+
+
+def rfp(*pathcomponents):
+ """
+ Return full path.
+ Shorthand convenience function.
+ """
+ return os.path.join(os.path.dirname(__file__), *pathcomponents)
+
+
+def _compare_generated_to_known_good(schema_file: str, known_good: str, foreign_keys: dict = None,
+ outfile: str = None) -> tuple:
+ """Generate an XLSX from the schema, then compare to known good output.
+
+Returns
+-------
+out: tuple
+ The generated and known good workbook objects.
+ """
+ generator = XLSXTemplateGenerator()
+ if foreign_keys is None:
+ foreign_keys = {}
+ with open(schema_file, encoding="utf-8") as schema_input:
+ schema = json.load(schema_input)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ if outfile is None:
+ outpath = os.path.join(tmpdir, 'generated.xlsx')
+ else:
+ outpath = outfile
+ assert not os.path.exists(outpath)
+ generator.generate(schema=schema,
+ foreign_keys=foreign_keys,
+ filepath=outpath)
+ assert os.path.exists(outpath)
+ generated = load_workbook(outpath)
+ good = load_workbook(known_good)
+ compare_workbooks(generated, good)
+ return generated, good
+
+
+def test_generate_sheets_from_schema():
+ # trivial case; we do not support this
+ schema = {}
+ generator = XLSXTemplateGenerator()
+ with pytest.raises(ValueError, match="Inappropriate JSON schema:.*"):
+ generator._generate_sheets_from_schema(schema)
+
+ # top level must be RT with Properties
+ schema = {
+ "type": "string"
+ }
+ with pytest.raises(ValueError, match="Inappropriate JSON schema:.*"):
+ generator._generate_sheets_from_schema(schema)
+
+ # bad type
+ schema = {
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "str",
+ "description": "The name of the Record to be created"
+ },
+ }
+ }
+ }
+ }
+ with pytest.raises(ValueError,
+ match="Inappropriate JSON schema: The following part "
+ "should define an object.*"):
+ generator._generate_sheets_from_schema(schema, {'Training': ['a']})
+
+ # bad schema
+ schema = {
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "object"
+ }
+ }
+ }
+ with pytest.raises(ValueError,
+ match="Inappropriate JSON schema: The following part "
+ "should define an object.*"):
+ generator._generate_sheets_from_schema(schema, {'Training': ['a']})
+
+ # minimal case: one RT with one P
+ schema = {
+ "type": "object",
+ "properties": {
+ "Training": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the Record to be created"
+ },
+ }
+ }
+ }
+ }
+ sdef = generator._generate_sheets_from_schema(schema, {'Training': ['a']})
+ assert "Training" in sdef
+ tdef = sdef['Training']
+ assert 'name' in tdef
+ assert tdef['name'] == (ColumnType.SCALAR, "The name of the Record to be created", ["Training", 'name'])
+
+ # example case
+ with open(rfp("data/simple_schema.json")) as sfi:
+ schema = json.load(sfi)
+ with pytest.raises(ValueError, match="A foreign key definition is missing.*"):
+ generator._generate_sheets_from_schema(schema)
+ sdef = generator._generate_sheets_from_schema(
+ schema,
+ foreign_keys={'Training': {"__this__": ['date', 'url']}})
+ assert "Training" in sdef
+ tdef = sdef['Training']
+ assert tdef['date'] == (ColumnType.SCALAR, 'The date of the training.', ["Training", 'date'])
+ assert tdef['url'] == (ColumnType.SCALAR, 'The URL', ["Training", 'url'])
+ assert tdef['supervisor.family_name'] == (ColumnType.SCALAR, None, ["Training", 'supervisor',
+ 'family_name'])
+ assert tdef['supervisor.given_name'] == (ColumnType.SCALAR, None, ["Training", 'supervisor',
+ 'given_name'])
+ assert tdef['supervisor.Organisation'] == (ColumnType.SCALAR, None, ["Training", 'supervisor',
+ 'Organisation'])
+ assert tdef['duration'] == (ColumnType.SCALAR, None, ["Training", 'duration'])
+ assert tdef['participants'] == (ColumnType.SCALAR, None, ["Training", 'participants'])
+ assert tdef['subjects'] == (ColumnType.LIST, None, ["Training", 'subjects'])
+ assert tdef['remote'] == (ColumnType.SCALAR, None, ["Training", 'remote'])
+ cdef = sdef['Training.coach']
+ assert cdef['family_name'] == (ColumnType.SCALAR, None, ["Training", 'coach', 'family_name'])
+ assert cdef['given_name'] == (ColumnType.SCALAR, None, ["Training", 'coach', 'given_name'])
+ assert cdef['Organisation'] == (ColumnType.SCALAR, None, ["Training", 'coach',
+ 'Organisation'])
+ assert cdef['Training.date'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'date'])
+ assert cdef['Training.url'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'url'])
+
+
+def test_get_foreign_keys():
+ generator = XLSXTemplateGenerator()
+ fkd = {"Training": ['a']}
+ assert [['a']] == generator._get_foreign_keys(fkd, ['Training'])
+
+ fkd = {"Training": {"__this__": ['a']}}
+ assert [['a']] == generator._get_foreign_keys(fkd, ['Training'])
+
+ fkd = {"Training": {'hallo'}}
+ with pytest.raises(ValueError, match=r"A foreign key definition is missing for path:\n\["
+ r"'Training'\]\nKeys are:\n{'Training': \{'hallo'\}\}"):
+ generator._get_foreign_keys(fkd, ['Training'])
+
+ fkd = {"Training": {"__this__": ['a'], 'b': ['c']}}
+ assert [['c']] == generator._get_foreign_keys(fkd, ['Training', 'b'])
+
+ with pytest.raises(ValueError, match=r"A foreign key definition is missing for .*"):
+ generator._get_foreign_keys({}, ['Training'])
+
+
+def test_get_max_path_length():
+ assert 4 == XLSXTemplateGenerator._get_max_path_length({'a': (1, 'desc', [1, 2, 3]),
+ 'b': (2, 'desc', [1, 2, 3, 4])})
+
+
+def test_template_generator():
+ generated, _ = _compare_generated_to_known_good(
+ schema_file=rfp("data/simple_schema.json"), known_good=rfp("data/simple_template.xlsx"),
+ foreign_keys={'Training': {"__this__": ['date', 'url']}},
+ outfile=None)
+ # test some hidden
+ ws = generated.active
+ assert ws.row_dimensions[1].hidden is True
+ assert ws.column_dimensions['A'].hidden is True
+
+ # TODO: remove the following after manual testing
+ di = '/home/henrik/CaosDB/management/external/dimr/eingabemaske/crawler/schemas'
+ if not os.path.exists(di):
+ return
+ for fi in os.listdir(di):
+ rp = os.path.join(di, fi)
+ if not fi.startswith("schema_"):
+ continue
+ with open(rp) as sfi:
+ schema = json.load(sfi)
+ fk_path = os.path.join(di, "foreign_keys"+fi[len('schema'):])
+ path = os.path.join(di, "template"+fi[len('schema'):-4]+"xlsx")
+ alreadydone = [
+ "Präventionsmaßnahmen",
+ "Beratungsstellen",
+ "Schutzeinrichtungen",
+ "Einzelfallversorgung",
+ "Strategiedokumente",
+ "Kooperationsvereinbarungen",
+ "Gremien",
+ "Verwaltungsvorschriften",
+ "Gewaltschutzkonzepte und -maßnahmen",
+ "Polizeilicher Opferschutz",
+ "Feedback",
+ ]
+ if any([path.startswith("template_"+k) for k in alreadydone]):
+ continue
+
+ if not os.path.exists(fk_path):
+ print(f"No foreign keys file for:\n{fk_path}")
+ assert False
+ with open(fk_path) as sfi:
+ fk = json.load(sfi)
+ generator = XLSXTemplateGenerator()
+ if not os.path.exists(path):
+ generator.generate(schema=schema, foreign_keys=fk, filepath=path)
+ os.system(f'libreoffice {path}')
+ else:
+ print(f"Not creating template because it exists:\n{path}")
+
+ # TODO test collisions of sheet or colnames
+ # TODO test escaping of values
+
+ # TODO finish enum example
+
+
+def test_model_with_multiple_refs():
+ _compare_generated_to_known_good(
+ schema_file=rfp("data/multiple_refs_schema.json"),
+ known_good=rfp("data/multiple_refs_template.xlsx"),
+ foreign_keys={"Training": {"__this__": ["date", "url"],
+ "Organisation": ["name"]}},
+ outfile=None)
+
+
+def test_model_with_indirect_reference():
+ _compare_generated_to_known_good(
+ schema_file=rfp("data/indirect_schema.json"),
+ known_good=rfp("data/indirect_template.xlsx"),
+ foreign_keys={"Wrapper": {"__this__": [["Training", "name"], ["Training", "url"]]}},
+ outfile=None)
+
+
+def test_model_with_multiple_choice():
+ _compare_generated_to_known_good(
+ schema_file=rfp("data/multiple_choice_schema.json"),
+ known_good=rfp("data/multiple_choice_template.xlsx"),
+ outfile=None)
+
+
+def test_exceptions():
+ # Foreign keys must be lists
+ with pytest.raises(ValueError, match="Foreign keys must be a list of strings, but a single "
+ r"string was given:\n\['Wrapper'\] -> name"):
+ _compare_generated_to_known_good(
+ schema_file=rfp("data/indirect_schema.json"),
+ known_good=rfp("data/multiple_refs_template.xlsx"),
+ foreign_keys={"Wrapper": {"__this__": "name"}},
+ outfile=None)
diff --git a/unittests/table_json_conversion/test_test_utils.py b/unittests/table_json_conversion/test_test_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..30171f61de26b1ae11fb25c730c96b31aa8f06a3
--- /dev/null
+++ b/unittests/table_json_conversion/test_test_utils.py
@@ -0,0 +1,42 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Testing the ``utils`` module in this folder."""
+
+
+from .utils import _is_recursively_none
+
+
+def test_recursively_none():
+ """Testing ``_is_recursively_none``."""
+ assert _is_recursively_none(None)
+ assert _is_recursively_none([])
+ assert _is_recursively_none({})
+ assert _is_recursively_none([None])
+ assert _is_recursively_none({"a": None})
+ assert _is_recursively_none([[], [None, None]])
+ assert _is_recursively_none({1: [], 2: [None], 3: {"3.1": None}, 4: {"4.1": [None]}})
+
+ assert not _is_recursively_none(1)
+ assert not _is_recursively_none([1])
+ assert not _is_recursively_none({1: 2})
+ assert not _is_recursively_none([[1]])
+ assert not _is_recursively_none({"a": None, "b": "b"})
+ assert not _is_recursively_none([[], [None, 2]])
+ assert not _is_recursively_none({1: [], 2: [None], 3: {"3.1": 3.141}, 4: {"4.1": [None]}})
diff --git a/unittests/table_json_conversion/utils.py b/unittests/table_json_conversion/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..b95715f72b08384f75857e48bcba328488313ad5
--- /dev/null
+++ b/unittests/table_json_conversion/utils.py
@@ -0,0 +1,116 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Utilities for the tests.
+"""
+
+from typing import Iterable, Union
+
+from openpyxl import Workbook
+
+
+def assert_equal_jsons(json1, json2, allow_none: bool = True, allow_empty: bool = True,
+ path: list = None) -> None:
+ """Compare two json objects for near equality.
+
+Raise an assertion exception if they are not equal."""
+ if path is None:
+ path = []
+ assert isinstance(json1, dict) == isinstance(json2, dict), f"Type mismatch, path: {path}"
+ if isinstance(json1, dict):
+ keys = set(json1.keys()).union(json2.keys())
+ for key in keys:
+ this_path = path + [key]
+ # Case 1: exists in both collections
+ if key in json1 and key in json2:
+ el1 = json1[key]
+ el2 = json2[key]
+ assert isinstance(el1, type(el2)), f"Type mismatch, path: {this_path}"
+ if isinstance(el1, (dict, list)):
+ # Iterables: Recursion
+ assert_equal_jsons(el1, el2, allow_none=allow_none, allow_empty=allow_empty,
+ path=this_path)
+ continue
+ assert el1 == el2, f"Values at path {this_path} are not equal:\n{el1},\n{el2}"
+ continue
+ # Case 2: exists only in one collection
+ existing = json1.get(key, json2.get(key))
+ assert ((allow_none and _is_recursively_none(existing))
+ or (allow_empty and existing == [])), (
+ f"Element at path {this_path} is None or empty in one json and does not exist in "
+ "the other.")
+ return
+ assert isinstance(json1, list) and isinstance(json2, list), f"Is not a list, path: {path}"
+ assert len(json1) == len(json2), f"Lists must have equal length, path: {path}"
+ for idx, (el1, el2) in enumerate(zip(json1, json2)):
+ this_path = path + [idx]
+ if isinstance(el1, dict):
+ assert_equal_jsons(el1, el2, allow_none=allow_none, allow_empty=allow_empty,
+ path=this_path)
+ else:
+ assert el1 == el2, f"Values at path {this_path} are not equal:\n{el1},\n{el2}"
+
+
+def compare_workbooks(wb1: Workbook, wb2: Workbook, hidden: bool = True):
+ """Compare two workbooks for equal content.
+
+Raises an error if differences are found.
+
+Parameters
+----------
+
+hidden: bool, optional
+ Test if the "hidden" status of rows and columns is the same.
+ """
+ assert wb1.sheetnames == wb2.sheetnames, (
+ f"Sheet names are different: \n{wb1.sheetnames}\n !=\n{wb2.sheetnames}"
+ )
+ for sheetname in wb2.sheetnames:
+ sheet_1 = wb1[sheetname]
+ sheet_2 = wb2[sheetname]
+ for irow, (row1, row2) in enumerate(zip(sheet_1.iter_rows(), sheet_2.iter_rows())):
+ if hidden:
+ assert (sheet_1.row_dimensions[irow].hidden
+ == sheet_2.row_dimensions[irow].hidden), f"hidden row: {sheetname}, {irow}"
+ for icol, (cell1, cell2) in enumerate(zip(row1, row2)):
+ if hidden:
+ assert (sheet_1.column_dimensions[cell1.column_letter].hidden
+ == sheet_2.column_dimensions[cell2.column_letter].hidden), (
+ f"hidden col: {sheetname}, {icol}")
+ assert cell1.value == cell2.value, (
+ f"Sheet: {sheetname}, cell: {cell1.coordinate}, Values: \n"
+ f"{cell1.value}\n{cell2.value}"
+ )
+
+
+def _is_recursively_none(obj: Union[list, dict] = None):
+ """Test if ``obj`` is None or recursively consists only of None-like objects."""
+ if obj is None:
+ return True
+ if isinstance(obj, (list, dict)):
+ if isinstance(obj, list):
+ mylist: Iterable = obj
+ else:
+ mylist = obj.values()
+ for element in mylist:
+ if not _is_recursively_none(element):
+ return False
+ return True
+ return False
diff --git a/unittests/test_cfood.py b/unittests/test_cfood.py
index 7055bc7c51962c0cbc487f29bcdacb391218a7d3..e2f15ffdc7929fbd67aee37bccdb0f44cacef104 100644
--- a/unittests/test_cfood.py
+++ b/unittests/test_cfood.py
@@ -32,7 +32,7 @@ from caosadvancedtools.cfood import (AbstractCFood, AbstractFileCFood, CMeal,
get_entity_for_path)
from caosadvancedtools.crawler import FileCrawler
from caosadvancedtools.example_cfood import ExampleCFood
-from caosdb.common.models import _parse_single_xml_element
+from linkahead.common.models import _parse_single_xml_element
from lxml import etree
from datetime import datetime, timezone
@@ -336,6 +336,7 @@ class MealTest(unittest.TestCase):
class FileCacheTest(unittest.TestCase):
def test(self):
+ # if there is no connection to a server an exception should be raised.
self.assertRaises(Exception, get_entity_for_path, "/lol")
FileCrawler(cfood_types=[], files=[db.File(path="/lol")])
get_entity_for_path("/lol")
diff --git a/unittests/test_data_model.py b/unittests/test_data_model.py
index 159adfca1d589bb092b6f59110828b5868401e25..cafeb6ca6a43d7e0409aee3352b43f26d5208732 100644
--- a/unittests/test_data_model.py
+++ b/unittests/test_data_model.py
@@ -2,6 +2,7 @@ import unittest
import caosdb as db
from caosadvancedtools.models.data_model import DataModel
+from caosadvancedtools.models.parser import parse_model_from_string
class DataModelTest(unittest.TestCase):
@@ -33,3 +34,49 @@ class DataModelTest(unittest.TestCase):
DataModel.sync_ids_by_name(l1, l2)
assert l1["TestRecord"].id == rt.id
assert l1["TestRecord2"].id < 0
+
+ def test_get_deep(self):
+ model_recursive_str = """
+RT1:
+ description: some description
+ obligatory_properties:
+ RT1:
+ """
+ model_recursive = parse_model_from_string(model_recursive_str)
+ prop1 = model_recursive["RT1"].get_property("RT1")
+ assert prop1.datatype is None
+ # TODO The next line actually changes model_recursive in place, is this OK?
+ RT1 = model_recursive.get_deep("RT1")
+ assert model_recursive["RT1"] == RT1
+
+ model_unresolved_str = """
+RT1:
+ description: some description
+ obligatory_properties:
+ unresolved:
+ """
+ model_unresolved = parse_model_from_string(model_unresolved_str)
+ rt1_unresolved = model_unresolved["RT1"]
+ prop_unresolved = model_unresolved.get_deep("unresolved")
+ assert prop_unresolved.datatype is None
+ rt1_deep = model_unresolved.get_deep("RT1")
+ assert rt1_deep == rt1_unresolved
+ assert rt1_deep is rt1_unresolved
+
+ model_double_property = """
+p1:
+ description: Hello world
+ datatype: TEXT
+RT1:
+ recommended_properties:
+ p1:
+RT2:
+ recommended_properties:
+ RT1:
+ p1:
+"""
+ model_unresolved = parse_model_from_string(model_double_property)
+ rt2_deep = model_unresolved.get_deep("RT2")
+ p1 = rt2_deep.get_property("p1")
+ assert p1.datatype == "TEXT"
+ assert p1.description == "Hello world"
diff --git a/unittests/test_h5.py b/unittests/test_h5.py
index 360d4b28938492d0f2af6d696e39dffb1cc3fead..961dd4246ef4b02208226ada5d3e1389133ddbcc 100644
--- a/unittests/test_h5.py
+++ b/unittests/test_h5.py
@@ -1,8 +1,8 @@
import unittest
from tempfile import NamedTemporaryFile
-import caosdb as db
-import caosdb.apiutils
+import linkahead as db
+import linkahead.apiutils
import h5py
import numpy as np
from caosadvancedtools.cfoods import h5
@@ -77,8 +77,8 @@ class H5CFoodTest(unittest.TestCase):
# TODO this does probably break the code: The function will not be
# restored correctly.
# Change it to use the BaseMockUpTest
- real_retrieve = caosdb.apiutils.retrieve_entity_with_id
- caosdb.apiutils.retrieve_entity_with_id = dummy_get
+ real_retrieve = linkahead.apiutils.retrieve_entity_with_id
+ linkahead.apiutils.retrieve_entity_with_id = dummy_get
# should run without problem
h5.collect_existing_structure(db.Record(), db.Record(id=234), h5.EntityMapping())
@@ -151,7 +151,7 @@ class H5CFoodTest(unittest.TestCase):
self.assertEqual(em.to_existing[r_child2._cuid], ENTS[101])
self.assertEqual(em.to_target[101], r_child2)
- caosdb.apiutils.retrieve_entity_with_id = real_retrieve
+ linkahead.apiutils.retrieve_entity_with_id = real_retrieve
def test_h5_attr_to_property(self):
@@ -160,7 +160,8 @@ class H5CFoodTest(unittest.TestCase):
test_float = np.float_(1.0)
test_str = "Test"
test_complex: complex = 2+3j
- self.assertRaises(NotImplementedError, h5_attr_to_property, test_int) # only numpy-integers processed?
+ self.assertRaises(NotImplementedError, h5_attr_to_property,
+ test_int) # only numpy-integers processed?
self.assertTupleEqual((1, db.INTEGER), h5_attr_to_property(test_integer))
self.assertTupleEqual((1.0, db.DOUBLE), h5_attr_to_property(test_float))
self.assertTupleEqual(("Test", db.TEXT), h5_attr_to_property(test_str))
@@ -187,4 +188,5 @@ class H5CFoodTest(unittest.TestCase):
# Test scalar values given as np.array
self.assertTupleEqual((1, db.INTEGER), h5_attr_to_property(np.array(1)))
self.assertTupleEqual((1.123, db.DOUBLE), h5_attr_to_property(np.array(1.123)))
- self.assertTupleEqual(('Hello World', db.TEXT), h5_attr_to_property(np.array("Hello World")))
+ self.assertTupleEqual(('Hello World', db.TEXT),
+ h5_attr_to_property(np.array("Hello World")))
diff --git a/unittests/test_json_schema_exporter.py b/unittests/test_json_schema_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..1cea2f58e42d28545b8354d0f821ab2c61e7a4f8
--- /dev/null
+++ b/unittests/test_json_schema_exporter.py
@@ -0,0 +1,1093 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Tests the Json schema exporter."""
+
+import json
+
+import linkahead as db
+import caosadvancedtools.json_schema_exporter as jsex
+
+from collections import OrderedDict
+
+from jsonschema import FormatChecker, validate, ValidationError
+from pytest import raises
+from unittest.mock import Mock, patch
+
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema as rtjs
+from caosadvancedtools.models.parser import parse_model_from_string
+
+GLOBAL_MODEL = parse_model_from_string("""
+RT1:
+ description: some description
+ obligatory_properties:
+ some_date:
+ datatype: DATETIME
+ description: Just some date
+RT21:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+RT31:
+ obligatory_properties:
+ RT1:
+
+""")
+
+RT1 = GLOBAL_MODEL.get_deep("RT1")
+RT21 = GLOBAL_MODEL.get_deep("RT21")
+RT31 = GLOBAL_MODEL.get_deep("RT31")
+
+
+def _make_unique(entity, unique: bool):
+ """Mock the `unique` behavior of execute_query().
+ """
+ if unique:
+ return entity
+ return db.Container().append(entity)
+
+
+def _mock_execute_query(query_string, unique=False, **kwargs):
+ """Mock the response to queries for references."""
+ all_records = db.Container()
+ all_files = db.Container()
+ other_type_rt = db.RecordType(name="OtherType")
+ other_type_rt.add_property(name="IntegerProp", datatype=db.INTEGER, importance=db.OBLIGATORY)
+ other_type_records = db.Container().extend([
+ db.Record(id=100, name="otherA").add_parent(other_type_rt),
+ db.Record(id=101, name="otherB").add_parent(other_type_rt),
+ db.Record(id=102).add_parent(other_type_rt)
+ ])
+ all_records.extend(other_type_records)
+
+ referencing_type_rt = db.RecordType(name="ReferencingType")
+ referencing_type_rt.add_property(name=other_type_rt.name, datatype=db.LIST(other_type_rt.name))
+ referencing_type_records = db.Container().extend([
+ db.Record(id=103).add_parent(referencing_type_rt),
+ db.Record(id=104, name="referencing").add_parent(referencing_type_rt)
+ ])
+ all_records.extend(referencing_type_records)
+
+ all_files.append(db.File(id=105, name="GenericFile.txt"))
+
+ if query_string == "SELECT name, id FROM RECORD 'OtherType'":
+ return other_type_records
+ elif query_string == "FIND RECORDTYPE WITH name='OtherType'":
+ return _make_unique(other_type_rt, unique)
+ elif query_string == "SELECT name, id FROM RECORD 'ReferencingType'":
+ return referencing_type_records
+ elif query_string == "FIND RECORDTYPE WITH name='ReferencingType'":
+ return _make_unique(referencing_type_rt, unique)
+ elif query_string == "SELECT name, id FROM RECORD 'RT1'":
+ return referencing_type_records # wrong types, but who cares for the test?
+ elif query_string == "FIND RECORDTYPE WITH name='RT1'":
+ return _make_unique(RT1, unique)
+ elif query_string == "FIND RECORDTYPE WITH name='RT21'":
+ return _make_unique(RT21, unique)
+ elif query_string == "FIND RECORDTYPE WITH name='RT31'":
+ return _make_unique(RT31, unique)
+ elif query_string == "SELECT name, id FROM RECORD":
+ return all_records
+ elif query_string == "SELECT name, id FROM FILE":
+ return all_files
+ else:
+ print(f"Query string: {query_string}")
+ if unique is True:
+ return db.Entity()
+ return db.Container()
+
+
+def test_empty_rt():
+
+ rt = db.RecordType(name="Test", description="descr")
+
+ schema = rtjs(rt)
+
+ assert schema["title"] == rt.name
+ assert schema["description"] == rt.description
+ assert len(schema["properties"]) == 0
+ assert len(schema["required"]) == 0
+ assert schema["additionalProperties"] is True
+
+ schema = rtjs(rt, additional_properties=False)
+
+ assert schema["title"] == rt.name
+ assert schema["description"] == rt.description
+ assert len(schema["properties"]) == 0
+ assert len(schema["required"]) == 0
+ assert schema["additionalProperties"] is False
+
+ schema = rtjs(rt, name_property_for_new_records=True,
+ description_property_for_new_records=True)
+
+ assert len(schema["properties"]) == 2
+ assert "name" in schema["properties"]
+ assert "description" in schema["properties"]
+ assert schema["properties"]["name"]["type"] == "string"
+ assert schema["properties"]["description"]["type"] == "string"
+
+
+def test_rt_with_scalar_props():
+
+ rt = db.RecordType(name="Test")
+ rt.add_property(name="SimpleText", datatype=db.TEXT, description="This is a simple text")
+ rt.add_property(name="ObligatoryDatetime", datatype=db.DATETIME, importance=db.OBLIGATORY)
+ rt.add_property(name="JustDateNoTime", datatype=db.DATETIME, description="Only dates, no times")
+ rt.add_property(name="ObligatoryInteger", datatype=db.INTEGER, importance=db.OBLIGATORY)
+ rt.add_property(name="Double", datatype=db.DOUBLE)
+ # Suggested shouldn't influence the result in any way.
+ rt.add_property(name="Boolean", datatype=db.BOOLEAN, importance=db.SUGGESTED)
+
+ schema = rtjs(rt, additional_options_for_text_props={"JustDateNoTime": {"format": "date"}})
+
+ assert "properties" in schema
+ props = schema["properties"]
+ assert len(props) == 6
+ assert "required" in schema
+ assert len(schema["required"]) == 2
+ assert "ObligatoryDatetime" in schema["required"]
+ assert "ObligatoryInteger" in schema["required"]
+
+ assert "SimpleText" in props
+ assert props["SimpleText"]["type"] == "string"
+ assert "format" not in props["SimpleText"]
+ assert "description" in props["SimpleText"]
+ assert props["SimpleText"]["description"] == "This is a simple text"
+
+ assert "ObligatoryDatetime" in props
+ assert "type" not in props["ObligatoryDatetime"]
+ assert "anyOf" in props["ObligatoryDatetime"]
+ assert len(props["ObligatoryDatetime"]["anyOf"]) == 2
+ date_found = 0
+ datetime_found = 0
+ for option in props["ObligatoryDatetime"]["anyOf"]:
+ assert option["type"] == "string"
+ fmt = option["format"]
+ if fmt == "date":
+ date_found += 1
+ if fmt == "date-time":
+ datetime_found += 1
+ assert date_found == 1
+ assert datetime_found == 1
+
+ assert "JustDateNoTime" in props
+ assert props["JustDateNoTime"]["type"] == "string"
+ assert "anyOf" not in props["JustDateNoTime"]
+ assert "pattern" not in props["JustDateNoTime"]
+ assert props["JustDateNoTime"]["format"] == "date"
+ assert props["JustDateNoTime"]["description"] == "Only dates, no times"
+
+ assert "ObligatoryInteger" in props
+ assert props["ObligatoryInteger"]["type"] == "integer"
+
+ assert "Double" in props
+ assert props["Double"]["type"] == "number"
+
+ assert "Boolean" in props
+ assert props["Boolean"]["type"] == "boolean"
+
+ # test validation (we turst the jsonschema.validat function, so only test
+ # some more or less tricky cases with format or required).
+ example = {
+ "SimpleText": "something",
+ "ObligatoryInteger": 23,
+ "ObligatoryDatetime": "1900-01-01T12:34:56.0Z",
+ "JustDateNoTime": "2023-10-13"
+ }
+
+ # We need to explicitly enable the FormatChecker, otherwise format will be
+ # ignored
+ # (https://python-jsonschema.readthedocs.io/en/latest/validate/#validating-formats)
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "SimpleText": "something",
+ "ObligatoryInteger": 23,
+ "ObligatoryDatetime": "1900-01-01",
+ "JustDateNoTime": "2023-10-13"
+ }
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "SimpleText": "something",
+ "ObligatoryDatetime": "1900-01-01T12:34:56.0Z",
+ "JustDateNoTime": "2023-10-13"
+ }
+
+ with raises(ValidationError):
+ # required missing
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "SimpleText": "something",
+ "ObligatoryInteger": 23,
+ "ObligatoryDatetime": "1900-01-01T12:34:56.0Z",
+ "JustDateNoTime": "2023-10-13T23:59:59.123Z"
+ }
+
+ with raises(ValidationError):
+ # date expected in JustDateNoTime, but datetime given
+ validate(example, schema, format_checker=FormatChecker())
+
+
+def test_units():
+
+ rt = db.RecordType()
+ rt.add_property(name="ScalarWithUnit", datatype=db.DOUBLE, unit="m")
+ rt.add_property(name="ListWithUnit", description="This is a list.",
+ datatype=db.LIST(db.DOUBLE), unit="m")
+
+ schema = rtjs(rt, units_in_description=True)
+
+ props = schema["properties"]
+ assert "ScalarWithUnit" in props
+ assert props["ScalarWithUnit"]["type"] == "number"
+ assert "description" in props["ScalarWithUnit"]
+ assert props["ScalarWithUnit"]["description"] == "Unit is m."
+ assert "unit" not in props["ScalarWithUnit"]
+
+ assert "ListWithUnit" in props
+ assert props["ListWithUnit"]["type"] == "array"
+ assert "items" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["items"]["type"] == ["number", "null"]
+ assert "description" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["description"] == "This is a list. Unit is m."
+ assert "unit" not in props["ListWithUnit"]
+
+ schema = rtjs(rt, units_in_description=False)
+
+ props = schema["properties"]
+ assert "ScalarWithUnit" in props
+ assert props["ScalarWithUnit"]["type"] == "number"
+ assert "description" not in props["ScalarWithUnit"]
+ assert "unit" in props["ScalarWithUnit"]
+ assert props["ScalarWithUnit"]["unit"] == "m"
+
+ assert "ListWithUnit" in props
+ assert props["ListWithUnit"]["type"] == "array"
+ assert "items" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["items"]["type"] == ["number", "null"]
+ assert "description" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["description"] == "This is a list."
+ assert "unit" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["unit"] == "m"
+
+
+def test_rt_with_list_props():
+
+ rt = db.RecordType()
+ rt.add_property(name="ListOfIntegers", datatype=db.LIST(
+ db.INTEGER), description="List of integers")
+ rt.add_property(name="ListOfPatterns", datatype=db.LIST(db.TEXT))
+
+ schema = rtjs(rt, additional_options_for_text_props={"ListOfPatterns": {"pattern": "[A-Z]+"}})
+
+ props = schema["properties"]
+
+ assert "ListOfIntegers" in props
+ assert props["ListOfIntegers"]["type"] == "array"
+ assert "items" in props["ListOfIntegers"]
+ assert props["ListOfIntegers"]["items"]["type"] == ["integer", "null"]
+ assert "description" not in props["ListOfIntegers"]["items"]
+ assert props["ListOfIntegers"]["description"] == "List of integers"
+
+ assert "ListOfPatterns" in props
+ assert props["ListOfPatterns"]["type"] == "array"
+ assert "items" in props["ListOfPatterns"]
+ assert props["ListOfPatterns"]["items"]["type"] == ["string", "null"]
+ assert props["ListOfPatterns"]["items"]["pattern"] == "[A-Z]+"
+
+ # Validation
+ example = {
+ "ListOfIntegers": [1, 2, 3],
+ "ListOfPatterns": ["A", "BB", "CCC"]
+ }
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "ListOfIntegers": 1,
+ "ListOfPatterns": ["A", "BB", "CCC"]
+ }
+ with raises(ValidationError):
+ # No list
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "ListOfIntegers": [1, 2, 3],
+ "ListOfPatterns": ["A", "bb", "CCC"]
+ }
+ with raises(ValidationError):
+ # Pattern doesn't match
+ validate(example, schema, format_checker=FormatChecker())
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+@patch("linkahead.cached.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_rt_with_references():
+
+ rt = db.RecordType()
+ rt.add_property(name="RefProp", datatype=db.REFERENCE)
+
+ schema = rtjs(rt)
+ props = schema["properties"]
+ assert "RefProp" in props
+ assert "enum" in props["RefProp"]
+ assert isinstance(props["RefProp"]["enum"], list)
+ assert len(props["RefProp"]["enum"]) == len(
+ db.execute_query("SELECT name, id FROM RECORD")) + len(
+ db.execute_query("SELECT name, id FROM FILE"))
+ assert "oneOf" not in props["RefProp"]
+
+ example = {
+ "RefProp": "otherB"
+ }
+ validate(example, schema)
+ example = {
+ "RefProp": "I don't exist"
+ }
+ with raises(ValidationError):
+ # Wrong enum value
+ validate(example, schema)
+ example = {
+ "RefProp": {
+ "IntegerProp": 12
+ }
+ }
+ with raises(ValidationError):
+ # Can't have objects in generic references
+ validate(example, schema)
+
+ rt = db.RecordType()
+ rt.add_property(name="RefProp", datatype="OtherType")
+ rt.add_property(name="OtherTextProp", datatype=db.TEXT)
+
+ schema = rtjs(rt)
+ props = schema["properties"]
+ assert "RefProp" in props
+ assert "oneOf" in props["RefProp"]
+ assert len(props["RefProp"]["oneOf"]) == 2
+ enum_index = 0
+ if "enum" not in props["RefProp"]["oneOf"][enum_index]:
+ # We can't really require the order here, so we just know that one of
+ # the two elements must be the enum, the other the object.
+ enum_index = 1 - enum_index
+ assert "enum" in props["RefProp"]["oneOf"][enum_index]
+ assert isinstance(props["RefProp"]["oneOf"][enum_index]["enum"], list)
+ assert len(props["RefProp"]["oneOf"][enum_index]["enum"]) == 3
+ assert "otherA" in props["RefProp"]["oneOf"][enum_index]["enum"]
+ assert "otherB" in props["RefProp"]["oneOf"][enum_index]["enum"]
+ assert "102" in props["RefProp"]["oneOf"][enum_index]["enum"]
+ # the other element of oneOf is the OtherType object
+ assert props["RefProp"]["oneOf"][1 - enum_index]["type"] == "object"
+ other_props = props["RefProp"]["oneOf"][1 - enum_index]["properties"]
+ assert "IntegerProp" in other_props
+ assert other_props["IntegerProp"]["type"] == "integer"
+ assert "required" in props["RefProp"]["oneOf"][1 - enum_index]
+ assert len(props["RefProp"]["oneOf"][1 - enum_index]["required"]) == 1
+ assert "IntegerProp" in props["RefProp"]["oneOf"][1 - enum_index]["required"]
+ # The other prop also works as before
+ assert "OtherTextProp" in props
+ assert props["OtherTextProp"]["type"] == "string"
+
+ example = {
+ "RefProp": {
+ "IntegerProp": 12
+ }
+ }
+ validate(example, schema)
+
+ example = {
+ "RefProp": "otherB",
+ "OtherTextProp": "something"
+ }
+ validate(example, schema)
+
+ rt = db.RecordType(name="TestType", description="Some description")
+ rt.add_property(name="RefProp", datatype=db.LIST(db.REFERENCE),
+ description="I'm a list of references.")
+
+ schema = rtjs(rt)
+ assert schema["title"] == rt.name
+ assert schema["description"] == rt.description
+ assert "RefProp" in schema["properties"]
+ ref_prop = schema["properties"]["RefProp"]
+ assert ref_prop["type"] == "array"
+ assert "description" in ref_prop
+ assert ref_prop["description"] == "I'm a list of references."
+ assert "items" in ref_prop
+ items = ref_prop["items"]
+ assert "enum" in items
+ assert isinstance(items["enum"], list)
+ assert len(items["enum"]) == len(
+ db.execute_query("SELECT name, id FROM RECORD")) + len(
+ db.execute_query("SELECT name, id FROM FILE"))
+ assert "oneOf" not in items
+ assert "description" not in items
+
+ example = {
+ "RefProp": "otherB"
+ }
+ with raises(ValidationError):
+ # Should be list but isn't
+ validate(example, schema)
+ example = {
+ "RefProp": ["otherB"]
+ }
+ validate(example, schema)
+ example = {
+ "RefProp": ["otherB", "102", "referencing"]
+ }
+ validate(example, schema)
+
+ rt = db.RecordType()
+ rt.add_property(name="RefProp", datatype=db.LIST("OtherType"))
+
+ schema = rtjs(rt, additional_properties=False, name_property_for_new_records=True,
+ description_property_for_new_records=True)
+ assert schema["additionalProperties"] is False
+ assert "name" in schema["properties"]
+ assert schema["properties"]["name"]["type"] == "string"
+ assert "description" in schema["properties"]
+ assert schema["properties"]["description"]["type"] == "string"
+ assert "RefProp" in schema["properties"]
+ assert schema["properties"]["RefProp"]["type"] == "array"
+ assert "additionalProperties" not in schema["properties"]["RefProp"]
+ assert "items" in schema["properties"]["RefProp"]
+ items = schema["properties"]["RefProp"]["items"]
+ assert "oneOf" in items
+ assert len(items["oneOf"]) == 2
+ # same as above, we can't rely on the order
+ enum_index = 0
+ if "enum" not in items["oneOf"][enum_index]:
+ enum_index = 1 - enum_index
+ assert "enum" in items["oneOf"][enum_index]
+ assert isinstance(items["oneOf"][enum_index]["enum"], list)
+ assert len(items["oneOf"][enum_index]["enum"]) == 3
+ assert "otherA" in items["oneOf"][enum_index]["enum"]
+ assert "otherB" in items["oneOf"][enum_index]["enum"]
+ assert "102" in items["oneOf"][enum_index]["enum"]
+ other_type = items["oneOf"][1 - enum_index]
+ assert other_type["type"] == "object"
+ assert other_type["additionalProperties"] is False
+ assert "IntegerProp" in other_type["properties"]
+ assert len(other_type["required"]) == 1
+ assert "IntegerProp" in other_type["required"]
+
+ example = {
+ "RefProp": ["otherB", "102", "referencing"]
+ }
+ with raises(ValidationError):
+ # Wrong value in enum
+ validate(example, schema)
+ example = {
+ "RefProp": [{"IntegerProp": 12}]
+ }
+ validate(example, schema)
+ example = {
+ "RefProp": [{"IntegerProp": 12, "additionalProperty": "something"}]
+ }
+ with raises(ValidationError):
+ # we have additional_properties=False which propagates to subschemas
+ validate(example, schema)
+ example = {
+ "RefProp": [{"IntegerProp": 12}, "otherB"]
+ }
+ validate(example, schema)
+
+ rt = db.RecordType(name="ReferenceofReferencesType")
+ rt.add_property(name="RefRefProp", datatype="ReferencingType")
+
+ schema = rtjs(rt)
+
+ assert "RefRefProp" in schema["properties"]
+ ref_ref = schema["properties"]["RefRefProp"]
+ assert "oneOf" in ref_ref
+ assert len(ref_ref["oneOf"]) == 2
+ enum_index = 0
+ if "enum" not in ref_ref["oneOf"][enum_index]:
+ enum_index = 1 - enum_index
+ assert len(ref_ref["oneOf"][enum_index]["enum"]) == 2
+ assert "103" in ref_ref["oneOf"][enum_index]["enum"]
+ assert "referencing" in ref_ref["oneOf"][enum_index]["enum"]
+ assert ref_ref["oneOf"][1 - enum_index]["type"] == "object"
+ assert "OtherType" in ref_ref["oneOf"][1 - enum_index]["properties"]
+ assert ref_ref["oneOf"][1 - enum_index]["properties"]["OtherType"]["type"] == "array"
+ items = ref_ref["oneOf"][1 - enum_index]["properties"]["OtherType"]["items"]
+ assert "oneOf" in items
+ assert len(items["oneOf"]) == 2
+ # same as above, we can't rely on the order
+ enum_index = 0
+ if "enum" not in items["oneOf"][enum_index]:
+ enum_index = 1 - enum_index
+ assert "enum" in items["oneOf"][enum_index]
+ assert isinstance(items["oneOf"][enum_index]["enum"], list)
+ assert len(items["oneOf"][enum_index]["enum"]) == 3
+ assert "otherA" in items["oneOf"][enum_index]["enum"]
+ assert "otherB" in items["oneOf"][enum_index]["enum"]
+ assert "102" in items["oneOf"][enum_index]["enum"]
+ other_type = items["oneOf"][1 - enum_index]
+ assert other_type["type"] == "object"
+ assert "IntegerProp" in other_type["properties"]
+ assert len(other_type["required"]) == 1
+ assert "IntegerProp" in other_type["required"]
+
+ example = {
+ "RefRefProp": {
+ "OtherType": [
+ "otherA",
+ {"IntegerProp": 12}
+ ]
+ }
+ }
+ validate(example, schema)
+
+ # Single file and multiple files
+ rt = db.RecordType()
+ rt.add_property(name="FileProp", datatype=db.FILE)
+
+ schema = rtjs(rt)
+ assert schema["properties"]["FileProp"]["type"] == "string"
+ assert schema["properties"]["FileProp"]["format"] == "data-url"
+
+ # wrap in array (cf. https://github.com/rjsf-team/react-jsonschema-form/issues/3957)
+ schema = rtjs(rt, wrap_files_in_objects=True)
+ assert schema["properties"]["FileProp"]["type"] == "array"
+ assert schema["properties"]["FileProp"]["maxItems"] == 1
+ assert "items" in schema["properties"]["FileProp"]
+ items = schema["properties"]["FileProp"]["items"]
+ assert items["type"] == "object"
+ assert len(items["required"]) == 0
+ assert items["additionalProperties"] is False
+ assert len(items["properties"]) == 1
+ assert "file" in items["properties"]
+ assert items["properties"]["file"]["type"] == "string"
+ assert items["properties"]["file"]["format"] == "data-url"
+
+ rt = db.RecordType()
+ rt.add_property(name="FileProp", datatype=db.LIST(db.FILE))
+
+ schema = rtjs(rt)
+ assert schema["properties"]["FileProp"]["type"] == "array"
+ assert schema["properties"]["FileProp"]["items"]["type"] == ["string", "null"]
+ assert schema["properties"]["FileProp"]["items"]["format"] == "data-url"
+
+ # wrap in array (cf. https://github.com/rjsf-team/react-jsonschema-form/issues/3957)
+ print(schema)
+ schema = rtjs(rt, wrap_files_in_objects=True)
+ assert schema["properties"]["FileProp"]["type"] == "array"
+ assert "maxItems" not in schema["properties"]["FileProp"]
+ assert "items" in schema["properties"]["FileProp"]
+ items = schema["properties"]["FileProp"]["items"]
+ assert items["type"] == "object"
+ assert len(items["required"]) == 0
+ assert items["additionalProperties"] is False
+ assert len(items["properties"]) == 1
+ assert "file" in items["properties"]
+ assert items["properties"]["file"]["type"] == "string"
+ assert items["properties"]["file"]["format"] == "data-url"
+
+ # Test reference property
+ model_string = """
+RT1:
+ description: Some recordtype
+RT2:
+ obligatory_properties:
+ prop1:
+ description: Some reference property
+ datatype: RT1
+ """
+ model = parse_model_from_string(model_string)
+ schema = rtjs(model.get_deep("RT2"), no_remote=True)
+ assert json.dumps(schema, indent=2) == """{
+ "type": "object",
+ "required": [
+ "prop1"
+ ],
+ "additionalProperties": true,
+ "title": "RT2",
+ "properties": {
+ "prop1": {
+ "type": "object",
+ "required": [],
+ "additionalProperties": true,
+ "description": "Some reference property",
+ "title": "prop1",
+ "properties": {}
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+
+
+def test_broken():
+
+ rt = db.RecordType()
+ rt.add_property(name="something", datatype=None)
+
+ with raises(ValueError) as ve:
+
+ rtjs(rt)
+ assert str(ve).startswith("Unknown or no property datatype.")
+
+ rt = db.RecordType()
+ rt.add_property(name="MultiProp", datatype=db.INTEGER)
+ rt.add_property(name="MultiProp", datatype=db.INTEGER)
+
+ with raises(NotImplementedError) as nie:
+
+ rtjs(rt)
+ assert "MultiProp" in str(nie)
+ assert str(nie).startswith("Creating a schema for multi-properties is not specified.")
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+@patch("linkahead.cached.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_reference_options():
+ """Testing miscellaneous options.
+ """
+
+ model_str = """
+RT1:
+ description: some description
+ obligatory_properties:
+ some_date:
+ datatype: DATETIME
+ description: Just some date
+RT2:
+ obligatory_properties:
+ RT1:
+
+RT3:
+ obligatory_properties:
+ RT1_prop:
+ datatype: RT1
+ description: property description
+ """
+ model = parse_model_from_string(model_str)
+ # First test: without reference
+ rt1_dict = rtjs(model.get_deep("RT1"))
+ assert json.dumps(rt1_dict, indent=2) == """{
+ "type": "object",
+ "required": [
+ "some_date"
+ ],
+ "additionalProperties": true,
+ "description": "some description",
+ "title": "RT1",
+ "properties": {
+ "some_date": {
+ "description": "Just some date",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+ # Second test: with reference
+ rt2_deep = model.get_deep("RT2")
+ rt2_dict = rtjs(rt2_deep)
+ assert json.dumps(rt2_dict, indent=2) == """{
+ "type": "object",
+ "required": [
+ "RT1"
+ ],
+ "additionalProperties": true,
+ "title": "RT2",
+ "properties": {
+ "RT1": {
+ "description": "some description",
+ "oneOf": [
+ {
+ "title": "Existing entries",
+ "enum": [
+ "103",
+ "referencing"
+ ]
+ },
+ {
+ "type": "object",
+ "required": [
+ "some_date"
+ ],
+ "additionalProperties": true,
+ "description": "some description",
+ "title": "Create new",
+ "properties": {
+ "some_date": {
+ "description": "Just some date",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+
+ # Third test: Reference prop shall be only existing references, no option to create new ones.
+ rt2_dict = rtjs(model.get_deep("RT2"), do_not_create=["RT1"])
+ assert json.dumps(rt2_dict, indent=2) == """{
+ "type": "object",
+ "required": [
+ "RT1"
+ ],
+ "additionalProperties": true,
+ "title": "RT2",
+ "properties": {
+ "RT1": {
+ "description": "some description",
+ "enum": [
+ "103",
+ "referencing"
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+ # No effect of do_not_create (real property name should be used)
+ rt3_dict = rtjs(model.get_deep("RT3"), do_not_create=["RT1"])
+ rt1_prop = rt3_dict["properties"]["RT1_prop"]
+ assert rt1_prop["description"] == "property description"
+ assert "oneOf" in rt1_prop.keys()
+ assert "enum" not in rt1_prop.keys()
+
+ # Now we use the real property name
+ rt3_dict = rtjs(model.get_deep("RT3"), do_not_create=["RT1_prop"])
+ rt1_prop = rt3_dict["properties"]["RT1_prop"]
+ assert rt1_prop["description"] == "property description"
+ assert "oneOf" not in rt1_prop.keys()
+ assert "enum" in rt1_prop.keys()
+ assert rt1_prop["enum"][0] == "103"
+
+
+def test_schema_modification():
+ """Testing functions which modify json schema dicts:
+
+- make_array()
+- merge_schemas().
+ """
+
+ model_str = """
+some_date:
+ datatype: DATETIME
+RT1:
+ obligatory_properties:
+ some_date:
+
+some_text:
+ datatype: TEXT
+RT2:
+ obligatory_properties:
+ some_text:
+ """
+ model = parse_model_from_string(model_str)
+ schema_RT1 = rtjs(model.get_deep("RT1"), additional_properties=False)
+ schema_RT2 = rtjs(model.get_deep("RT2"), additional_properties=False)
+
+ # Merge the schemata
+ merged_list = jsex.merge_schemas([schema_RT1, schema_RT2])
+ with raises(ValidationError):
+ validate({}, merged_list)
+ assert merged_list["type"] == "object"
+ assert merged_list["properties"]["RT1"]["title"] == "RT1"
+ assert merged_list["properties"]["RT2"]["properties"]["some_text"]["type"] == "string"
+
+ merged_dict = jsex.merge_schemas({"schema1": schema_RT1, "schema2": schema_RT2})
+ with raises(ValidationError):
+ validate({}, merged_dict)
+ assert merged_dict["type"] == "object"
+ assert merged_dict["properties"]["schema1"]["title"] == "RT1"
+ assert merged_dict["properties"]["schema2"]["properties"]["some_text"]["type"] == "string"
+
+ # Make an array
+ array = jsex.make_array(schema_RT1)
+ with raises(ValidationError):
+ validate({}, array)
+ assert array["type"] == "array"
+ assert array["items"] == schema_RT1
+
+
+def test_inheritance():
+ """Test data models with inherited properties."""
+ model_str = """
+some_date:
+ datatype: DATETIME
+RT1:
+ obligatory_properties:
+ some_date:
+RT2:
+ inherit_from_suggested:
+ - RT1
+ """
+ model = parse_model_from_string(model_str)
+ rt2_deep = model.get_deep("RT2")
+ assert "some_date" in [prop.name for prop in rt2_deep.properties]
+
+ model_str = """
+RT1:
+ obligatory_properties:
+ RT2:
+RT2:
+ inherit_from_suggested:
+ - RT1
+RT3:
+ inherit_from_suggested:
+ - RT4
+RT4:
+ inherit_from_suggested:
+ - RT3
+RT5:
+ inherit_from_suggested:
+ - RT5
+ """
+ model = parse_model_from_string(model_str)
+ # This must not lead to an infinite recursion
+ rt1_deep = model.get_deep("RT1")
+ rt2_deep = model.get_deep("RT2")
+ assert rt2_deep.get_property("RT2").name == rt1_deep.get_property("RT2").name
+ rt3_deep = model.get_deep("RT3")
+ assert rt3_deep.get_parents()[0].name == "RT4"
+ rt4_deep = model.get_deep("RT4")
+ assert rt4_deep.get_parents()[0].name == "RT3"
+ rt5_deep = model.get_deep("RT5")
+ assert rt5_deep.get_parents()[0].name == "RT5"
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+@patch("linkahead.cached.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_empty_retrieve():
+ """Special case: ``do_not_retrieve`` is set, or the retrieve result is empty."""
+ model_str = """
+RT1:
+ description: Some text.
+RT2:
+ obligatory_properties:
+ RT1:
+# some_text:
+# datatype: TEXT
+NoRecords:
+ description: A RecordType without Records.
+ recommended_properties:
+ some_text:
+ datatype: TEXT
+RT3:
+ obligatory_properties:
+ NoRecords:
+ """
+ model = parse_model_from_string(model_str)
+ schema_default = rtjs(model.get_deep("RT2"))
+ assert "oneOf" in schema_default["properties"]["RT1"]
+ assert any([el.get("title") == "Existing entries" for el in
+ schema_default["properties"]["RT1"]["oneOf"]])
+
+ schema_noexist = rtjs(model.get_deep("RT3"))
+ assert schema_noexist["properties"]["NoRecords"].get("type") == "object"
+
+ schema_noexist_noremote = rtjs(model.get_deep("RT3"), no_remote=True)
+ assert schema_noexist_noremote["properties"]["NoRecords"].get("type") == "object"
+ assert (schema_noexist_noremote["properties"]["NoRecords"].get("properties")
+ == OrderedDict([('some_text', {'type': 'string'})]))
+
+ uischema = {}
+ schema_noexist_noretrieve = rtjs(model.get_deep("RT2"), do_not_retrieve=["RT1"],
+ rjsf=uischema)
+ assert schema_noexist_noretrieve["properties"]["RT1"].get("type") == "object"
+ assert "some_date" in schema_noexist_noretrieve["properties"]["RT1"].get("properties")
+ assert not uischema
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+@patch("linkahead.cached.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_multiple_choice():
+ """Multiple choice is mostyly a matter of UI."""
+ model_str = """
+RT1:
+RT21:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+RT3:
+ obligatory_properties:
+ RT21:
+RT4:
+ obligatory_properties:
+ RT21:
+ datatype: LIST<RT21>
+ """
+ model = parse_model_from_string(model_str)
+ # generate a multiple choice, in first level
+ schema, uischema = rtjs(model.get_deep("RT21"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ assert schema["properties"]["RT1"]["uniqueItems"] is True
+ assert str(uischema) == "{'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}"
+
+ # second level
+ schema, uischema = rtjs(model.get_deep("RT3"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ assert schema["properties"]["RT21"]["properties"]["RT1"]["uniqueItems"] is True
+ assert (str(uischema)
+ == "{'RT21': {'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}}")
+
+ # second level with lists
+ schema, uischema = rtjs(model.get_deep("RT4"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ assert schema["properties"]["RT21"]["items"]["properties"]["RT1"]["uniqueItems"] is True
+ assert (str(uischema) ==
+ "{'RT21': {'items': {'RT1': {'ui:widget': 'checkboxes', "
+ "'ui:inline': True}}}}")
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+@patch("linkahead.cached.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_uischema():
+ model_str = """
+RT1:
+RT2:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+RT3:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+ """
+ model = parse_model_from_string(model_str)
+ schema_2, uischema_2 = rtjs(model.get_deep("RT2"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ schema_3, uischema_3 = rtjs(model.get_deep("RT3"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+
+ # Merging #################################################################
+ # Using dictionaries
+ schemas_dict = {"schema_2": schema_2, "schema_3": schema_3}
+ uischemas_dict = {"schema_2": uischema_2, "schema_3": uischema_3}
+ merged_dict, merged_dict_ui = jsex.merge_schemas(schemas_dict, uischemas_dict)
+ assert merged_dict_ui["schema_2"] == merged_dict_ui["schema_3"]
+ assert (str(merged_dict_ui["schema_2"])
+ == "{'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}")
+
+ # Using lists
+ schemas_list = [schema_2, schema_3]
+ uischemas_list = [uischema_2, uischema_3]
+ merged_list, merged_list_ui = jsex.merge_schemas(schemas_list, uischemas_list)
+ assert merged_list["properties"]["RT2"] == merged_dict["properties"]["schema_2"]
+ assert merged_list_ui["RT2"] == merged_list_ui["RT3"]
+ assert merged_list_ui["RT2"] == merged_dict_ui["schema_2"]
+
+ # Asserting failures
+ with raises(ValueError):
+ jsex.merge_schemas(schemas_dict, uischemas_list)
+ with raises(ValueError):
+ jsex.merge_schemas(schemas_list, uischemas_dict)
+
+ # Arraying ################################################################
+ array2, array2_ui = jsex.make_array(schema_2, uischema_2)
+ assert array2["items"] == schema_2
+ assert array2_ui["items"] == uischema_2
+ assert (str(array2_ui["items"])
+ == "{'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}")
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+@patch("linkahead.cached.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_schema_customization_with_dicts():
+ """Testing the ``additional_json_schema`` and ``additional_ui_schema`` parameters."""
+ model_str = """
+RT1:
+RT21:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+ text:
+ datatype: TEXT
+ description: Some description
+RT3:
+ obligatory_properties:
+ number:
+ datatype: INTEGER
+ """
+ model = parse_model_from_string(model_str)
+
+ custom_schema = {
+ "RT21": {
+ "minProperties": 2,
+ },
+ "text": {
+ "format": "email",
+ "description": "Better description.",
+ },
+ "number": {
+ "minimum": 0,
+ "exclusiveMaximum": 100,
+ },
+ }
+
+ custom_ui_schema = {
+ "text": {
+ "ui:help": "Hint: keep it short.",
+ "ui:widget": "password",
+ },
+ "number": {
+ "ui:order": 2,
+ }
+ }
+
+ schema_21, uischema_21 = rtjs(model.get_deep("RT21"), additional_properties=False,
+ do_not_create=["RT1"], rjsf=True)
+ assert len(uischema_21) == 0
+ assert schema_21["properties"]["text"]["description"] == "Some description"
+ assert "format" not in schema_21["properties"]["text"]
+
+ schema_21, uischema_21 = rtjs(model.get_deep("RT21"), additional_properties=False,
+ additional_json_schema=custom_schema,
+ additional_ui_schema=custom_ui_schema, do_not_create=["RT1"],
+ rjsf=True)
+ assert (str(uischema_21)
+ == "{'text': {'ui:help': 'Hint: keep it short.', 'ui:widget': 'password'}}")
+ assert schema_21["properties"]["text"]["description"] == "Better description."
+ assert schema_21["properties"]["text"].get("format") == "email"
+ assert schema_21.get("minProperties") == 2
+
+ schema_3, uischema_3 = rtjs(model.get_deep("RT3"), additional_properties=False,
+ additional_json_schema=custom_schema,
+ additional_ui_schema=custom_ui_schema, rjsf=True)
+ assert (json.dumps(schema_3["properties"]["number"]) ==
+ '{"type": "integer", "minimum": 0, "exclusiveMaximum": 100}')
+ assert (str(uischema_3) == "{'number': {'ui:order': 2}}")
diff --git a/unittests/test_json_schema_model_parser.py b/unittests/test_json_schema_model_parser.py
index 7f47890f413dce5511cd498fe802e03a1af3be70..a991076e6a1e1a3e92cafc7f1bb88b42b4b2ab3d 100644
--- a/unittests/test_json_schema_model_parser.py
+++ b/unittests/test_json_schema_model_parser.py
@@ -103,7 +103,7 @@ def test_datamodel_with_atomic_properties():
assert isinstance(rt2, db.RecordType)
assert rt2.name == "Dataset2"
assert not rt2.description
- assert len(rt2.get_properties()) == 5
+ assert len(rt2.get_properties()) == 6
date_prop = rt2.get_property("date")
assert date_prop.datatype == db.DATETIME
@@ -121,6 +121,9 @@ def test_datamodel_with_atomic_properties():
float_prop2 = rt2.get_property("number_prop")
assert float_prop.datatype == float_prop2.datatype
+ null_prop = rt2.get_property("null_prop")
+ assert null_prop.datatype == db.TEXT
+
def test_required_no_list():
"""Exception must be raised when "required" is not a list."""
@@ -164,7 +167,7 @@ def test_enum():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["license"])
+ assert model[name].has_parent(model["license"], retrieve=False)
# Also allow enums with non-string types
number_enums = ["1.1", "2.2", "3.3"]
@@ -181,7 +184,7 @@ def test_enum():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["number_enum"])
+ assert model[name].has_parent(model["number_enum"], retrieve=False)
@pytest.mark.xfail(reason="Don't allow integer enums until https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/224 has been fixed")
@@ -207,7 +210,7 @@ def test_int_enum():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["int_enum"])
+ assert model[name].has_parent(model["int_enum"], retrieve=False)
def test_references():
@@ -339,7 +342,7 @@ def test_list():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["license"])
+ assert model[name].has_parent(model["license"], retrieve=False)
def test_name_property():
@@ -356,3 +359,130 @@ def test_name_property():
assert str(err.value).startswith(
"The 'name' property must be string-typed, otherwise it cannot be identified with CaosDB's "
"name property.")
+
+
+def test_no_toplevel_entity():
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_no_toplevel_entity.schema.json"), top_level_recordtype=False)
+
+ assert "Dataset1" in model
+ rt1 = model["Dataset1"]
+
+ assert rt1.name == "Dataset1"
+ assert rt1.description == "Some description"
+ assert len(rt1.get_properties()) == 4
+
+ assert rt1.get_property("title") is not None
+ assert rt1.get_property("campaign") is not None
+ assert rt1.get_property("number_prop") is not None
+ assert rt1.get_property("user_id") is not None
+
+ title_prop = rt1.get_property("title")
+ assert title_prop.datatype == db.TEXT
+ assert rt1.get_importance(title_prop.name) == db.OBLIGATORY
+
+ campaign_prop = rt1.get_property("campaign")
+ assert campaign_prop.datatype == db.TEXT
+ assert rt1.get_importance(campaign_prop.name) == db.RECOMMENDED
+
+ float_prop = rt1.get_property("number_prop")
+ assert float_prop.datatype == db.DOUBLE
+ assert rt1.get_importance(float_prop.name) == db.OBLIGATORY
+
+ uid_prop = rt1.get_property("user_id")
+ assert uid_prop.datatype == db.TEXT
+ assert rt1.get_importance(uid_prop.name) == db.RECOMMENDED
+
+ # pattern properties without top-level entity:
+ assert "__PatternEntry_1" in model
+ assert "__PatternEntry_2" in model
+
+ pattern_boolean_rt = model["__PatternEntry_1"]
+ assert "pattern: " in pattern_boolean_rt.description
+ assert len(pattern_boolean_rt.properties) == 2
+ pp = pattern_boolean_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_boolean_rt.get_importance(pp.name) == db.OBLIGATORY
+ value_prop = pattern_boolean_rt.get_property("__PatternEntry_1_value")
+ assert value_prop.datatype == db.BOOLEAN
+
+ pattern_object_rt = model["__PatternEntry_2"]
+ assert "pattern: " in pattern_object_rt.description
+ assert len(pattern_object_rt.properties) == 2
+ pp = pattern_object_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_object_rt.get_importance(pp.name) == db.OBLIGATORY
+ date_id_prop = pattern_object_rt.get_property("date_id")
+ assert date_id_prop.datatype == db.TEXT
+
+
+def test_missing_array_items():
+
+ # strict behavior
+ with pytest.raises(JsonSchemaDefinitionError) as err:
+ parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_missing_array_items.schema.json"))
+
+ assert "{'type': 'array'}" in str(err)
+
+ # ignore all problems, so a RT is created that does not have the property
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_missing_array_items.schema.json"), ignore_unspecified_array_items=True)
+ assert "something_with_missing_array_items" in model
+ rt = model["something_with_missing_array_items"]
+ assert isinstance(rt, db.RecordType)
+ assert rt.get_property("missing") is None
+
+ # specify the type:
+ type_dict = {"missing": db.FILE}
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_missing_array_items.schema.json"), types_for_missing_array_items=type_dict)
+ assert "something_with_missing_array_items" in model
+ rt = model["something_with_missing_array_items"]
+ assert rt.get_property("missing") is not None
+ assert rt.get_property("missing").datatype == db.LIST(db.FILE)
+
+
+def test_pattern_properties():
+
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_pattern_properties.schema.json"))
+
+ assert "Dataset" in model
+ rt1 = model["Dataset"]
+ assert len(rt1.properties) == 2
+ for name in ["DatasetEntry_1", "DatasetEntry_2"]:
+ assert rt1.get_property(name) is not None
+ assert rt1.get_property(name).is_reference()
+
+ pattern_boolean_rt = model["DatasetEntry_1"]
+ assert "pattern: " in pattern_boolean_rt.description
+ assert len(pattern_boolean_rt.properties) == 2
+ pp = pattern_boolean_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_boolean_rt.get_importance(pp.name) == db.OBLIGATORY
+ value_prop = pattern_boolean_rt.get_property("DatasetEntry_1_value")
+ assert value_prop.datatype == db.BOOLEAN
+
+ pattern_object_rt = model["DatasetEntry_2"]
+ assert "pattern: " in pattern_object_rt.description
+ assert len(pattern_object_rt.properties) == 2
+ pp = pattern_object_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_object_rt.get_importance(pp.name) == db.OBLIGATORY
+ date_id_prop = pattern_object_rt.get_property("date_id")
+ assert date_id_prop.datatype == db.TEXT
+
+ assert "Dataset2" in model
+ rt2 = model["Dataset2"]
+ assert len(rt2.properties) == 2
+ # This has been tested elsewhere, just make sure that it is properly created
+ # in the presence of pattern properties, too.
+ assert rt2.get_property("datetime") is not None
+
+ assert rt2.get_property("Literally anything") is not None
+ assert rt2.get_property("Literally anything").is_reference()
+
+ pattern_named_rt = model["Literally anything"]
+ assert len(pattern_named_rt.properties) == 1
+ assert pattern_named_rt.get_property("__matched_pattern") is not None
diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py
index dbf593de4b63e031777c109c26b971171e660638..9b1ac11b7c9ed5251f05c921fb08f7c42079c91a 100644
--- a/unittests/test_table_converter.py
+++ b/unittests/test_table_converter.py
@@ -27,6 +27,7 @@ from tempfile import NamedTemporaryFile
import caosdb as db
import pandas as pd
from caosdb.apiutils import compare_entities
+from numpy import nan
from caosadvancedtools.table_converter import (from_table, from_tsv, to_table,
to_tsv)
@@ -42,7 +43,8 @@ class TableTest(unittest.TestCase):
def test_empty(self):
c = db.Container()
- self.assertRaises(ValueError, to_table, c)
+ df = to_table(c)
+ assert df.shape == (0, 0)
def test_different_props(self):
r1 = db.Record()
@@ -65,6 +67,36 @@ class TableTest(unittest.TestCase):
c.extend([r1, r2])
self.assertRaises(ValueError, to_table, c)
+ def test_list(self):
+ r1 = db.Record()
+ r1.add_parent("no1")
+ r1.add_property("p1", value=1)
+ r1.add_property("p3", value=23)
+ r1.add_property("p4", value=[1])
+ r2 = db.Record()
+ r2.add_parent("no1")
+ r2.add_property("p1")
+ r2.add_property("p2", value=[20, 21])
+ r2.add_property("p3", value=[30, 31])
+ r2.add_property("p4", value=[40.0, 41.0])
+ r3 = db.Record()
+ r3.add_parent("no1")
+ r3.add_property("p5", value=[50, 51])
+ c = db.Container()
+ c.extend([r1, r2, r3])
+ result = to_table(c)
+ # NaN is hard to compare, so we replace it by -999
+ # autopep8: off
+ assert result.replace(to_replace=nan, value=-999).to_dict() == {
+ 'p1': {0: 1, 1: -999, 2: -999}, # noqa: E202
+ 'p3': {0: 23, 1: [30, 31], 2: -999}, # noqa: E202
+ 'p4': {0: [1], 1: [40.0, 41.0], 2: -999}, # noqa: E202
+ 'p2': {0: -999, 1: [20, 21], 2: -999}, # noqa: E202
+ 'p5': {0: -999, 1: -999, 2: [50, 51]}
+ }
+ # autopep8: on
+ assert list(result.dtypes) == [float, object, object, object, object]
+
class FromTsvTest(unittest.TestCase):
def test_basic(self):
diff --git a/unittests/test_table_importer.py b/unittests/test_table_importer.py
index 70f0f87f8706d72c386b18f54b7a9a10908eb477..599ea535d95d0b6c1216a935813d71c8e90c1d3b 100644
--- a/unittests/test_table_importer.py
+++ b/unittests/test_table_importer.py
@@ -41,6 +41,16 @@ from caosadvancedtools.table_importer import (CSVImporter, TableImporter,
from test_utils import BaseMockUpTest
+# For testing the table importer
+IMPORTER_KWARGS = dict(
+ converters={'c': float, 'd': yes_no_converter, 'x': float}, # x does not exist
+ datatypes={'a': str, 'b': int, 'float': float, 'x': int}, # x does not exist
+ obligatory_columns=['a', 'b'], unique_keys=[('a', 'b')],
+ existing_columns=['e'],
+)
+VALID_DF = pd.DataFrame(
+ [['a', 1, 2.0, 'yes', np.nan]], columns=['a', 'b', 'c', 'd', 'e'])
+
class ConverterTest(unittest.TestCase):
def test_yes_no(self):
@@ -143,22 +153,16 @@ class ConverterTest(unittest.TestCase):
class TableImporterTest(unittest.TestCase):
def setUp(self):
- self.importer_kwargs = dict(
- converters={'c': float, 'd': yes_no_converter},
- datatypes={'a': str, 'b': int},
- obligatory_columns=['a', 'b'], unique_keys=[('a', 'b')])
- self.valid_df = pd.DataFrame(
- [['a', 1, 2.0, 'yes']], columns=['a', 'b', 'c', 'd'])
+ self.importer_kwargs = IMPORTER_KWARGS
+ self.valid_df = VALID_DF
def test_missing_col(self):
- # check missing from converters
- df = pd.DataFrame(columns=['a', 'b', 'c'])
- importer = TableImporter(**self.importer_kwargs)
- self.assertRaises(ValueError, importer.check_columns, df)
- # check missing from datatypes
- df = pd.DataFrame(columns=['a', 'd', 'c'])
+ # check missing from existing
+ df = pd.DataFrame(columns=['a', 'b'])
importer = TableImporter(**self.importer_kwargs)
- self.assertRaises(ValueError, importer.check_columns, df)
+ with pytest.raises(DataInconsistencyError) as die:
+ importer.check_columns(df)
+ assert "Column 'e' missing" in str(die.value)
# check valid
importer.check_columns(self.valid_df)
@@ -177,12 +181,47 @@ class TableImporterTest(unittest.TestCase):
self.assertEqual(df_new.shape[1], 4)
self.assertEqual(df_new.iloc[0].b, 5)
+ # check that missing array-valued fields are detected correctly:
+ df = pd.DataFrame([[[None, None], 4, 2.0, 'yes'],
+ ['b', 5, 3.0, 'no']],
+ columns=['a', 'b', 'c', 'd'])
+ df_new = importer.check_missing(df)
+ self.assertEqual(df_new.shape[0], 1)
+ self.assertEqual(df_new.shape[1], 4)
+ self.assertEqual(df_new.iloc[0].b, 5)
+
def test_wrong_datatype(self):
importer = TableImporter(**self.importer_kwargs)
- df = pd.DataFrame([[None, np.nan, 2.0, 'yes'],
+ df = pd.DataFrame([[1234, 0, 2.0, 3, 'yes'],
+ [5678, 1, 2.0, 3, 'yes']],
+ columns=['a', 'b', 'c', 'float', 'd'])
+ # wrong datatypes before
+ assert df["a"].dtype == int
+ assert df["float"].dtype == int
+ # strict = False by default, so this shouldn't raise an error
+ importer.check_datatype(df)
+ # The types should be correct now.
+ assert df["a"].dtype == pd.StringDtype
+ assert df["float"].dtype == float
+
+ # Resetting `df` since check_datatype may change datatypes
+ df = pd.DataFrame([[None, 0, 2.0, 'yes'],
[5, 1, 2.0, 'yes']],
columns=['a', 'b', 'c', 'd'])
- self.assertRaises(DataInconsistencyError, importer.check_datatype, df)
+ # strict=True, so number in str column raises an error
+ self.assertRaises(DataInconsistencyError, importer.check_datatype, df, None, True)
+
+ df = pd.DataFrame([[0],
+ [1]],
+ columns=['float'])
+ # strict=True, so int in float column raises an error
+ self.assertRaises(DataInconsistencyError, importer.check_datatype, df, None, True)
+
+ # This is always wrong (float in int column)
+ df = pd.DataFrame([[None, np.nan, 2.0, 'yes'],
+ [5, 1.7, 2.0, 'yes']],
+ columns=['a', 'b', 'c', 'd'])
+ self.assertRaises(DataInconsistencyError, importer.check_datatype, df, None, False)
def test_unique(self):
importer = TableImporter(**self.importer_kwargs)
@@ -193,6 +232,35 @@ class TableImporterTest(unittest.TestCase):
self.assertEqual(df_new.shape[0], 1)
+def test_check_dataframe_existing_obligatory_columns(caplog):
+ """Needs caplog so remove from above class."""
+ # stricter test case; column 'a' must exist and have a value
+ strict_kwargs = IMPORTER_KWARGS.copy()
+ strict_kwargs["existing_columns"].append('a')
+
+ importer = TableImporter(**strict_kwargs)
+
+ # the valid df is still valid, since 'a' has a value
+ importer.check_dataframe(VALID_DF)
+
+ # Now 'a' doesn't
+ df_missing_a = pd.DataFrame(
+ [[np.nan, 1, 2.0, 'yes', 'e']], columns=['a', 'b', 'c', 'd', 'e'])
+
+ new_df = importer.check_dataframe(df_missing_a)
+ # Column is removed and a warning is in the logger:
+ assert new_df.shape[0] == 0
+ assert "Required information is missing (a) in 1. row" in caplog.text
+
+ df_missing_c = pd.DataFrame(
+ [['a', 1, 'yes', np.nan]], columns=['a', 'b', 'd', 'e'])
+ new_df = importer.check_dataframe(df_missing_c)
+ assert new_df.shape[0] == 1
+ assert new_df.shape[1] == 4
+
+ caplog.clear()
+
+
class XLSImporterTest(TableImporterTest):
def test_full(self):
""" test full run with example data """
@@ -233,6 +301,30 @@ class CSVImporterTest(TableImporterTest):
importer = CSVImporter(**self.importer_kwargs)
importer.read_file(tmp.name)
+ def test_with_generous_datatypes(self):
+ """Same as above but check that values are converted as expected."""
+ tmp = NamedTemporaryFile(delete=False, suffix=".csv")
+ tmp.close()
+ self.valid_df.to_csv(tmp.name)
+ # Copy and use float for columns with integer values, string for columns
+ # with numeric values
+ kwargs = self.importer_kwargs.copy()
+ kwargs["datatypes"] = {
+ 'a': str,
+ 'b': float,
+ 'c': str
+ }
+ importer = CSVImporter(**kwargs)
+ importer.read_file(tmp.name)
+
+ kwargs["datatypes"] = {
+ 'a': str,
+ 'b': str,
+ 'c': str
+ }
+ importer = CSVImporter(**kwargs)
+ importer.read_file(tmp.name)
+
class TSVImporterTest(TableImporterTest):
def test_full(self):
diff --git a/unittests/test_utils.py b/unittests/test_utils.py
index 7369931799b00eba5a835458a6fad474de1d9039..468e9200de723c65c75e21912b5b3940d758821c 100644
--- a/unittests/test_utils.py
+++ b/unittests/test_utils.py
@@ -26,7 +26,7 @@ from tempfile import NamedTemporaryFile
import caosdb as db
from caosadvancedtools.utils import (check_win_path, get_referenced_files,
- string_to_person)
+ string_to_person, create_entity_link)
from caosdb import RecordType, configure_connection, get_config
from caosdb.connection.mockup import MockUpResponse, MockUpServerConnection
from caosdb.exceptions import TransactionError
@@ -140,3 +140,8 @@ class PathTest(unittest.TestCase):
assert check_win_path(r"C:\hallo")
assert check_win_path(r"\hallo")
assert not check_win_path("/hallo")
+
+
+class EntityLinkTest(unittest.TestCase):
+ def test_link(self):
+ assert "<a href='/Entity/1'>a</a>" == create_entity_link(db.Entity(id=1, name='a'))
diff --git a/unittests/test_yaml_model_parser.py b/unittests/test_yaml_model_parser.py
index 6cdea7922a8503be082e8947edecd7e8c849730b..97c3450f654e7b836734335cafac37adc6e700bb 100644
--- a/unittests/test_yaml_model_parser.py
+++ b/unittests/test_yaml_model_parser.py
@@ -1,9 +1,27 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
import unittest
from datetime import date
from tempfile import NamedTemporaryFile
-from pytest import deprecated_call, raises
+from pytest import raises, mark
-import caosdb as db
+import linkahead as db
from caosadvancedtools.models.parser import (TwiceDefinedException,
YamlDefinitionError,
parse_model_from_string,
@@ -284,10 +302,12 @@ A:
def test_reference_property(self):
"""Test correct creation of reference property using an RT."""
- modeldef = """A:
+ modeldef = """
+A:
recommended_properties:
ref:
datatype: LIST<A>
+ description: new description
"""
model = parse_model_from_string(modeldef)
self.assertEqual(len(model), 2)
@@ -297,6 +317,7 @@ A:
elif key == "ref":
self.assertTrue(isinstance(value, db.Property))
self.assertEqual(value.datatype, "LIST<A>")
+ assert value.description == "new description"
class ExternTest(unittest.TestCase):
@@ -340,6 +361,35 @@ A:
assert "line {}".format(line) in yde.exception.args[0]
+def test_existing_model():
+ """Parsing more than one model may require to append to existing models."""
+ model_str_1 = """
+A:
+ obligatory_properties:
+ number:
+ datatype: INTEGER
+ """
+ model_str_2 = """
+B:
+ obligatory_properties:
+ A:
+ """
+ model_1 = parse_model_from_string(model_str_1)
+ model_2 = parse_model_from_string(model_str_2, existing_model=model_1)
+ for ent in ["A", "B", "number"]:
+ assert ent in model_2
+
+ model_str_redefine = """
+number:
+ datatype: DOUBLE
+ description: Hello number!
+ """
+ model_redefine = parse_model_from_string(model_str_redefine, existing_model=model_1)
+ print(model_redefine)
+ assert model_redefine["number"].description == "Hello number!"
+ assert model_redefine["number"].datatype == db.INTEGER # FIXME Shouldn't this be DOUBLE?
+
+
def test_define_role():
model = """
A:
@@ -477,7 +527,7 @@ F:
def test_issue_36():
- """Test whether the `parent` keyword is deprecated.
+ """Test whether the `parent` keyword is removed.
See https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36.
@@ -500,14 +550,92 @@ R3:
inherit_from_obligatory:
- R1
"""
- with deprecated_call():
- # Check whether this is actually deprecated
+ with raises(ValueError) as ve:
+ # The keyword has been removed, so it should raise a regular ValueError.
model = parse_model_from_string(model_string)
- assert "R3" in model
- r3 = model["R3"]
- assert isinstance(r3, db.RecordType)
- for par in ["R1", "R2"]:
- # Until removal, both do the same
- assert has_parent(r3, par)
- assert r3.get_parent(par)._flags["inheritance"] == db.OBLIGATORY
+ assert "invalid keyword" in str(ve.value)
+ assert "parent" in str(ve.value)
+
+
+def test_yaml_error():
+ """Testing error while parsing a yaml.
+ """
+
+ with raises(ValueError, match=r"line 2: .*"):
+ parse_model_from_yaml("unittests/models/model_invalid.yml")
+
+
+def test_inherit_error():
+ """Must fail with an understandable exception."""
+ model_string = """
+prop1:
+ inherit_from_obligatory: prop2
+ """
+ with raises(YamlDefinitionError,
+ match=r"Parents must be a list but is given as string: prop1 > prop2"):
+ parse_model_from_string(model_string)
+
+
+@mark.xfail(reason="""Issue is
+ https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/57""")
+def test_inherit_properties():
+ # TODO Is not even specified yet.
+ model_string = """
+prop1:
+ datatype: DOUBLE
+prop2:
+# role: Property
+ inherit_from_obligatory:
+ - prop1
+ """
+ model = parse_model_from_string(model_string)
+ prop2 = model["prop2"]
+ assert prop2.role == "Property"
+
+
+def test_fancy_yaml():
+ """Testing aliases and other fancy YAML features."""
+ # Simple aliasing
+ model_string = """
+foo:
+ datatype: INTEGER
+RT1:
+ obligatory_properties: &RT1_oblig
+ foo:
+RT2:
+ obligatory_properties: *RT1_oblig
+ """
+ model = parse_model_from_string(model_string)
+ assert str(model) == """{'foo': <Property name="foo" datatype="INTEGER"/>
+, 'RT1': <RecordType name="RT1">
+ <Property name="foo" importance="OBLIGATORY" flag="inheritance:FIX"/>
+</RecordType>
+, 'RT2': <RecordType name="RT2">
+ <Property name="foo" importance="OBLIGATORY" flag="inheritance:FIX"/>
+</RecordType>
+}"""
+
+ # Aliasing with override
+ model_string = """
+foo:
+ datatype: INTEGER
+RT1:
+ obligatory_properties: &RT1_oblig
+ foo:
+RT2:
+ obligatory_properties:
+ <<: *RT1_oblig
+ bar:
+ """
+ model = parse_model_from_string(model_string)
+ assert str(model) == """{'foo': <Property name="foo" datatype="INTEGER"/>
+, 'RT1': <RecordType name="RT1">
+ <Property name="foo" importance="OBLIGATORY" flag="inheritance:FIX"/>
+</RecordType>
+, 'RT2': <RecordType name="RT2">
+ <Property name="foo" importance="OBLIGATORY" flag="inheritance:FIX"/>
+ <Property name="bar" importance="OBLIGATORY" flag="inheritance:FIX"/>
+</RecordType>
+, 'bar': <RecordType name="bar"/>
+}"""
diff --git a/utils/branch_exists.py b/utils/branch_exists.py
new file mode 100755
index 0000000000000000000000000000000000000000..9626e4aa81e4ee2bd9a239f6a0650dc4e383593f
--- /dev/null
+++ b/utils/branch_exists.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""
+Exit with error code 2 if the branch does not exist.
+"""
+import sys
+import argparse
+import requests
+from ref_to_commit import get_remote
+
+
+def branch_exists(repository, branch):
+ remote = get_remote(repository)
+ resp = requests.get(remote+"/repository/branches/"+branch).json()
+ return "message" not in resp
+
+
+def define_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("repository")
+ parser.add_argument("branchname")
+
+ return parser
+
+
+if __name__ == "__main__":
+ parser = define_parser()
+ args = parser.parse_args()
+ ret = branch_exists(repository=args.repository, branch=args.branchname)
+ if ret is False:
+ print("branch does not exist.")
+ sys.exit(2)
+ else:
+ print("branch exists.")
diff --git a/utils/ref_to_commit.py b/utils/ref_to_commit.py
new file mode 100755
index 0000000000000000000000000000000000000000..93f15f31b6158172cfca5a5095b13f6a4fcb22ab
--- /dev/null
+++ b/utils/ref_to_commit.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+replaces git branch names with the newest commit hash using gitlab api
+"""
+import argparse
+
+import requests
+
+
+_REPOS = {
+ "SERVER": "https://gitlab.indiscale.com/api/v4/projects/100",
+ "WEBUI": "https://gitlab.indiscale.com/api/v4/projects/98",
+ "PYLIB": "https://gitlab.indiscale.com/api/v4/projects/97",
+ "MYSQLBACKEND": "https://gitlab.indiscale.com/api/v4/projects/101",
+ "PYINT": "https://gitlab.indiscale.com/api/v4/projects/99",
+ "CPPLIB": "https://gitlab.indiscale.com/api/v4/projects/107",
+ "CPPINT": "https://gitlab.indiscale.com/api/v4/projects/111",
+ "ADVANCEDUSERTOOLS": "https://gitlab.indiscale.com/api/v4/projects/104"
+}
+
+
+def get_remote(repository):
+ return _REPOS[repository]
+
+
+def ref_to_commit(repository, reference):
+ remote = get_remote(repository)
+ r = requests.get(remote+"/repository/branches/"+reference).json()
+
+ if "name" in r:
+ return r["commit"]["short_id"]
+
+ return reference
+
+
+def define_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("repository")
+ parser.add_argument("reference")
+
+ return parser
+
+
+if __name__ == "__main__":
+ parser = define_parser()
+ args = parser.parse_args()
+ ret = ref_to_commit(repository=args.repository, reference=args.reference)
+ print(ret)