diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000000000000000000000000000000000000..1dc4f3d8c8d375ba3f7b352aa3e18702ec731d83
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[html]
+show_contexts = True
diff --git a/.docker-base/Dockerfile b/.docker-base/Dockerfile
index 923924e75e03c6ca8346b17cdf87eda78efd766f..02c7d5c3045f527c1f17c678633c0f42ee8ce3a5 100644
--- a/.docker-base/Dockerfile
+++ b/.docker-base/Dockerfile
@@ -1,12 +1,12 @@
# Use docker as parent image
-FROM docker:19.03.0
+FROM docker:20.10
# http://bugs.python.org/issue19846
ENV LANG C.UTF-8
# install dependencies
RUN apk add --no-cache py3-pip python3 python3-dev gcc make \
- git bash curl gettext py3-requests
+ git bash curl gettext py3-requests
RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \
libxml2 libxml2-dev
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 876f252299991f2fa4410994b73259c3593c2198..5b26c03c927780e804f190ee6c8dee1e6813a18e 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -25,7 +25,7 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && git checkout dev && pip3 install .
# At least recommonmark 0.6 required.
-RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme
+RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme gitignore-parser
COPY . /git
RUN rm -r /git/.git \
&& mv /git/.docker/pycaosdb.ini /git/integrationtests
diff --git a/.gitignore b/.gitignore
index e2526574b37539d054397d49bbefcadcc9dce654..4c175607e5327472c301949f187c58d925f0d05e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,5 @@ build/
# documentation
_apidoc
+/dist/
+*~
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ebbefaa39650ddaff45b856a8a4d44a2ac495d1..d2abdcd653c3315335c29058a8ca2774dad34577 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ variables:
CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
-stages:
+stages:
- setup
- cert
- style
@@ -53,15 +53,15 @@ test:
- time docker load < /image-cache/mariadb.tar || true
- time docker load < /image-cache/caosdb-dev.tar || true
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
+ - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
-f .docker/docker-compose.yml up -d
- - cd .docker
+ - cd .docker
- /bin/sh ./run.sh
- - cd ..
- - docker logs docker_caosdb-server_1 &> caosdb_log.txt
- - docker logs docker_sqldb_1 &> mariadb_log.txt
+ - cd ..
+ - docker logs docker-caosdb-server-1 &> caosdb_log.txt
+ - docker logs docker-sqldb-1 &> mariadb_log.txt
- docker-compose -f .docker/docker-compose.yml down
- - rc=`cat .docker/result`
+ - rc=`cat .docker/result`
- exit $rc
dependencies: [cert]
needs: [cert]
@@ -76,19 +76,19 @@ build-testenv:
tags: [cached-dind]
image: docker:18.09
stage: setup
- # Hint: do not use only here; the image needs always to be build since it
+ # Hint: do not use only here; the image needs always to be build since it
# contains the repo code
#only:
- script:
+ script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- - docker build
+ - docker build
--file .docker/Dockerfile
-t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE
- docker save $CI_REGISTRY_IMAGE > /image-cache/caosdb-advanced-testenv.tar
- cd .docker-base
- - docker build
+ - docker build
-t $CI_REGISTRY_IMAGE_BASE .
- docker push $CI_REGISTRY_IMAGE_BASE
@@ -123,13 +123,44 @@ linting:
- make lint
allow_failure: true
-unittest:
+unittest_py39:
tags: [docker]
stage: unittest
image: $CI_REGISTRY_IMAGE
needs: [build-testenv]
script:
- - tox
+ # First verify that system Python actually is 3.9
+ - python3 -c "import sys; assert sys.version.startswith('3.9')"
+ - python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
+ - tox
+
+unittest_py37:
+ tags: [docker]
+ stage: unittest
+ image: python:3.7
+ script: &python_test_script
+ - pip install nose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
+ - pip install .
+ - pytest --cov=caosadvancedtools unittests
+
+unittest_py38:
+ tags: [docker]
+ stage: unittest
+ image: python:3.8
+ script: *python_test_script
+
+unittest_py310:
+ tags: [docker]
+ stage: unittest
+ image: python:3.10
+ script: *python_test_script
+
+unittest_py311:
+ tags: [docker]
+ stage: unittest
+ image: python:3.11
+ script: *python_test_script
# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md
new file mode 100644
index 0000000000000000000000000000000000000000..aa1a65aca363b87aff50280e1a86824009d2098b
--- /dev/null
+++ b/.gitlab/issue_templates/Default.md
@@ -0,0 +1,28 @@
+## Summary
+
+*Please give a short summary of what the issue is.*
+
+## Expected Behavior
+
+*What did you expect how the software should behave?*
+
+## Actual Behavior
+
+*What did the software actually do?*
+
+## Steps to Reproduce the Problem
+
+*Please describe, step by step, how others can reproduce the problem. Please try these steps for yourself on a clean system.*
+
+1.
+2.
+3.
+
+## Specifications
+
+- Version: *Which version of this software?*
+- Platform: *Which operating system, which other relevant software versions?*
+
+## Possible fixes
+
+*Do you have ideas how the issue can be resolved?*
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
new file mode 100644
index 0000000000000000000000000000000000000000..3629e0ca3695000863d8c254516f64bf59a7bf60
--- /dev/null
+++ b/.gitlab/merge_request_templates/Default.md
@@ -0,0 +1,56 @@
+# Summary
+
+*Insert a meaningful description for this merge request here: What is the new/changed behavior?
+Which bug has been fixed? Are there related issues?*
+
+
+# Focus
+
+*Point the reviewer to the core of the code change. Where should they start reading? What should
+they focus on (e.g. security, performance, maintainability, user-friendliness, compliance with the
+specs, finding more corner cases, concrete questions)?*
+
+
+# Test Environment
+
+*How to set up a test environment for manual testing?*
+
+
+# Check List for the Author
+
+Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab
+comments for the reviewer. They should guide the reviewer through the changes, explain your changes
+and also point out open questions. For further good practices have a look at [our review
+guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
+
+- [ ] All automated tests pass
+- [ ] Reference related issues
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Up-to-date JSON schema (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary)
+ - Update / write published documentation (`make doc`).
+ - How do I use the software? Assume "stupid" users.
+ - How do I develop or debug the software? Assume novice developers.
+- [ ] Annotations in code (Gitlab comments)
+ - Intent of new code
+ - Problems with old code
+ - Why this implementation?
+
+
+# Check List for the Reviewer
+
+- [ ] I understand the intent of this MR
+- [ ] All automated tests pass
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+ documentation.
+- [ ] The test environment setup works and the intended behavior is reproducible in the test
+ environment
+- [ ] In-code documentation and comments are up-to-date.
+- [ ] Check: Are there specifications? Are they satisfied?
+
+For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
+
+
+/assign me
+/target_branch dev
diff --git a/CHANGELOG.md b/CHANGELOG.md
index be44a47d1a0c79c8a4fa39f382d4d3a0e22439f6..5b6446abe1cd8092bb03d143f902427931350f47 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,94 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security ###
+### Documentation ###
+
+## [0.9.0] - 2023-11-27 ##
+
+### Added ###
+
+* Added support for passing callables as `find_func` to the `BaseTableExporter`.
+* Added member `BaseTableExporter.all_keys`
+* Parsing from YAML now allows to give an existing model to which the YAML data model shall be
+ added.
+* The `json_schema_exporter` module which introduces tools to create a json
+ schema from a RecordType, e.g., for the usage in web forms.
+* `DataModel.get_deep(name: str)` method which uses the DataModel as a kind of cache pool.
+
+### Changed ###
+
+* A bit better error handling in the yaml model parser.
+* `TableImporter.check_datatypes` allows numeric values in string columns if
+ `strict=False` (default).
+
+### Fixed ###
+
+* `TableImporter.check_missing` in case of array-valued fields in table
+* YAML model parser has better description handling.
+
+### Documentation ###
+
+* Test coverage reports are now generated in `.tox/cov_html/` by tox.
+
+## [0.8.0] - 2023-05-30 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- TableImporter now accepts a `existing_columns` argument which demands that certain columns exist
+- The `JsonSchemaParser` class supports `patternProperties`
+- The `JsonSchemaParser` calss supports json-schema references (`$ref`)
+
+### Changed ###
+
+- The converters and datatype arguments of TableImporter now may have keys for nonexisting columns
+- The `JsonSchemaParser` class does not require the top-level entry of a json
+ schema definition to specify a RecordType.
+
+### Fixed ###
+
+- refactored to work with the new default key word in FIND queries: RECORD
+
+## [0.7.0] - 2023-03-09 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- `create_entity_link` function to create html links to entities; useful for
+ logging
+
+## [0.6.1] - 2023-01-20##
+
+### Added ###
+
+* Re-introduced support for Python 3.7
+
+## [0.6.0] - 2022-10-11 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- Unauthorized inserts can now be cached. Note that the Crawler cannot postpone
+ inserts but the Cache has the functionality now.
+- caosdbignore; You can add one or more `.caosdbignore` files to the directory
+ structure that you want to make available in CaosDB and the run loadFiles.
+ The syntax is that of `.gitignore` files. For more information see `loadFiles`
+ section of the Crawler in the documentation.
+
+## [0.5.0] - 2022-09-05 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- You can now use `python -m caosadvancedtools.models.parser model_file` to
+ parse and potentially synchronize data models.
+
+### Deprecated ###
+
+- [#36](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36)
+ `parent` keyword in yaml datamodel definition (replaced by
+ `inherit_from_{obligatory|recommended|suggested}` keywords).
+
## [0.4.1] - 2022-05-03 ##
(Henrik tom Wörden)
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000000000000000000000000000000000000..eea049338996c494a88076b9e9e0f3131ed44a66
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+ - family-names: Fitschen
+ given-names: Timm
+ orcid: https://orcid.org/0000-0002-4022-432X
+ - family-names: Schlemmer
+ given-names: Alexander
+ orcid: https://orcid.org/0000-0003-4124-9649
+ - family-names: Hornung
+ given-names: Daniel
+ orcid: https://orcid.org/0000-0002-7846-6375
+ - family-names: tom Wörden
+ given-names: Henrik
+ orcid: https://orcid.org/0000-0002-5549-578X
+ - family-names: Parlitz
+ given-names: Ulrich
+ orcid: https://orcid.org/0000-0003-3058-1435
+ - family-names: Luther
+ given-names: Stefan
+ orcid: https://orcid.org/0000-0001-7214-8125
+title: CaosDB - Advanced User Tools
+version: 0.9.0
+doi: 10.3390/data4020083
+date-released: 2023-11-27
\ No newline at end of file
diff --git a/README.md b/README.md
index 83a767476286acba98d113b8fa7ab6b482751230..662bf6a6309aaa307505b0b8027b30664756bf10 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,7 @@
This is the **CaosDB Advanced User Tools** repository and a part of the
CaosDB project.
+
This project contains tools that are beyond the typical use of
the CaosDB python client. Especially, this includes the crawler which will
typically be used by a data curator.
@@ -43,7 +44,7 @@ Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
[documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/), the
preferred way is also a merge request as describe above (the documentation
resides in `src/doc`). However, you can also create an issue for it.
-- You can also contact us at **info (AT) caosdb.de** and join the CaosDB
+- You can also contact us at **info (AT) caosdb.org** and join the CaosDB
community on
[#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
diff --git a/README_SETUP.md b/README_SETUP.md
index 43047d554afbe8ffba11aef67b20dde44d29bdcf..bf4f25d92106c19cccc276389b6c97aa22904923 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -11,7 +11,7 @@ git clone 'https://gitlab.com/caosdb/caosdb-advanced-user-tools'
Dependencies will be installed automatically if you use the below described
procedure.
- `caosdb>=0.6.0`
-- `openpyxl>=3.0.0`
+- `openpyxl>=3.0.7`
- `xlrd>=1.2.0`
- `pandas>=1.2.0`
- `numpy>=1.17.3`
@@ -32,7 +32,10 @@ Optional h5-crawler:
- `pip install .[h5-crawler] --user`
## Run Unit Tests
-`tox`
+
+- All tests: `tox`
+- One specific test with tox: `tox -- unittests/test_myusecase.py -k expression`
+- Or even using only pytest: `pytest unittests/test_myusecase.py -k expression`
## Run Integration Tests Locally
@@ -52,6 +55,8 @@ Optional h5-crawler:
`make style`
## Documentation #
+We use sphinx to create the documentation. Docstrings in the code should comply
+with the Googly style (see link below).
Build documentation in `build/` with `make doc`.
@@ -60,3 +65,9 @@ Build documentation in `build/` with `make doc`.
- `sphinx`
- `sphinx-autoapi`
- `recommonmark >= 0.6.0`
+
+### How to contribute ###
+
+- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
+- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index 7592b02d8084d3a5e6419ae66b61331026f2766c..adeab4ddf8a321bdf4e2794f07f18f5c0f4425b3 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -24,6 +24,7 @@ guidelines of the CaosDB Project
- `version` variables in `src/doc/conf.py`
- Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
`ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
+ - `CITATION.cff` (update version and date)
5. Merge the release branch into the main branch.
@@ -41,3 +42,7 @@ guidelines of the CaosDB Project
11. After the merge of main to dev, start a new development version by
setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
version in [setup.py](./setup.py) and preparing CHANGELOG.md.
+
+12. Create releases on gitlab.com and gitlab.indiscale.com that contain (at
+ least) the most recent section of the CHANGELOG as the description and link
+ to the PyPi package.
diff --git a/extra/emacs/readme.md b/extra/emacs/readme.md
new file mode 100644
index 0000000000000000000000000000000000000000..2636eab68b9521acf75c190e3fcf34e6e86fc60b
--- /dev/null
+++ b/extra/emacs/readme.md
@@ -0,0 +1,12 @@
+# Emacs extras #
+
+This directory contains extra utils for use with Emacs.
+
+## Snippets ##
+
+if you copy the contents of the `snippets` directory to your `~/.emacs.d/snippets/`, the following
+*yasnippet* snippets will become available:
+
+- yaml-mode:
+ - `RT`: Insert a new RecordType, with inheritance and properties sections.
+ - `prop`: Insert a new Property into a RecordType, with datatype and description.
diff --git a/extra/emacs/snippets/yaml-mode/Property inside RecordType b/extra/emacs/snippets/yaml-mode/Property inside RecordType
new file mode 100644
index 0000000000000000000000000000000000000000..92769b78e5496ec4cb545556b3eff3fc924c872d
--- /dev/null
+++ b/extra/emacs/snippets/yaml-mode/Property inside RecordType
@@ -0,0 +1,34 @@
+# -*- mode: snippet -*-
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# name: Property inside RecordType
+# key: prop
+# expand-env: ((yas-indent-line 'fixed))
+# --
+${1:property_name}:
+ datatype: ${2:$$(yas-choose-value '("BOOLEAN"
+ "DATETIME"
+ "DOUBLE"
+ "FILE"
+ "INTEGER"
+ "LIST"
+ "REFERENCE"
+ "TEXT"))}
+ description: ${3:description text}
+$0
\ No newline at end of file
diff --git a/extra/emacs/snippets/yaml-mode/RecordType b/extra/emacs/snippets/yaml-mode/RecordType
new file mode 100644
index 0000000000000000000000000000000000000000..6b4a9c263806b6d57442470a11a2770d3d417741
--- /dev/null
+++ b/extra/emacs/snippets/yaml-mode/RecordType
@@ -0,0 +1,30 @@
+# -*- mode: snippet -*-
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# name: RecordType
+# key: RT
+# expand-env: ((yas-indent-line 'fixed))
+# --
+${1:RecordTypeName}:
+ inherit_from_obligatory:$0
+ inherit_from_recommended:
+ inherit_from_suggested:
+ obligatory_properties:
+ recommended_properties:
+ suggested_properties:
diff --git a/integrationtests/caosdbignore b/integrationtests/caosdbignore
new file mode 100644
index 0000000000000000000000000000000000000000..84b3636238e655e9ea7569ca3989f0ca73abb27c
--- /dev/null
+++ b/integrationtests/caosdbignore
@@ -0,0 +1,2 @@
+lol
+~README.md
diff --git a/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f2e41136eac73c39554dede1fd7e67b12502d577
--- /dev/null
+++ b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md
@@ -0,0 +1 @@
+stuff
diff --git a/integrationtests/extroot/Software/2010_TestSoftware/lol b/integrationtests/extroot/Software/2010_TestSoftware/lol
new file mode 100644
index 0000000000000000000000000000000000000000..afd4a162a20a53aaeebf053a257f147759a52d98
--- /dev/null
+++ b/integrationtests/extroot/Software/2010_TestSoftware/lol
@@ -0,0 +1,2 @@
+lol
+
diff --git a/integrationtests/filldb.sh b/integrationtests/filldb.sh
index 9f55365eb595537b43caa9b197c8bc31ea1e69cb..9ee9060e5078b5d48ea5a9972b8ade119378dbc6 100755
--- a/integrationtests/filldb.sh
+++ b/integrationtests/filldb.sh
@@ -5,7 +5,7 @@ python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/ExperimentalData
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/DataAnalysis
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Publications
-python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Software
+python3 -m caosadvancedtools.loadFiles -c caosdbignore -l $PWD/extroot/Software /opt/caosdb/mnt/extroot/Software
python3 insert_model.py
python3 insert_some.py
python3 crawl.py /
diff --git a/integrationtests/test.sh b/integrationtests/test.sh
index 5bb013db6e70a3a8393e7e3b7c7993a6da6bf9b9..a31afcfd2f74770b656eef41002b2f444b7962de 100755
--- a/integrationtests/test.sh
+++ b/integrationtests/test.sh
@@ -14,9 +14,10 @@ then
fi
fi
OUT=/tmp/crawler.output
-ls
+ls
cat pycaosdb.ini
-rm -rf cache.db
+python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
+rm -rf /tmp/caosdb_identifiable_cache.db
set -e
echo "Clearing database"
python3 clear_database.py
@@ -42,21 +43,23 @@ mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx \
DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back
cd ..
echo "run crawler"
-./crawl.py / | tee $OUT
+./crawl.py / | tee "$OUT"
# rename the moved file
mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back \
extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx
# check whether there was something UNAUTHORIZED
-grep "There where unauthorized changes" $OUT
+grep "There where unauthorized changes" "$OUT"
# get the id of the run which is the last field of the output string
-RUN_ID=$(grep "run id:" $OUT | awk '{ print $NF }')
+RUN_ID=$(grep "run id:" "$OUT" | awk '{ print $NF }')
echo $RUN_ID
echo "run crawler again"
echo "./crawl.py -a $RUN_ID /"
-./crawl.py -a $RUN_ID / | tee $OUT
+./crawl.py -a $RUN_ID / | tee "$OUT"
set +e
-if grep "There where unauthorized changes" $OUT
-then
+if grep "There where unauthorized changes" "$OUT"
+then
+ echo "There still were unauthorized changes, which should not have happend!"
+ echo "Test FAILED"
exit 1
fi
set -e
@@ -93,5 +96,8 @@ python3 -m pytest test_json_schema_datamodel_parser.py
echo "Testing yaml datamodel parser"
python3 -m pytest test_yaml_parser.py
+echo "Testing json-schema exporter"
+python3 -m pytest test_json_schema_exporter.py
+
# Obsolete due to teardown in the above test.
# echo "/n/n/n YOU NEED TO RESTART THE SERVER TO REDO TESTS!!!"
diff --git a/integrationtests/test_assure_functions.py b/integrationtests/test_assure_functions.py
index b1c731dbbf25f33b54fc3a005402f292525d2d05..e04d481f230936ae96b02fe910401f50e7138a01 100644
--- a/integrationtests/test_assure_functions.py
+++ b/integrationtests/test_assure_functions.py
@@ -33,7 +33,7 @@ from caosadvancedtools.guard import (global_guard, RETRIEVE, UPDATE)
def setup():
"""Delete all test entities."""
- db.execute_query("FIND Test*").delete(raise_exception_on_error=False)
+ db.execute_query("FIND ENTITY Test*").delete(raise_exception_on_error=False)
def setup_module():
@@ -105,13 +105,13 @@ def test_add_to_empty_list():
db.Record(name="TestReferencingRecord").add_parent(
referencing_rt).add_property(list_prop, value=[]).insert()
- referenced_rec = db.execute_query("FIND TestReferencedRecord", unique=True)
+ referenced_rec = db.execute_query("FIND ENTITY TestReferencedRecord", unique=True)
referencing_rec = db.execute_query(
- "FIND TestReferencingRecord", unique=True)
+ "FIND ENTITY TestReferencingRecord", unique=True)
assure_object_is_in_list(referenced_rec, referencing_rec, list_prop.name)
referencing_rec = db.execute_query(
- "FIND TestReferencingRecord", unique=True)
+ "FIND ENTITY TestReferencingRecord", unique=True)
assert referencing_rec.get_property(list_prop.name).value == [
referenced_rec.id]
diff --git a/integrationtests/test_base_table_exporter_integration.py b/integrationtests/test_base_table_exporter_integration.py
index 9d79e857fe706d78103ade3b92ee38498a2a1607..5af9caa3e83184f77c37c24073d85ee5aae2184b 100644
--- a/integrationtests/test_base_table_exporter_integration.py
+++ b/integrationtests/test_base_table_exporter_integration.py
@@ -81,7 +81,7 @@ def insert_entities():
def setup_module():
"""Clear all test entities"""
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND ENTITY Test*").delete()
except BaseException:
pass
@@ -146,7 +146,7 @@ def test_queries():
"Test_Property_2").value
# test guessing of selector
- del(export_dict["Test_Property_2"]["selector"])
+ del (export_dict["Test_Property_2"]["selector"])
my_exporter = te.BaseTableExporter(
export_dict=export_dict, record=rec1, raise_error_if_missing=True)
assert my_exporter.export_dict["Test_Property_2"]["selector"] == "Test_Property_2"
diff --git a/integrationtests/test_cache.py b/integrationtests/test_cache.py
index 4b0a6cedc390b1268e8d2d89393e19a27a83b3be..aacef1792e6028bf056093c517f45f6367f471d6 100644
--- a/integrationtests/test_cache.py
+++ b/integrationtests/test_cache.py
@@ -33,7 +33,7 @@ from caosadvancedtools.cache import UpdateCache
class CacheTest(unittest.TestCase):
def empty_db(self):
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND ENTITY Test*").delete()
except Exception:
pass
@@ -63,6 +63,12 @@ class CacheTest(unittest.TestCase):
update = UpdateCache(db_file=self.cache)
run_id = "a"
+ print(db.execute_query("FIND Record TestRecord", unique=True))
+ print(db.execute_query("FIND entity with id="+str(rec.id), unique=True))
+ try:
+ print(db.execute_query("FIND Record "+str(rec.id), unique=True))
+ except BaseException:
+ print("Query does not work as expected")
update.insert(cont, run_id)
assert len(update.get_updates(run_id)) == 1
diff --git a/integrationtests/test_crawl_with_datamodel_problems.py b/integrationtests/test_crawl_with_datamodel_problems.py
index 0c6a145afdab682f82af09a17fb9aa0770769959..8623d57d60ded38987953ffaf78b1d30e15a8011 100644
--- a/integrationtests/test_crawl_with_datamodel_problems.py
+++ b/integrationtests/test_crawl_with_datamodel_problems.py
@@ -74,7 +74,7 @@ def test_crawler_with_data_model_problems():
deleted_entities = {"Experiment", "Poster", "results"}
for ent in deleted_entities:
- db.execute_query("FIND "+ent).delete()
+ db.execute_query("FIND ENTITY "+ent).delete()
# Do the crawling
def access(x): return "extroot" + x
diff --git a/integrationtests/test_crawler_basics.py b/integrationtests/test_crawler_basics.py
index 7da90844f14cf0d1eaded9d4fc8f37320da46aad..60c09d73e954c39d752b5fa4ae5e272d28000ca1 100644
--- a/integrationtests/test_crawler_basics.py
+++ b/integrationtests/test_crawler_basics.py
@@ -40,7 +40,7 @@ def setup_module():
"""Clear all test entities. Allow insertions."""
guard.set_level(INSERT)
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND ENTITY Test*").delete()
except Exception:
pass
diff --git a/integrationtests/test_crawler_with_cfoods.py b/integrationtests/test_crawler_with_cfoods.py
index 4efef87cef52e4a2a20a615afe210c32f52a276a..1fa5eaa5a4f050d7282b863aae626982ff738c43 100755
--- a/integrationtests/test_crawler_with_cfoods.py
+++ b/integrationtests/test_crawler_with_cfoods.py
@@ -30,7 +30,16 @@ from caosdb.apiutils import retrieve_entity_with_id
def get_entity_with_id(eid):
- return db.execute_query("FIND "+str(eid), unique=True)
+ return db.execute_query("FIND ENTITY "+str(eid), unique=True)
+
+
+class LoadFilesTest(unittest.TestCase):
+ def test_lol(self):
+ # check whether ignored files were insered
+ res = db.execute_query("FIND FILE WHICH IS STORED AT '**/lol'")
+ assert len(res) == 0
+ res = db.execute_query("FIND FILE WHICH IS STORED AT '**/~README.md'")
+ assert len(res) == 0
class CrawlerTest(unittest.TestCase):
@@ -40,7 +49,7 @@ class CrawlerTest(unittest.TestCase):
# # dummy for dependency test experiment # #
########################
exp = db.execute_query(
- "FIND Experiment with date=2019-02-04 and identifier=empty_identifier",
+ "FIND ENTITY Experiment with date=2019-02-04 and identifier=empty_identifier",
unique=True)
########################
@@ -50,7 +59,7 @@ class CrawlerTest(unittest.TestCase):
# vanishing of the property
# thus an x is used here. Needs to be fixed.
exp = db.execute_query(
- "FIND Experiment with date=2019-02-03 and identifier=empty_identifier",
+ "FIND ENTITY Experiment with date=2019-02-03 and identifier=empty_identifier",
unique=True)
# There should be a Project with name TestProject which is referenced
@@ -90,7 +99,7 @@ class CrawlerTest(unittest.TestCase):
# # second experiment # #
#########################
exp = db.execute_query(
- "FIND Experiment with date=2019-02-03 and identifier='something'",
+ "FIND ENTITY Experiment with date=2019-02-03 and identifier='something'",
unique=True)
# Should be the same project
@@ -111,7 +120,7 @@ class CrawlerTest(unittest.TestCase):
# # first analysis # #
######################
ana = db.execute_query(
- "FIND Analysis with date=2019-02-03 and identifier='empty_identifier'",
+ "FIND ENTITY Analysis with date=2019-02-03 and identifier='empty_identifier'",
unique=True)
# There should be a Project with name TestProject which is referenced
@@ -155,7 +164,7 @@ class CrawlerTest(unittest.TestCase):
# # second analysis # #
#######################
ana = db.execute_query(
- "FIND Analysis with date=2019-02-03 and identifier='something'",
+ "FIND ENTITY Analysis with date=2019-02-03 and identifier='something'",
unique=True)
# Should be the same project
@@ -188,7 +197,7 @@ class CrawlerTest(unittest.TestCase):
# # first simulation # #
######################
sim = db.execute_query(
- "FIND Simulation with date=2019-02-03 and identifier='empty_identifier'",
+ "FIND ENTITY Simulation with date=2019-02-03 and identifier='empty_identifier'",
unique=True)
# There should be a Project with name TestProject which is referenced
@@ -219,7 +228,7 @@ class CrawlerTest(unittest.TestCase):
# # second simulation # #
#########################
sim = db.execute_query(
- "FIND Simulation with date=2019-02-03 and identifier='something'",
+ "FIND ENTITY Simulation with date=2019-02-03 and identifier='something'",
unique=True)
sources = [get_entity_with_id(el) for el in
@@ -264,7 +273,7 @@ class CrawlerTest(unittest.TestCase):
#########################
# # first publication # #
#########################
- pub = db.execute_query("FIND *really_cool_finding", unique=True)
+ pub = db.execute_query("FIND ENTITY *really_cool_finding", unique=True)
# There should be a file as result attached with path poster.pdf
datfile_id = pub.get_property("results").value[0]
@@ -282,7 +291,7 @@ class CrawlerTest(unittest.TestCase):
##########################
# # second publication # #
##########################
- pub = db.execute_query("FIND *paper_on_exciting_stuff ", unique=True)
+ pub = db.execute_query("FIND ENTITY *paper_on_exciting_stuff ", unique=True)
# Test type
self.assertEqual(pub.parents[0].name, "Thesis")
@@ -302,10 +311,10 @@ class CrawlerTest(unittest.TestCase):
# # first software version # #
##############################
ana = db.execute_query(
- "FIND Software with version='V1.0-rc1'", unique=True)
+ "FIND ENTITY Software with version='V1.0-rc1'", unique=True)
sw = db.execute_query(
- "FIND Software with name='2010_TestSoftware'", unique=True)
+ "FIND ENTITY Software with name='2010_TestSoftware'", unique=True)
assert sw.get_property("alias").value == "TestSoftware"
# The software record should inherit from the correct software
@@ -351,10 +360,10 @@ class CrawlerTest(unittest.TestCase):
# # second software version # #
#######################
ana = db.execute_query(
- "FIND Software with version='v0.1'", unique=True)
+ "FIND ENTITY Software with version='v0.1'", unique=True)
sw = db.execute_query(
- "FIND Software with name='2010_TestSoftware'", unique=True)
+ "FIND ENTITY Software with name='2010_TestSoftware'", unique=True)
# The software record should inherit from the correct software
assert sw.id == ana.get_parents()[0].id
@@ -384,11 +393,11 @@ class CrawlerTest(unittest.TestCase):
# # third software version # #
#######################
ana = db.execute_query(
- "FIND Software with date='2020-02-04' and not version",
+ "FIND ENTITY Software with date='2020-02-04' and not version",
unique=True)
sw = db.execute_query(
- "FIND Software with name='2020NewProject0X'", unique=True)
+ "FIND ENTITY Software with name='2020NewProject0X'", unique=True)
# The software record should inherit from the correct software
assert sw.id == ana.get_parents()[0].id
@@ -429,11 +438,11 @@ class CrawlerTest(unittest.TestCase):
# # fourth software version # #
#######################
ana = db.execute_query(
- "FIND Software with date='2020-02-03' and not version",
+ "FIND ENTITY Software with date='2020-02-03' and not version",
unique=True)
sw = db.execute_query(
- "FIND Software with name='2020NewProject0X'", unique=True)
+ "FIND ENTITY Software with name='2020NewProject0X'", unique=True)
assert sw.get_property("alias").value == "NewProject0X"
# The software record should inherit from the correct software
@@ -470,10 +479,10 @@ class CrawlerTest(unittest.TestCase):
# # fifth software version # #
##############################
ana = db.execute_query(
- "FIND Software with version='second'", unique=True)
+ "FIND ENTITY Software with version='second'", unique=True)
sw = db.execute_query(
- "FIND Software with name='2020NewProject0X'", unique=True)
+ "FIND ENTITY Software with name='2020NewProject0X'", unique=True)
assert sw.get_property("alias").value == "NewProject0X"
# The software record should inherit from the correct software
diff --git a/integrationtests/test_data_model.py b/integrationtests/test_data_model.py
index 2949fa81727a6c61a8646a48c249204fa87542d8..bd74a40bde2540bb57245de1de464a1bfd84bc72 100644
--- a/integrationtests/test_data_model.py
+++ b/integrationtests/test_data_model.py
@@ -1,4 +1,5 @@
import unittest
+import pytest
import caosdb as db
from caosadvancedtools.models.data_model import DataModel
@@ -55,9 +56,22 @@ class DataModelTest(unittest.TestCase):
assert len(exist) == 1
assert exist[0].name == "TestRecord"
+ def test_large_data_model(self):
+ # create RT and one property
+ dm = DataModel()
+ long = "Long" * 50
+ first_RT = db.RecordType(name=f"TestRecord_first")
+ for index in range(20):
+ this_RT = db.RecordType(name=f"TestRecord_{long}_{index:02d}")
+ first_RT.add_property(this_RT)
+ dm.append(this_RT)
+ dm.append(first_RT)
+ dm.sync_data_model(noquestion=True) # Insert
+ dm.sync_data_model(noquestion=True) # Check again
+
def tearDown(self):
try:
- tests = db.execute_query("FIND test*")
+ tests = db.execute_query("FIND ENTITY test*")
tests.delete()
except Exception:
pass
diff --git a/integrationtests/test_datamodel_problems.py b/integrationtests/test_datamodel_problems.py
index 3bca302dd2a337cee7fd023ee6a64c5185bc99f5..855170338fbc81493e407fbe235415d60958c0f0 100644
--- a/integrationtests/test_datamodel_problems.py
+++ b/integrationtests/test_datamodel_problems.py
@@ -39,7 +39,7 @@ def setup_module():
"""Clear problem sets and delete possible test entities"""
DataModelProblems.missing.clear()
try:
- db.execute_query("FIND Test*").delete()
+ db.execute_query("FIND Entity Test*").delete()
except Exception as delete_exc:
print(delete_exc)
diff --git a/integrationtests/test_im_und_export.py b/integrationtests/test_im_und_export.py
index 8ea45fd2cebbcb2c3be6c8cb79805204486f7862..407faa1a1d3eb609ffd01b9c78d74f1c6a9b231b 100644
--- a/integrationtests/test_im_und_export.py
+++ b/integrationtests/test_im_und_export.py
@@ -8,13 +8,14 @@ from caosadvancedtools.import_from_xml import import_xml
if __name__ == "__main__":
print("Conducting im- and export tests")
- rec = db.execute_query("FIND 2019-02-03_really_cool_finding", unique=True)
+ rec = db.execute_query("FIND ENTITY 2019-02-03_really_cool_finding", unique=True)
directory = TemporaryDirectory()
export_related_to(rec.id, directory=directory.name)
# delete everything
print("Clearing database")
recs = db.execute_query("FIND entity with id>99")
- recs.delete()
+ if len(recs) > 0:
+ recs.delete()
assert 0 == len(db.execute_query("FIND File which is stored at "
"**/poster.pdf"))
print("Importing stored elements")
@@ -22,7 +23,7 @@ if __name__ == "__main__":
# The following tests the existence of some required entities.
# However, this is not a full list.
- db.execute_query("FIND 2019-02-03_really_cool_finding", unique=True)
+ db.execute_query("FIND ENTITY 2019-02-03_really_cool_finding", unique=True)
db.execute_query("FIND RecordType Poster", unique=True)
db.execute_query("FIND RecordType Analysis", unique=True)
db.execute_query("FIND RecordType Person", unique=True)
diff --git a/integrationtests/test_json_schema_exporter.py b/integrationtests/test_json_schema_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..69edcf42d1fd285c030ad6d6ccb7f73f2d1b5536
--- /dev/null
+++ b/integrationtests/test_json_schema_exporter.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import linkahead as db
+
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema as rtjs
+
+
+def _delete_everything():
+ ents = db.execute_query("FIND ENTITY WITH ID > 99")
+ if ents:
+ ents.delete()
+
+
+def setup_module():
+ _delete_everything()
+
+
+def teardown_module():
+ _delete_everything()
+
+
+def test_uniqueness_of_reference_types():
+ parent_type = db.RecordType(name="ParentType").insert()
+ int_prop = db.Property(name="IntegerProp", datatype=db.INTEGER).insert()
+ sub_type = db.RecordType(name="SubType").add_parent(parent_type).add_property(
+ int_prop, importance=db.RECOMMENDED).insert()
+ referencing_type = db.RecordType(name="ReferencingType")
+ referencing_type.add_property(int_prop, importance=db.OBLIGATORY)
+ referencing_type.add_property(parent_type)
+ referencing_type.insert()
+ recA = db.Record(name="RecAParent").add_parent(parent_type).insert()
+ recB = db.Record(name="RecBSub").add_parent(sub_type).insert()
+
+ rt = db.execute_query(f"FIND RECORDTYPE WITH name='{referencing_type.name}'", unique=True)
+
+ schema = rtjs(rt)
+ assert schema["title"] == referencing_type.name
+ assert schema["type"] == "object"
+ assert len(schema["required"]) == 1
+ assert "IntegerProp" in schema["required"]
+ assert "IntegerProp" in schema["properties"]
+ assert schema["properties"]["IntegerProp"]["type"] == "integer"
+ assert parent_type.name in schema["properties"]
+ assert "oneOf" in schema["properties"][parent_type.name]
+ one_of = schema["properties"][parent_type.name]["oneOf"]
+ assert len(one_of) == 2
+ enum_index = 0
+ if "enum" not in one_of[enum_index]:
+ # As in unittests, we can't rely on the order of oneOf.
+ enum_index = 1 - enum_index
+ assert "enum" in one_of[enum_index]
+ assert len(one_of[enum_index]["enum"]) == 2
+ assert recA.name in one_of[enum_index]["enum"]
+ assert recB.name in one_of[enum_index]["enum"]
+ assert one_of[1 - enum_index]["type"] == "object"
+ # No properties in parent_type
+ assert len(one_of[1 - enum_index]["properties"]) == 0
diff --git a/integrationtests/test_table.py b/integrationtests/test_table.py
index 15b851fb5c81611d0faba93edfc58f46f9d75e79..b8dfe349f3dac3be9bb741f937f2be4f73b6b2af 100644
--- a/integrationtests/test_table.py
+++ b/integrationtests/test_table.py
@@ -34,14 +34,14 @@ if __name__ == "__main__":
table = pd.read_csv("example_table.csv")
- assert 0 == len(db.execute_query("FIND Person with firstname=Henrik"))
+ assert 0 == len(db.execute_query("FIND ENTITY Person with firstname=Henrik"))
first = table.loc[table.firstName == "Henrik"]
tcr = TableCrawler(table=first, unique_cols=["firstName", "lastName"],
recordtype="Person", interactive=False)
tcr.crawl(security_level=UPDATE)
- assert 1 == len(db.execute_query("FIND Person with firstname=Henrik"))
+ assert 1 == len(db.execute_query("FIND ENTITY Person with firstname=Henrik"))
tcr = TableCrawler(table=table, unique_cols=["firstName", "lastName"],
recordtype="Person", interactive=False)
tcr.crawl(security_level=UPDATE)
- assert 1 == len(db.execute_query("FIND Person with firstname=Henrik"))
- assert 1 == len(db.execute_query("FIND Person with firstname=Max"))
+ assert 1 == len(db.execute_query("FIND ENTITY Person with firstname=Henrik"))
+ assert 1 == len(db.execute_query("FIND ENTITY Person with firstname=Max"))
diff --git a/integrationtests/update_analysis.py b/integrationtests/update_analysis.py
index bd18ab375437bec02320dcfd269896c2ba7e2bb0..ddebc049f449026400278a26226d341d64e678c8 100644
--- a/integrationtests/update_analysis.py
+++ b/integrationtests/update_analysis.py
@@ -39,7 +39,7 @@ from caosadvancedtools.serverside.generic_analysis import run
def main():
- da = db.execute_query("FIND Analysis with identifier=TEST", unique=True)
+ da = db.execute_query("FIND ENTITY Analysis with identifier=TEST", unique=True)
run(da)
diff --git a/manual_tests/test_labfolder_import.py b/manual_tests/test_labfolder_import.py
index e1e9d3266478900b7fae02b3493fbc3d41ea2bd5..c767feb55cdf3958343d8d9780d01fa10c70f6ec 100644
--- a/manual_tests/test_labfolder_import.py
+++ b/manual_tests/test_labfolder_import.py
@@ -32,7 +32,7 @@ from caosadvancedtools.converter import labfolder_export as labfolder
def main(args):
"""The main function."""
- model = parse_model_from_yaml("./model.yml")
+ model = parse_model_from_yaml("./models/model.yml")
model.sync_data_model()
labfolder.import_data(args.folder)
diff --git a/manual_tests/test_labfolder_retrieve.py b/manual_tests/test_labfolder_retrieve.py
index 8c3f12d84a8990412d0d19cd6026a3452677f943..5bbaf91d0221a402e3a39246a129413adfa5f871 100644
--- a/manual_tests/test_labfolder_retrieve.py
+++ b/manual_tests/test_labfolder_retrieve.py
@@ -31,7 +31,7 @@ from caosadvancedtools.converter.labfolder_api import Importer
def main(args):
"""The main function."""
- model = parse_model_from_yaml("./model.yml")
+ model = parse_model_from_yaml("./models/model.yml")
# model.sync_data_model()
importer = Importer()
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index e65efaf9aaf061a8a1ec0040f87d682536fac4c2..0000000000000000000000000000000000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[pytest]
-testpaths = unittests
-addopts = -vv
diff --git a/release.sh b/release.sh
index 1af097f014de6cd9eb3d3e8ba5da34aea0fe1671..f6335ae20d0c29e760b508aac831a35460a59ef3 100755
--- a/release.sh
+++ b/release.sh
@@ -1,4 +1,4 @@
#!/bin/bash
rm -rf dist/ build/ .eggs/
python setup.py sdist bdist_wheel
-python -m twine upload -s dist/*
+python -m twine upload dist/*
diff --git a/setup.py b/setup.py
index 929613de35de01da98b02c77cd76b17b04784bd8..a35bcad4979ea272c38563cfe89562d814ee5901 100755
--- a/setup.py
+++ b/setup.py
@@ -46,8 +46,8 @@ from setuptools import find_packages, setup
########################################################################
MAJOR = 0
-MINOR = 4
-MICRO = 2
+MINOR = 9
+MICRO = 1
PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = False
@@ -154,14 +154,17 @@ def setup_package():
long_description_content_type="text/markdown",
author='Henrik tom Wörden',
author_email='h.tomwoerden@indiscale.com',
- install_requires=["caosdb>=0.7.0",
- "jsonschema>=4.4.0",
+ python_requires='>=3.7',
+ install_requires=["linkahead>=0.13.1",
+ "jsonref",
+ "jsonschema[format]>=4.4.0",
"numpy>=1.17.3",
- "openpyxl>=3.0.0",
+ "openpyxl>=3.0.7",
"pandas>=1.2.0",
"xlrd>=2.0",
],
extras_require={"h5-crawler": ["h5py>=3.3.0", ],
+ "gitignore-parser": ["gitignore-parser >=0.1.0", ],
},
packages=find_packages('src'),
package_dir={'': 'src'},
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/__init__.py b/src/caosadvancedtools/bloxberg/swagger_client/__init__.py
index 136c5b27a37cfbd9135230468ae5a29cb0eb2b77..255d6d3124dc352f10366e22f1eb8b461ff6593d 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/__init__.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/__init__.py
@@ -13,22 +13,23 @@
"""
from __future__ import absolute_import
+from swagger_client.models.validation_error import ValidationError
+from swagger_client.models.http_validation_error import HTTPValidationError
+from swagger_client.models.controller_cert_tools_generate_unsigned_certificate_json_certificate import ControllerCertToolsGenerateUnsignedCertificateJsonCertificate
+from swagger_client.models.controller_cert_tools_generate_pdf_json_certificate import ControllerCertToolsGeneratePdfJsonCertificate
+from swagger_client.models.batch import Batch
+from swagger_client.configuration import Configuration
+from swagger_client.api_client import ApiClient
+from swagger_client.api.pdf_api import PdfApi
+from swagger_client.api.certificate_api import CertificateApi
# Fake the installation
-import sys, pathlib
+import sys
+import pathlib
__this_dir = str(pathlib.Path(__file__).parent.parent)
if __this_dir not in sys.path:
sys.path.append(__this_dir)
# import apis into sdk package
-from swagger_client.api.certificate_api import CertificateApi
-from swagger_client.api.pdf_api import PdfApi
# import ApiClient
-from swagger_client.api_client import ApiClient
-from swagger_client.configuration import Configuration
# import models into sdk package
-from swagger_client.models.batch import Batch
-from swagger_client.models.controller_cert_tools_generate_pdf_json_certificate import ControllerCertToolsGeneratePdfJsonCertificate
-from swagger_client.models.controller_cert_tools_generate_unsigned_certificate_json_certificate import ControllerCertToolsGenerateUnsignedCertificateJsonCertificate
-from swagger_client.models.http_validation_error import HTTPValidationError
-from swagger_client.models.validation_error import ValidationError
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/api_client.py b/src/caosadvancedtools/bloxberg/swagger_client/api_client.py
index 25e6501a4e36b09bca266f2eb375807053a58870..7337ca334c545b2c2502a20cb5369db331149037 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/api_client.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/api_client.py
@@ -591,7 +591,7 @@ class ApiClient(object):
)
def __hasattr(self, object, name):
- return name in object.__class__.__dict__
+ return name in object.__class__.__dict__
def __deserialize_model(self, data, klass):
"""Deserializes list or dict to model.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py b/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py
index 7a347cf7ac9148df8ec9a43200f4058f127447b9..474ca01a69a6a06c93b7e9a640695fa709890997 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/batch.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class Batch(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py
index 2d7fd2d763ba40c9a384203301aa3e70efdf7783..8c1b50d8816b09c1a466cf7d11cee1ca605dfd3a 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_pdf_json_certificate.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class ControllerCertToolsGeneratePdfJsonCertificate(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py
index 4a6d2d3f0e15faa8672f001e964d66c6e0a27780..fa0da3cb0c09e384cdddbd4ce458a4baf14f4b5d 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/controller_cert_tools_generate_unsigned_certificate_json_certificate.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class ControllerCertToolsGenerateUnsignedCertificateJsonCertificate(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py b/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py
index 21c9e467311c596499f3f408c5ac670b5852c6fa..67c23fba87467a7888bff82fc7f11e9d90e15f15 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/http_validation_error.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class HTTPValidationError(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py b/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py
index 7ae6bf0900449ff3612798a4503692c4e38e1c11..96d1e23734698efbdad8423c33012473e9aac03b 100644
--- a/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py
+++ b/src/caosadvancedtools/bloxberg/swagger_client/models/validation_error.py
@@ -15,6 +15,7 @@ import re # noqa: F401
import six
+
class ValidationError(object):
"""NOTE: This class is auto generated by the swagger code generator program.
diff --git a/src/caosadvancedtools/cache.py b/src/caosadvancedtools/cache.py
index ff807f2aba6210d643e675e7e3dd91d7c3b30906..cf74e330d3efb754d8e79d84ba816877c295c784 100644
--- a/src/caosadvancedtools/cache.py
+++ b/src/caosadvancedtools/cache.py
@@ -23,17 +23,19 @@
#
# ** end header
-# TODO this is implementing a cache on client side. Should it be on
-# server side?
+# Note: This is implementing a cache on client side. It would be great if the server would provide
+# something to replace this.
import os
import sqlite3
+import tempfile
+import warnings
+from abc import ABC, abstractmethod
+from copy import deepcopy
from hashlib import sha256
import caosdb as db
from lxml import etree
-import tempfile
-
def put_in_container(stuff):
if isinstance(stuff, list):
@@ -64,59 +66,181 @@ def get_pretty_xml(cont):
return etree.tounicode(xml, pretty_print=True)
-class Cache(object):
- """
- stores identifiables (as a hash of xml) and their respective ID.
+class AbstractCache(ABC):
+ def __init__(self, db_file=None, force_creation=False):
+ """
+ db_file: The path of the database file.
- This allows to retrieve the Record corresponding to an indentifiable
- without querying.
- """
+ if force_creation is set to True, the file will be created
+ regardless of a file at the same path already exists.
+ """
- def __init__(self, db_file=None):
if db_file is None:
- self.db_file = "cache.db"
+ tmppath = tempfile.gettempdir()
+ self.db_file = os.path.join(tmppath, self.get_default_file_name())
else:
self.db_file = db_file
- if not os.path.exists(self.db_file):
+ if not os.path.exists(self.db_file) or force_creation:
self.create_cache()
+ else:
+ self.check_cache()
+
+ @abstractmethod
+ def get_cache_schema_version(self):
+ """
+ A method that has to be overloaded that sets the version of the
+ SQLITE database schema. The schema is saved in table version column schema.
+
+ Increase this variable, when changes to the cache tables are made.
+ """
+ pass
+ @abstractmethod
def create_cache(self):
+ """
+ Provide an overloaded function here that creates the cache in
+ the most recent version.
+ """
+ pass
+
+ @abstractmethod
+ def get_default_file_name(self):
+ """
+ Supply a default file name for the cache here.
+ """
+ pass
+
+ def check_cache(self):
+ """
+ Check whether the cache in db file self.db_file exists and conforms
+ to the latest database schema.
+
+ If it does not exist, it will be created using the newest database schema.
+
+ If it exists, but the schema is outdated, an exception will be raised.
+ """
+ try:
+ current_schema = self.get_cache_version()
+ except sqlite3.OperationalError:
+ current_schema = 1
+
+ if current_schema > self.get_cache_schema_version():
+ raise RuntimeError(
+ "Cache is corrupt or was created with a future version of this program.")
+ elif current_schema < self.get_cache_schema_version():
+ raise RuntimeError("Cache version too old. Please remove the current cache file:\n"
+ + self.db_file)
+
+ def get_cache_version(self):
+ """
+ Return the version of the cache stored in self.db_file.
+ The version is stored as the only entry in colum schema of table version.
+ """
+ try:
+ conn = sqlite3.connect(self.db_file)
+ c = conn.cursor()
+ c.execute("SELECT schema FROM version")
+ version_row = c.fetchall()
+
+ if len(version_row) != 1:
+ raise RuntimeError("Cache version table broken.")
+
+ return version_row[0][0]
+ finally:
+ conn.close()
+
+ def run_sql_commands(self, commands, fetchall=False):
+ """
+ Run a list of SQL commands on self.db_file.
+
+ commands: list of sql commands (tuples) to execute
+ fetchall: When True, run fetchall as last command and return the results.
+ Otherwise nothing is returned.
+ """
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
- c.execute(
- '''CREATE TABLE identifiables (digest text primary key, caosdb_id integer)''')
+
+ for sql in commands:
+ c.execute(*sql)
+
+ if fetchall:
+ results = c.fetchall()
conn.commit()
conn.close()
+ if fetchall:
+ return results
+
+
+class IdentifiableCache(AbstractCache):
+ """
+ stores identifiables (as a hash of xml) and their respective ID.
+
+ This allows to retrieve the Record corresponding to an indentifiable
+ without querying.
+ """
+
+ def get_cache_schema_version(self):
+ return 2
+
+ def get_default_file_name(self):
+ return "caosdb_identifiable_cache.db"
+
+ def __init__(self, db_file=None, force_creation=False):
+ super().__init__(db_file, force_creation)
+
+ def create_cache(self):
+ """
+ Create a new SQLITE cache file in self.db_file.
+
+ Two tables will be created:
+ - identifiables is the actual cache.
+ - version is a table with version information about the cache.
+ """
+ self.run_sql_commands([
+ ('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))])
+
@staticmethod
def hash_entity(ent):
- xml = get_pretty_xml(ent)
+ """
+ Format an entity as "pretty" XML and return the SHA256 hash.
+ """
+ xml = get_pretty_xml(deepcopy(ent))
digest = sha256(xml.encode("utf-8")).hexdigest()
return digest
- def insert(self, ent_hash, ent_id):
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''INSERT INTO identifiables VALUES (?, ?)''',
- (ent_hash, ent_id))
- conn.commit()
- conn.close()
+ def insert(self, ent_hash, ent_id, ent_version):
+ """
+ Insert a new cache entry.
+
+ ent_hash: Hash of the entity. Should be generated with Cache.hash_entity
+ ent_id: ID of the entity
+ ent_version: Version string of the entity
+ """
+ self.run_sql_commands([
+ ('''INSERT INTO identifiables VALUES (?, ?, ?)''',
+ (ent_hash, ent_id, ent_version))])
def check_existing(self, ent_hash):
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''Select * FROM identifiables WHERE digest=?''',
- (ent_hash,))
- res = c.fetchone()
- conn.commit()
- conn.close()
+ """
+ Check the cache for a hash.
+
+ ent_hash: The hash to search for.
- if res is None:
- return res
+ Return the ID and the version ID of the hashed entity.
+ Return None if no entity with that hash is in the cache.
+ """
+ res = self.run_sql_commands([('''Select * FROM identifiables WHERE digest=?''',
+ (ent_hash,))], True)
+
+ if len(res) == 0:
+ return None
else:
- return res[1]
+ return res[0][1:]
def update_ids_from_cache(self, entities):
""" sets ids of those entities that are in cache
@@ -131,7 +255,7 @@ class Cache(object):
eid = self.check_existing(ehash)
if eid is not None:
- ent.id = eid
+ ent.id = eid[0]
return hashes
@@ -141,25 +265,75 @@ class Cache(object):
The hashes must correspond to the entities in the list
"""
+ # Check whether all entities have IDs and versions:
+
+ for ent in entities:
+ if ent.id is None:
+ raise RuntimeError("Entity has no ID.")
+
+ if ent.version is None or ent.version.id is None:
+ raise RuntimeError("Entity has no version ID.")
+
for ehash, ent in zip(hashes, entities):
if self.check_existing(ehash) is None:
- self.insert(ehash, ent.id)
+ self.insert(ehash, ent.id, ent.version.id)
+
+ def validate_cache(self, entities=None):
+ """
+ Runs through all entities stored in the cache and checks
+ whether the version still matches the most recent version.
+ Non-matching entities will be removed from the cache.
+
+ entities: When set to a db.Container or a list of Entities
+ the IDs from the cache will not be retrieved from the CaosDB database,
+ but the versions from the cache will be checked against the versions
+ contained in that collection. Only entries in the cache that have
+ a corresponding version in the collection will be checked, all others
+ will be ignored. Useful for testing.
+
+ Return a list of invalidated entries or an empty list if no elements have been invalidated.
+ """
+
+ res = self.run_sql_commands([(
+ "SELECT caosdb_id, caosdb_version FROM identifiables", ())], True)
+
+ if entities is None:
+ # TODO this might become a problem. If many entities are cached,
+ # then all of them are retrieved here...
+ ids = [c_id for c_id, _ in res]
+ ids = set(ids)
+ entities = db.Container()
+ entities.extend([db.Entity(id=c_id) for c_id in ids])
+ entities.retrieve()
+
+ v = {c_id: c_version for c_id, c_version in res}
+
+ invalidate_list = []
+
+ for ent in entities:
+ if ent.version.id != v[ent.id]:
+ invalidate_list.append(ent.id)
+ self.run_sql_commands([(
+ "DELETE FROM identifiables WHERE caosdb_id IN ({})".format(
+ ", ".join([str(caosdb_id) for caosdb_id in invalidate_list])), ())])
-class UpdateCache(Cache):
+ return invalidate_list
+
+
+class UpdateCache(AbstractCache):
"""
- stores unauthorized updates
+ stores unauthorized inserts and updates
- If the Guard is set to a mode that does not allow an update, the update can
- be stored in this cache such that it can be authorized and done later.
+ If the Guard is set to a mode that does not allow an insert or update, the insert or update can
+ be stored in this cache such that it can be authorized and performed later.
"""
- def __init__(self, db_file=None):
- if db_file is None:
- tmppath = tempfile.gettempdir()
- tmpf = os.path.join(tmppath, "crawler_update_cache.db")
- db_file = tmpf
- super().__init__(db_file=db_file)
+ def get_cache_schema_version(self):
+ return 3
+
+ def get_default_file_name(self):
+ return "/tmp/crawler_update_cache.db"
@staticmethod
def get_previous_version(cont):
@@ -169,46 +343,70 @@ class UpdateCache(Cache):
old_ones = db.Container()
for ent in cont:
- old_ones.append(db.execute_query("FIND {}".format(ent.id),
+ old_ones.append(db.execute_query("FIND ENTITY WITH ID={}".format(ent.id),
unique=True))
return old_ones
- def insert(self, cont, run_id):
- """Insert a pending, unauthorized update
+ def insert(self, cont, run_id, insert=False):
+ """Insert a pending, unauthorized insert or update
Parameters
----------
- cont: Container with the records to be updated containing the desired
+ cont: Container with the records to be inserted or updated containing the desired
version, i.e. the state after the update.
run_id: int
The id of the crawler run
+ insert: bool
+ Whether the entities in the container shall be inserted or updated.
"""
cont = put_in_container(cont)
- old_ones = UpdateCache.get_previous_version(cont)
+
+ if insert:
+ old_ones = ""
+ else:
+ old_ones = UpdateCache.get_previous_version(cont)
new_ones = cont
- old_hash = Cache.hash_entity(old_ones)
+ if insert:
+ old_hash = ""
+ else:
+ old_hash = Cache.hash_entity(old_ones)
new_hash = Cache.hash_entity(new_ones)
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''INSERT INTO updates VALUES (?, ?, ?, ?, ?)''',
- (old_hash, new_hash, str(old_ones), str(new_ones),
- str(run_id)))
- conn.commit()
- conn.close()
+ self.run_sql_commands([('''INSERT INTO updates VALUES (?, ?, ?, ?, ?)''',
+ (old_hash, new_hash, str(old_ones), str(new_ones),
+ str(run_id)))])
def create_cache(self):
""" initialize the cache """
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''CREATE TABLE updates (olddigest text, newdigest text,
- oldrep text, newrep text, run_id text,
- primary key (olddigest, newdigest, run_id))''')
- conn.commit()
- conn.close()
+ self.run_sql_commands([
+ ('''CREATE TABLE updates (olddigest TEXT, newdigest TEXT, oldrep TEXT,
+ newrep TEXT, run_id TEXT, primary key (olddigest, newdigest, run_id))''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))])
+
+ def get(self, run_id, querystring):
+ """ returns the pending updates for a given run id
+
+ Parameters:
+ -----------
+ run_id: the id of the crawler run
+ querystring: the sql query
+ """
+
+ return self.run_sql_commands([(querystring, (str(run_id),))], fetchall=True)
+
+ def get_inserts(self, run_id):
+ """ returns the pending updates for a given run id
+
+ Parameters:
+ -----------
+ run_id: the id of the crawler run
+ """
+
+ return self.get(run_id, '''Select * FROM updates WHERE olddigest='' AND run_id=?''')
def get_updates(self, run_id):
""" returns the pending updates for a given run id
@@ -218,12 +416,10 @@ class UpdateCache(Cache):
run_id: the id of the crawler run
"""
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''Select * FROM updates WHERE run_id=?''',
- (str(run_id),))
- res = c.fetchall()
- conn.commit()
- conn.close()
+ return self.get(run_id, '''Select * FROM updates WHERE olddigest!='' AND run_id=?''')
+
- return res
+class Cache(IdentifiableCache):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use IdentifiableCache."))
+ super().__init__(*args, **kwargs)
diff --git a/src/caosadvancedtools/cfood.py b/src/caosadvancedtools/cfood.py
index 4a9f955a17fc429deb6cdd10c3645700e579b4df..c0da4f0156dc2af48a4ba80b4d0af69c62cd5c3e 100644
--- a/src/caosadvancedtools/cfood.py
+++ b/src/caosadvancedtools/cfood.py
@@ -807,7 +807,7 @@ class RowCFood(AbstractCFood):
def update_identifiables(self):
rec = self.identifiables[0]
- for key, value in self.item.iteritems():
+ for key, value in self.item.items():
if key in self.unique_cols:
continue
assure_property_is(rec, key,
diff --git a/src/caosadvancedtools/cfoods/h5.py b/src/caosadvancedtools/cfoods/h5.py
index cbf9d0baefa435b71eeaeefe63a9b018faabe7ea..4e6832f2e96e0950ed99146d4907f1ffb70d8494 100644
--- a/src/caosadvancedtools/cfoods/h5.py
+++ b/src/caosadvancedtools/cfoods/h5.py
@@ -97,7 +97,7 @@ def h5_attr_to_property(val):
# TODO this can eventually be removed
- if(hasattr(val, 'ndim')):
+ if hasattr(val, 'ndim'):
if not isinstance(val, np.ndarray) and val.ndim != 0:
print(val, val.ndim)
raise Exception(
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 0159688c7c7d59e779d576aed54b176e802fca85..5e84bc8a60c1b358150c4db389efb62656af0631 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -50,7 +50,7 @@ from sqlite3 import IntegrityError
import caosdb as db
from caosdb.exceptions import BadQueryError
-from .cache import Cache, UpdateCache, get_pretty_xml
+from .cache import IdentifiableCache, UpdateCache, get_pretty_xml
from .cfood import RowCFood, add_files, get_ids_for_entities_with_names
from .datainconsistency import DataInconsistencyError
from .datamodel_problems import DataModelProblems
@@ -58,6 +58,7 @@ from .guard import RETRIEVE, ProhibitedException
from .guard import global_guard as guard
from .serverside.helper import send_mail as main_send_mail
from .suppressKnown import SuppressKnown
+from .utils import create_entity_link
logger = logging.getLogger(__name__)
@@ -103,15 +104,8 @@ def apply_list_of_updates(to_be_updated, update_flags={},
info = "UPDATE: updating the following entities\n"
- baseurl = db.configuration.get_config()["Connection"]["url"]
-
- def make_clickable(txt, id):
- return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt)
-
for el in to_be_updated:
- info += str("\t" + make_clickable(el.name, el.id)
- if el.name is not None
- else "\t" + make_clickable(str(el.id), el.id))
+ info += str("\t" + create_entity_link(el))
info += "\n"
logger.info(info)
@@ -190,7 +184,8 @@ class Crawler(object):
self.filterKnown.reset(cat)
if self.use_cache:
- self.cache = Cache(db_file=cache_file)
+ self.cache = IdentifiableCache(db_file=cache_file)
+ self.cache.validate_cache()
def iteritems(self):
""" generates items to be crawled with an index"""
@@ -208,28 +203,69 @@ class Crawler(object):
run_id: the id of the crawler run
"""
cache = UpdateCache()
+ inserts = cache.get_inserts(run_id)
+ all_inserts = 0
+ all_updates = 0
+ for _, _, _, new, _ in inserts:
+ new_cont = db.Container()
+ new_cont = new_cont.from_xml(new)
+ new_cont.insert(unique=False)
+ logger.info("Successfully inserted {} records!".format(len(new_cont)))
+ all_inserts += len(new_cont)
+ logger.info("Finished with authorized inserts.")
+
changes = cache.get_updates(run_id)
for _, _, old, new, _ in changes:
- current = db.Container()
- new_cont = db.Container()
- new_cont = new_cont.from_xml(new)
+ new_cont = db.Container.from_xml(new)
+ ids = []
+ tmp = db.Container()
+ update_incomplete = False
+ # remove duplicate entities
+ for el in new_cont:
+ if el.id not in ids:
+ ids.append(el.id)
+ tmp.append(el)
+ else:
+ update_incomplete = True
+ new_cont = tmp
+ if new_cont[0].version:
+ valids = db.Container()
+ nonvalids = db.Container()
+
+ for ent in new_cont:
+ remote_ent = db.Entity(id=ent.id).retrieve()
+ if ent.version == remote_ent.version:
+ valids.append(ent)
+ else:
+ update_incomplete = True
+ nonvalids.append(remote_ent)
+ valids.update(unique=False)
+ logger.info("Successfully updated {} records!".format(
+ len(valids)))
+ logger.info("{} Records were not updated because the version in the server "
+ "changed!".format(len(nonvalids)))
+ all_updates += len(valids)
+ else:
+ current = db.Container()
- for ent in new_cont:
- current.append(db.execute_query("FIND {}".format(ent.id),
- unique=True))
- current_xml = get_pretty_xml(current)
+ for ent in new_cont:
+ current.append(db.Entity(id=ent.id).retrieve())
+ current_xml = get_pretty_xml(current)
- # check whether previous version equals current version
- # if not, the update must not be done
+ # check whether previous version equals current version
+ # if not, the update must not be done
- if current_xml != old:
- continue
+ if current_xml != old:
+ continue
- new_cont.update(unique=False)
- logger.info("Successfully updated {} records!".format(
- len(new_cont)))
+ new_cont.update(unique=False)
+ logger.info("Successfully updated {} records!".format(
+ len(new_cont)))
+ all_updates += len(new_cont)
+ logger.info("Some updates could not be applied. Crawler has to rerun.")
logger.info("Finished with authorized updates.")
+ return all_inserts, all_updates
def collect_cfoods(self):
"""
@@ -428,9 +464,9 @@ class Crawler(object):
# only done in SSS mode
if "SHARED_DIR" in os.environ:
- filename = self.save_form([el[3]
- for el in pending_changes], path)
- self.send_mail([el[3] for el in pending_changes], filename)
+ filename = Crawler.save_form([el[3]
+ for el in pending_changes], path, self.run_id)
+ Crawler.send_mail([el[3] for el in pending_changes], filename)
for i, el in enumerate(pending_changes):
@@ -441,8 +477,7 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
logger.info("There where unauthorized changes (see above). An "
"email was sent to the curator.\n"
"You can authorize the updates by invoking the crawler"
- " with the run id: {rid}\n".format(rid=self.run_id,
- path=path))
+ " with the run id: {rid}\n".format(rid=self.run_id))
if len(DataModelProblems.missing) > 0:
err_msg = ("There were problems with one or more RecordType or "
@@ -465,7 +500,8 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
else:
logger.info("Crawler terminated successfully!")
- def save_form(self, changes, path):
+ @staticmethod
+ def save_form(changes, path, run_id):
"""
Saves an html website to a file that contains a form with a button to
authorize the given changes.
@@ -547,13 +583,13 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
</body>
</html>
""".format(url=db.configuration.get_config()["Connection"]["url"],
- rid=self.run_id,
+ rid=run_id,
changes=escape("\n".join(changes)),
path=path)
if "SHARED_DIR" in os.environ:
directory = os.environ["SHARED_DIR"]
- filename = str(self.run_id)+".html"
+ filename = str(run_id)+".html"
randname = os.path.basename(os.path.abspath(directory))
filepath = os.path.abspath(os.path.join(directory, filename))
filename = os.path.join(randname, filename)
@@ -561,7 +597,8 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
f.write(form)
return filename
- def send_mail(self, changes, filename):
+ @staticmethod
+ def send_mail(changes, filename):
""" calls sendmail in order to send a mail to the curator about pending
changes
diff --git a/src/caosadvancedtools/export_related.py b/src/caosadvancedtools/export_related.py
index 69b588c34cc7c8123ab4291f6d8f76f06e7400be..7ae3a4dbba65faed551f75a1627eb504a3275f48 100755
--- a/src/caosadvancedtools/export_related.py
+++ b/src/caosadvancedtools/export_related.py
@@ -99,7 +99,7 @@ def invert_ids(entities):
def export_related_to(rec_id, directory="."):
if not isinstance(rec_id, int):
raise ValueError("rec_id needs to be an integer")
- ent = db.execute_query("FIND {}".format(rec_id), unique=True)
+ ent = db.execute_query("FIND ENTITY {}".format(rec_id), unique=True)
cont = recursively_collect_related(ent)
export(cont, directory=directory)
diff --git a/src/caosadvancedtools/json_schema_exporter.py b/src/caosadvancedtools/json_schema_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..700c24e890c36a5b4219a1c2cc7d74ce38d6d398
--- /dev/null
+++ b/src/caosadvancedtools/json_schema_exporter.py
@@ -0,0 +1,691 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"""Module for converting a data model into a json schema compatible dictionary.
+
+The scope of this json schema is the automatic generation of user interfaces.
+"""
+
+from collections import OrderedDict
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+
+import linkahead as db
+from linkahead.common.datatype import get_list_datatype, is_list_datatype
+
+
+class JsonSchemaExporter:
+ """A class which collects everything needed for the conversion.
+ """
+
+ def __init__(self, additional_properties: bool = True,
+ name_property_for_new_records: bool = False,
+ description_property_for_new_records: bool = False,
+ additional_options_for_text_props: dict = None,
+ additional_json_schema: Dict[str, dict] = None,
+ additional_ui_schema: Dict[str, dict] = None,
+ units_in_description: bool = True,
+ do_not_create: List[str] = None,
+ do_not_retrieve: List[str] = None,
+ no_remote: bool = False,
+ multiple_choice: List[str] = None,
+ wrap_files_in_objects: bool = False,
+ ):
+ """Set up a JsonSchemaExporter, which can then be applied on RecordTypes.
+
+ Parameters
+ ----------
+ additional_properties : bool, optional
+ Whether additional properties will be admitted in the resulting
+ schema. Optional, default is True.
+ name_property_for_new_records : bool, optional
+ Whether objects shall generally have a `name` property in the generated schema.
+ Optional, default is False.
+ description_property_for_new_records : bool, optional
+ Whether objects shall generally have a `description` property in the generated schema.
+ Optional, default is False.
+ additional_options_for_text_props : dict, optional
+ Dictionary containing additional "pattern" or "format" options for
+ string-typed properties. Optional, default is empty.
+ additional_json_schema : dict[str, dict], optional
+ Additional schema content for elements of the given names.
+ additional_ui_schema : dict[str, dict], optional
+ Additional ui schema content for elements of the given names.
+ units_in_description : bool, optional
+ Whether to add the unit of a LinkAhead property (if it has any) to the
+ description of the corresponding schema entry. If set to false, an
+ additional `unit` key is added to the schema itself which is purely
+ annotational and ignored, e.g., in validation. Default is True.
+ do_not_create : list[str]
+ A list of reference Property names, for which there should be no option
+ to create them. Instead, only the choice of existing elements should
+ be given.
+ do_not_retrieve : list[str]
+ A list of RedcordType names, for which no Records shall be retrieved. Instead, only an
+ object description should be given. If this list overlaps with the `do_not_create`
+ parameter, the behavior is undefined.
+ no_remote : bool
+ If True, do not attempt to connect to a LinkAhead server at all. Default is False.
+ multiple_choice : list[str], optional
+ A list of reference Property names which shall be denoted as multiple choice properties.
+ This means that each option in this property may be selected at most once. This is not
+ implemented yet if the Property is not in ``do_not_create`` as well.
+ wrap_files_in_objects : bool, optional
+ Whether (lists of) files should be wrapped into an array of objects
+ that have a file property. The sole purpose of this wrapping is to
+ provide a workaround for a `react-jsonschema-form
+ bug<https://github.com/rjsf-team/react-jsonschema-form/issues/3957>`_
+ so only set this to True if you're using the exported schema with
+ react-json-form and you are experiencing the bug. Default is False.
+ """
+ if not additional_options_for_text_props:
+ additional_options_for_text_props = {}
+ if not additional_json_schema:
+ additional_json_schema = {}
+ if not additional_ui_schema:
+ additional_ui_schema = {}
+ if not do_not_create:
+ do_not_create = []
+ if not do_not_retrieve:
+ do_not_retrieve = []
+ if not multiple_choice:
+ multiple_choice = []
+
+ self._additional_properties = additional_properties
+ self._name_property_for_new_records = name_property_for_new_records
+ self._description_property_for_new_records = description_property_for_new_records
+ self._additional_options_for_text_props = additional_options_for_text_props
+ self._additional_json_schema = additional_json_schema
+ self._additional_ui_schema = additional_ui_schema
+ self._units_in_description = units_in_description
+ self._do_not_create = do_not_create
+ self._do_not_retrieve = do_not_retrieve
+ self._no_remote = no_remote
+ self._multiple_choice = multiple_choice
+ self._wrap_files_in_objects = wrap_files_in_objects
+
+ @staticmethod
+ def _make_required_list(rt: db.RecordType):
+ """Return the list of names of properties with importance db.OBLIGATORY."""
+ required_list = []
+ for prop in rt.properties:
+ if rt.get_importance(prop.name) != db.OBLIGATORY:
+ continue
+ prop_name = prop.name
+ if isinstance(prop.datatype, db.Entity):
+ prop_name = prop.datatype.name
+ required_list.append(prop_name)
+
+ return required_list
+
+ def _make_segment_from_prop(self, prop: db.Property) -> Tuple[OrderedDict, dict]:
+ """Return the JSON Schema and ui schema segments for the given property.
+
+ The result may either be a simple json schema segment, such as a `string
+ <https://json-schema.org/understanding-json-schema/reference/string>`_ element (or another
+ simple type), a combination such as `anyOf
+ <https://json-schema.org/understanding-json-schema/reference/combining#anyof>`_ or an `array
+ <https://json-schema.org/understanding-json-schema/reference/array>`_ element
+
+ Parameters
+ ----------
+ prop : db.Property
+ The property to be transformed.
+ """
+ json_prop = OrderedDict()
+ ui_schema: dict = {}
+ if prop.datatype == db.TEXT or prop.datatype == db.DATETIME:
+ text_format = None
+ text_pattern = None
+ if prop.name in self._additional_options_for_text_props:
+ if "pattern" in self._additional_options_for_text_props[prop.name]:
+ text_pattern = self._additional_options_for_text_props[prop.name]["pattern"]
+ if "format" in self._additional_options_for_text_props[prop.name]:
+ text_format = self._additional_options_for_text_props[prop.name]["format"]
+ elif prop.datatype == db.DATETIME:
+ # Set the date or datetime format if only a pattern is given ...
+ text_format = ["date", "date-time"]
+ elif prop.datatype == db.DATETIME:
+ # ... again, for those props that don't appear in the additional
+ # options list.
+ text_format = ["date", "date-time"]
+
+ json_prop = self._make_text_property(prop.description, text_format, text_pattern)
+ return self._customize(json_prop, ui_schema, prop)
+
+ if prop.description:
+ json_prop["description"] = prop.description
+ if self._units_in_description and prop.unit:
+ if "description" in json_prop:
+ json_prop["description"] += f" Unit is {prop.unit}."
+ else:
+ json_prop["description"] = f"Unit is {prop.unit}."
+ elif prop.unit:
+ json_prop["unit"] = prop.unit
+
+ if prop.datatype == db.BOOLEAN:
+ json_prop["type"] = "boolean"
+ elif prop.datatype == db.INTEGER:
+ json_prop["type"] = "integer"
+ elif prop.datatype == db.DOUBLE:
+ json_prop["type"] = "number"
+ elif is_list_datatype(prop.datatype) and not (
+ self._wrap_files_in_objects and get_list_datatype(prop.datatype,
+ strict=True) == db.FILE):
+ json_prop["type"] = "array"
+ list_element_prop = db.Property(
+ name=prop.name, datatype=get_list_datatype(prop.datatype, strict=True))
+ json_prop["items"], inner_ui_schema = self._make_segment_from_prop(list_element_prop)
+ if prop.name in self._multiple_choice and prop.name in self._do_not_create:
+ # TODO: if not multiple_choice, but do_not_create:
+ # "ui:widget" = "radio" & "ui:inline" = true
+ # TODO: set threshold for number of items.
+ json_prop["uniqueItems"] = True
+ ui_schema["ui:widget"] = "checkboxes"
+ ui_schema["ui:inline"] = True
+ if inner_ui_schema:
+ ui_schema["items"] = inner_ui_schema
+ elif prop.is_reference():
+ if prop.datatype == db.REFERENCE:
+ # No Record creation since no RT is specified and we don't know what
+ # schema to use, so only enum of all Records and all Files.
+ values = self._retrieve_enum_values("RECORD") + self._retrieve_enum_values("FILE")
+ json_prop["enum"] = values
+ if prop.name in self._multiple_choice:
+ json_prop["uniqueItems"] = True
+ elif prop.datatype == db.FILE or (
+ self._wrap_files_in_objects and
+ is_list_datatype(prop.datatype) and
+ get_list_datatype(prop.datatype, strict=True) == db.FILE
+ ):
+ # Singular FILE (wrapped or unwrapped), or wrapped LIST<FILE>
+ if self._wrap_files_in_objects:
+ # Workaround for react-jsonschema-form bug
+ # https://github.com/rjsf-team/react-jsonschema-form/issues/3957:
+ # Wrap all FILE references (regardless whether lists or
+ # scalars) in an array of objects that have a file property,
+ # since objects can be deleted, files can't.
+ json_prop["type"] = "array"
+ json_prop["items"] = {
+ "type": "object",
+ "title": "Next file",
+ # The wrapper object must wrap a file and can't be empty.
+ "required": ["file"],
+ # Wrapper objects must only contain the wrapped file.
+ "additionalProperties": False,
+ "properties": {
+ "file": {
+ "title": "Enter your file.",
+ "type": "string",
+ "format": "data-url"
+ }
+ }
+ }
+ if not is_list_datatype(prop.datatype):
+ # Scalar file, so the array has maximum length 1
+ json_prop["maxItems"] = 1
+ else:
+ json_prop["type"] = "string"
+ json_prop["format"] = "data-url"
+ else:
+ prop_name = prop.datatype
+ if isinstance(prop.datatype, db.Entity):
+ prop_name = prop.datatype.name
+ if prop.name in self._do_not_retrieve:
+ values = []
+ else:
+ values = self._retrieve_enum_values(f"RECORD '{prop_name}'")
+ if prop.name in self._do_not_create:
+ # Only a simple list of values
+ json_prop["enum"] = values
+ else:
+ if self._no_remote:
+ rt = prop.datatype
+ else:
+ rt = db.execute_query(f"FIND RECORDTYPE WITH name='{prop_name}'",
+ unique=True)
+ subschema, ui_schema = self._make_segment_from_recordtype(rt)
+ # if inner_ui_schema:
+ # ui_schema = inner_ui_schema
+ if values:
+ subschema["title"] = "Create new"
+ json_prop["oneOf"] = [
+ {
+ "title": "Existing entries",
+ "enum": values,
+ },
+ subschema
+ ]
+ else:
+ json_prop = subschema
+
+ else:
+ raise ValueError(
+ f"Unknown or no property datatype. Property {prop.name} with type {prop.datatype}")
+
+ return self._customize(json_prop, ui_schema, prop)
+
+ @staticmethod
+ def _make_text_property(description="", text_format=None, text_pattern=None) -> OrderedDict:
+ """Create a text element.
+
+ Can be a `string <https://json-schema.org/understanding-json-schema/reference/string>`_
+ element or an `anyOf
+ <https://json-schema.org/understanding-json-schema/reference/combining#anyof>`_ combination
+ thereof.
+
+ Example:
+
+ .. code-block:: json
+
+ {
+ "type": "string",
+ "description": "Some description",
+ "pattern": "[0-9]{2..4}-[0-9]{2-4}",
+ "format": "hostname",
+ }
+ """
+ prop: OrderedDict[str, Union[str, list]] = OrderedDict({
+ "type": "string"
+ })
+ if description:
+ prop["description"] = description
+ if text_format is not None:
+ if isinstance(text_format, list):
+ # We want the type inside the options, not in the head:
+ # "datetime property": {
+ # "anyOf": [
+ # {
+ # "type": "string",
+ # "format": "date"
+ # },
+ # {
+ # "type": "string",
+ # "format": "date-time"
+ # }]}
+ prop.pop("type")
+ prop["anyOf"] = [{"type": "string", "format": tf} for tf in text_format]
+ else:
+ prop["format"] = text_format
+ if text_pattern is not None:
+ prop["pattern"] = text_pattern
+
+ return prop
+
+ def _retrieve_enum_values(self, role: str):
+
+ if self._no_remote:
+ return []
+
+ possible_values = db.execute_query(f"SELECT name, id FROM {role}")
+
+ vals = []
+ for val in possible_values:
+ if val.name:
+ vals.append(f"{val.name}")
+ else:
+ vals.append(f"{val.id}")
+
+ return vals
+
+ def _make_segment_from_recordtype(self, rt: db.RecordType) -> Tuple[OrderedDict, dict]:
+ """Return Json schema and uischema segments for the given RecordType.
+
+ The result is an element of type `object
+ <https://json-schema.org/understanding-json-schema/reference/object>`_ and typically
+ contains more properties:
+
+ .. code-block:: json
+
+ {
+ "type": "object",
+ "title": "MyRecordtypeName",
+ "properties": {
+ "number": { "type": "number" },
+ "street_name": { "type": "string" },
+ "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+ }
+ }
+ """
+ schema: OrderedDict[str, Any] = OrderedDict({
+ "type": "object"
+ })
+ ui_schema = {}
+
+ schema["required"] = self._make_required_list(rt)
+ schema["additionalProperties"] = self._additional_properties
+ if rt.description:
+ schema["description"] = rt.description
+
+ if rt.name:
+ schema["title"] = rt.name
+
+ props = OrderedDict()
+ if self._name_property_for_new_records:
+ props["name"] = self._make_text_property("The name of the Record to be created")
+ if self._description_property_for_new_records:
+ props["description"] = self._make_text_property(
+ "The description of the Record to be created")
+
+ for prop in rt.properties:
+ if prop.name in props:
+ # Multi property
+ raise NotImplementedError(
+ "Creating a schema for multi-properties is not specified. "
+ f"Property {prop.name} occurs more than once."
+ )
+ props[prop.name], inner_ui_schema = self._make_segment_from_prop(prop)
+ if inner_ui_schema:
+ ui_schema[prop.name] = inner_ui_schema
+
+ schema["properties"] = props
+
+ return schema, ui_schema
+
+ def _customize(self, schema: OrderedDict, ui_schema: dict, entity: db.Entity = None) -> (
+ Tuple[OrderedDict, dict]):
+ """Generic customization method.
+
+Walk over the available customization stores and apply all applicable ones. No specific order is
+guaranteed (as of now).
+
+ Parameters
+ ----------
+ schema, ui_schema : dict
+ The input schemata.
+ entity: db.Entity : , optional
+ An Entity object, may be useful in the future for customizers.
+
+ Returns
+ -------
+ out : Tuple[dict, dict]
+ The modified input schemata.
+ """
+
+ name = schema.get("title", None)
+ if entity and entity.name:
+ name = entity.name
+ for key, add_schema in self._additional_json_schema.items():
+ if key == name:
+ schema.update(add_schema)
+ for key, add_schema in self._additional_ui_schema.items():
+ if key == name:
+ ui_schema.update(add_schema)
+
+ return schema, ui_schema
+
+ def recordtype_to_json_schema(self, rt: db.RecordType, rjsf: bool = False) -> Union[
+ dict, Tuple[dict, dict]]:
+ """Create a jsonschema from a given RecordType that can be used, e.g., to
+ validate a json specifying a record of the given type.
+
+ Parameters
+ ----------
+ rt : RecordType
+ The RecordType from which a json schema will be created.
+ rjsf : bool, optional
+ If True, uiSchema definitions for react-jsonschema-forms will be output as the second
+ return value. Default is False
+
+ Returns
+ -------
+ schema : dict
+ A dict containing the json schema created from the given RecordType's properties.
+
+ ui_schema : dict, optional
+ A ui schema. Only if a parameter asks for it (e.g. ``rjsf``).
+ """
+ if rt is None:
+ raise ValueError(
+ "recordtype_to_json_schema(...) cannot be called with a `None` RecordType.")
+ schema, inner_uischema = self._make_segment_from_recordtype(rt)
+ schema["$schema"] = "https://json-schema.org/draft/2020-12/schema"
+ if rt.description:
+ schema["description"] = rt.description
+ schema, inner_uischema = self._customize(schema, inner_uischema, rt)
+
+ if rjsf:
+ uischema = {}
+ if inner_uischema:
+ uischema = inner_uischema
+ return schema, uischema
+ return schema
+
+
+def recordtype_to_json_schema(rt: db.RecordType, additional_properties: bool = True,
+ name_property_for_new_records: bool = False,
+ description_property_for_new_records: bool = False,
+ additional_options_for_text_props: Optional[dict] = None,
+ additional_json_schema: Dict[str, dict] = None,
+ additional_ui_schema: Dict[str, dict] = None,
+ units_in_description: bool = True,
+ do_not_create: List[str] = None,
+ do_not_retrieve: List[str] = None,
+ no_remote: bool = False,
+ multiple_choice: List[str] = None,
+ rjsf: bool = False,
+ wrap_files_in_objects: bool = False
+ ) -> Union[dict, Tuple[dict, dict]]:
+ """Create a jsonschema from a given RecordType that can be used, e.g., to
+ validate a json specifying a record of the given type.
+
+ This is a standalone function which works without manually creating a
+ JsonSchemaExporter object.
+
+ Parameters
+ ----------
+ rt : RecordType
+ The RecordType from which a json schema will be created.
+ additional_properties : bool, optional
+ Whether additional properties will be admitted in the resulting
+ schema. Optional, default is True.
+ name_property_for_new_records : bool, optional
+ Whether objects shall generally have a `name` property in the generated schema. Optional,
+ default is False.
+ description_property_for_new_records : bool, optional
+ Whether objects shall generally have a `description` property in the generated schema.
+ Optional, default is False.
+ additional_options_for_text_props : dict, optional
+ Dictionary containing additional "pattern" or "format" options for
+ string-typed properties. Optional, default is empty.
+ additional_json_schema : dict[str, dict], optional
+ Additional schema content for elements of the given names.
+ additional_ui_schema : dict[str, dict], optional
+ Additional ui schema content for elements of the given names.
+ units_in_description : bool, optional
+ Whether to add the unit of a LinkAhead property (if it has any) to the
+ description of the corresponding schema entry. If set to false, an
+ additional `unit` key is added to the schema itself which is purely
+ annotational and ignored, e.g., in validation. Default is True.
+ do_not_create : list[str], optional
+ A list of reference Property names, for which there should be no option
+ to create them. Instead, only the choice of existing elements should
+ be given.
+ do_not_retrieve : list[str], optional
+ A list of RedcordType names, for which no Records shall be retrieved. Instead, only an
+ object description should be given. If this list overlaps with the `do_not_create`
+ parameter, the behavior is undefined.
+ no_remote : bool, optional
+ If True, do not attempt to connect to a LinkAhead server at all. Default is False.
+ multiple_choice : list[str], optional
+ A list of reference Property names which shall be denoted as multiple choice properties.
+ This means that each option in this property may be selected at most once. This is not
+ implemented yet if the Property is not in ``do_not_create`` as well.
+ rjsf : bool, optional
+ If True, uiSchema definitions for react-jsonschema-forms will be output as the second return
+ value. Default is False.
+ wrap_files_in_objects : bool, optional
+ Whether (lists of) files should be wrapped into an array of objects that
+ have a file property. The sole purpose of this wrapping is to provide a
+ workaround for a `react-jsonschema-form
+ bug<https://github.com/rjsf-team/react-jsonschema-form/issues/3957>`_ so
+ only set this to True if you're using the exported schema with
+ react-json-form and you are experiencing the bug. Default is False.
+
+
+ Returns
+ -------
+ schema : dict
+ A dict containing the json schema created from the given RecordType's properties.
+
+ ui_schema : dict, optional
+ A ui schema. Only if a parameter asks for it (e.g. ``rjsf``).
+ """
+
+ exporter = JsonSchemaExporter(
+ additional_properties=additional_properties,
+ name_property_for_new_records=name_property_for_new_records,
+ description_property_for_new_records=description_property_for_new_records,
+ additional_options_for_text_props=additional_options_for_text_props,
+ additional_json_schema=additional_json_schema,
+ additional_ui_schema=additional_ui_schema,
+ units_in_description=units_in_description,
+ do_not_create=do_not_create,
+ do_not_retrieve=do_not_retrieve,
+ no_remote=no_remote,
+ multiple_choice=multiple_choice,
+ wrap_files_in_objects=wrap_files_in_objects
+ )
+ return exporter.recordtype_to_json_schema(rt, rjsf=rjsf)
+
+
+def make_array(schema: dict, rjsf_uischema: dict = None) -> Union[dict, Tuple[dict, dict]]:
+ """Create an array of the given schema.
+
+The result will look like this:
+
+.. code:: js
+
+ { "type": "array",
+ "items": {
+ // the schema
+ }
+ }
+
+Parameters
+----------
+
+schema : dict
+ The JSON schema which shall be packed into an array.
+
+rjsf_uischema : dict, optional
+ A react-jsonschema-forms ui schema that shall be wrapped as well.
+
+Returns
+-------
+
+schema : dict
+ A JSON schema dict with a top-level array which contains instances of the given schema.
+
+ui_schema : dict, optional
+ The wrapped ui schema. Only returned if ``rjsf_uischema`` is given as parameter.
+ """
+ result = {
+ "type": "array",
+ "items": schema,
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ }
+
+ if rjsf_uischema is not None:
+ ui_schema = {"items": rjsf_uischema}
+ return result, ui_schema
+ return result
+
+
+def merge_schemas(schemas: Union[Dict[str, dict], Iterable[dict]],
+ rjsf_uischemas: Union[Dict[str, dict], Sequence[dict]] = None) -> (
+ Union[dict, Tuple[dict, dict]]):
+ """Merge the given schemata into a single schema.
+
+The result will look like this:
+
+.. code:: js
+
+ {
+ "type": "object",
+ "properties": {
+ // A, B, C
+ },
+ "required": [
+ // "A", "B", "C"
+ ],
+ "additionalProperties": false
+ }
+
+
+Parameters
+----------
+
+schemas : dict[str, dict] | Iterable[dict]
+ A dict or iterable of schemata which shall be merged together. If this is a dict, the keys will
+ be used as property names, otherwise the titles of the submitted schemata. If they have no title,
+ numbers will be used as a fallback. Note that even with a dict, the original schema's "title" is
+ not changed.
+rjsf_uischemas : dict[str, dict] | Iterable[dict], optional
+ If given, also merge the react-jsonschema-forms from this argument and return as the second return
+ value. If ``schemas`` is a dict, this parameter must also be a dict, if ``schemas`` is only an
+ iterable, this paramater must support numerical indexing.
+
+Returns
+-------
+
+schema : dict
+ A JSON schema dict with a top-level object which contains the given schemata as properties.
+
+uischema : dict
+ If ``rjsf_uischemas`` was given, this contains the merged UI schemata.
+ """
+ sub_schemas: dict[str, dict] = OrderedDict()
+ required = []
+ ui_schema = None
+
+ if isinstance(schemas, dict):
+ sub_schemas = schemas
+ required = [str(k) for k in schemas.keys()]
+ if rjsf_uischemas is not None:
+ if not isinstance(rjsf_uischemas, dict):
+ raise ValueError("Parameter `rjsf_uischemas` must be a dict, because `schemas` is "
+ f"as well, but it is a {type(rjsf_uischemas)}.")
+ ui_schema = {k: rjsf_uischemas[k] for k in schemas.keys()}
+ else:
+ for i, schema in enumerate(schemas, start=1):
+ title = schema.get("title", str(i))
+ sub_schemas[title] = schema
+ required.append(title)
+ if rjsf_uischemas is not None:
+ if not isinstance(rjsf_uischemas, Sequence):
+ raise ValueError("Parameter `rjsf_uischemas` must be a sequence, because `schemas` "
+ f"is as well, but it is a {type(rjsf_uischemas)}.")
+ ui_schema = {}
+ for i, title in enumerate(sub_schemas.keys()):
+ ui_schema[title] = rjsf_uischemas[i]
+ # ui_schema = {"index": ui_schema}
+
+ result = {
+ "type": "object",
+ "properties": sub_schemas,
+ "required": required,
+ "additionalProperties": False,
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ }
+
+ if ui_schema is not None:
+ return result, ui_schema
+ return result
diff --git a/src/caosadvancedtools/loadFiles.py b/src/caosadvancedtools/loadFiles.py
index 445374b3b3bddf94eefd7952e190bf52155765a8..405b3d135c8af89e32c74015bd04f76f21828e20 100755
--- a/src/caosadvancedtools/loadFiles.py
+++ b/src/caosadvancedtools/loadFiles.py
@@ -25,10 +25,14 @@
import argparse
import logging
+import os
import math
import sys
+import re
from argparse import ArgumentParser
+from tempfile import NamedTemporaryFile
+import shutil
import caosdb as db
logger = logging.getLogger(__name__)
@@ -46,37 +50,133 @@ def convert_size(size):
return '%s %s' % (s, size_name[i])
-def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks):
+def combine_ignore_files(caosdbignore, localignore, dirname=None):
+ """appends the contents of localignore to caosdbignore and saves the result
+ and returns the name
- if dryrun:
- logger.info("Performin a dryrun!")
- files = db.Container().retrieve(
- unique=False,
- raise_exception_on_error=True,
- flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
- + ("-e " + exclude + " " if exclude else "")
- + ("-i " + include + " " if include else "")
- + ("--force-allow-symlinks " if forceAllowSymlinks else "")
- + path})
+ """
+
+ tmp = NamedTemporaryFile(delete=False, mode="w",
+ dir=dirname, prefix=".caosdbignore")
+ with open(caosdbignore, "r") as base:
+ tmp.write(base.read())
+ with open(localignore, "r") as local:
+ tmp.write(local.read())
+ tmp.close()
+ return tmp.name
+
+
+def compile_file_list(caosdbignore, localpath):
+ """creates a list of files that contain all files under localpath except
+ those excluded by caosdbignore
+
+ """
+
+ from gitignore_parser import parse_gitignore
+ matches = parse_gitignore(caosdbignore)
+ current_ignore = caosdbignore
+ non_ignored_files = []
+ ignore_files = []
+ for root, dirs, files in os.walk(localpath):
+ # remove local ignore files that do no longer apply to the current subtree (branch switch)
+ while len(ignore_files) > 0 and not root.startswith(ignore_files[-1][0]):
+ shutil.os.remove(ignore_files[-1][1])
+ ignore_files.pop()
+
+ # use the global one if there are no more local ones
+ if len(ignore_files) > 0:
+ current_ignore = ignore_files[-1][1]
+ matches = parse_gitignore(current_ignore)
+ else:
+ current_ignore = caosdbignore
+ matches = parse_gitignore(current_ignore)
+
+ # create a new local ignore file
+ if ".caosdbignore" in files:
+ current_ignore = combine_ignore_files(current_ignore,
+ os.path.join(
+ root, ".caosdbignore"),
+ # due to the logic of gitignore_parser the file
+ # has to be written to this folder
+ dirname=root)
+ ignore_files.append((root, current_ignore))
+ matches = parse_gitignore(current_ignore)
+
+ # actually append files that are not ignored
+ for fi in files:
+ fullpath = os.path.join(root, fi)
+ if not matches(fullpath):
+ non_ignored_files.append(fullpath)
+ return non_ignored_files
+
+
+def create_re_for_file_list(files, localroot, remoteroot):
+ """creates a regular expression that matches file paths contained in the
+ files argument and all parent directories. The prefix localroot is replaced
+ by the prefix remoteroot.
+
+ """
+ regexp = ""
+ for fi in files:
+ path = fi
+ reg = ""
+ while path != localroot and path != "/" and path != "":
+ print(path, localroot)
+ reg = "(/"+re.escape(os.path.basename(path)) + reg + ")?"
+ path = os.path.dirname(path)
+ regexp += "|" + re.escape(remoteroot) + reg
+ return "^("+regexp[1:]+")$"
+
+
+def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbignore=None,
+ localpath=None):
+
+ if caosdbignore:
+ # create list of files and create regular expression for small chunks
+ filelist = compile_file_list(caosdbignore, localpath)
+ fulllist = filelist
+
+ index = 0
+ step_size = 3
+ includes = []
+ while index < len(fulllist):
+ subset = fulllist[index:min(index+step_size, len(fulllist))]
+ includes.append(create_re_for_file_list(subset, localpath, path))
+ index += step_size
else:
- # new files (inserting them using the insertFilesInDir feature of
- # the server, which inserts files via symlinks)
- files = db.Container().insert(
- unique=False,
- raise_exception_on_error=True,
- flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
- + ("-e " + exclude + " " if exclude else "")
- + ("-i " + include + " " if include else "")
- + ("--force-allow-symlinks " if forceAllowSymlinks else "")
- + path})
+ includes = [include]
+
+ # if no caosdbignore file is used, this iterates over a single include
+ for include in includes:
+ if dryrun:
+ logger.info("Performin a dryrun!")
+ files = db.Container().retrieve(
+ unique=False,
+ raise_exception_on_error=True,
+ flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+ + ("-e " + exclude + " " if exclude else "")
+ + ("-i " + include + " " if include else "")
+ + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+ + path})
+ else:
+ # new files (inserting them using the insertFilesInDir feature of
+ # the server, which inserts files via symlinks)
+ files = db.Container().insert(
+ unique=False,
+ raise_exception_on_error=True,
+ flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+ + ("-e " + exclude + " " if exclude else "")
+ + ("-i " + include + " " if include else "")
+ + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+ + path})
- totalsize = 0 # collecting total size of all new files
+ totalsize = 0 # collecting total size of all new files
- for f in files:
- totalsize += f.size
+ for f in files:
+ totalsize += f.size
- logger.info("Made in total {} new files with a combined size of {} "
- "accessible.".format(len(files), convert_size(totalsize)))
+ logger.info(
+ f"Made new files accessible: {len(files)}, combined size: {convert_size(totalsize)} ")
return
@@ -91,6 +191,18 @@ def main(argv=None):
# Setup argument parser
parser = ArgumentParser()
+ parser.add_argument("-p", "--prefix", dest="prefix",
+ help="store files with this prefix into the server's"
+ " file system.")
+ parser.add_argument("-c", "--caosdbignore", help="""
+Path to a caosdbignore file that defines which files shall be included and which do not.
+The syntax is the same as in a gitignore file. You must also provide the localpath option
+since the check is done locally.
+"""
+ )
+ parser.add_argument("-l", "--localpath", help="Path to the root directory on this machine. "
+ "This is needed if a caosdbignore file is used since the check is done "
+ "locally")
parser.add_argument("-i", "--include", dest="include",
help="""
only include paths matching this regex pattern.
@@ -104,9 +216,6 @@ exclude is given preference over include.
parser.add_argument("-e", "--exclude", dest="exclude",
help="exclude paths matching this regex pattern.",
metavar="RE")
- parser.add_argument("-p", "--prefix", dest="prefix",
- help="store files with this prefix into the server's"
- " file system.")
parser.add_argument("-d", "--dry-run", dest="dryrun", action="store_true",
help="Just simulate the insertion of the files.")
parser.add_argument('-t', '--timeout', dest="timeout",
@@ -127,6 +236,21 @@ exclude is given preference over include.
"directory tree.", action="store_true")
args = parser.parse_args()
+ if args.caosdbignore and (args.exclude or args.include):
+ raise ValueError(
+ "Do not use a caosdbignore file and in- or exclude simultaneously!")
+
+ if args.caosdbignore and not args.localpath:
+ raise ValueError("To use caosdbignore you must supply a local path!")
+
+ if args.localpath and (args.exclude or args.include):
+ raise ValueError(
+ "Do not use a localpath and in- or exclude simultaneously!")
+
+ # configure logging
+ logger.addHandler(logging.StreamHandler(stream=sys.stdout))
+ logger.setLevel(logging.INFO)
+
con = db.get_connection()
con.timeout = float(args.timeout)
con._login()
@@ -137,8 +261,9 @@ exclude is given preference over include.
exclude=args.exclude,
prefix=args.prefix,
dryrun=args.dryrun,
-
forceAllowSymlinks=args.forceAllowSymlinks,
+ caosdbignore=args.caosdbignore,
+ localpath=args.localpath,
)
return 0
diff --git a/src/caosadvancedtools/models/data_model.py b/src/caosadvancedtools/models/data_model.py
index d9079e6196b4751ca86ba41275108330b946d57c..266414893bcdf1ab45ee1345fc549e15f4a66250 100644
--- a/src/caosadvancedtools/models/data_model.py
+++ b/src/caosadvancedtools/models/data_model.py
@@ -29,8 +29,9 @@ from copy import deepcopy
# remove this, when we drop support for old Python versions.
from typing import List
-import caosdb as db
-from caosdb.apiutils import compare_entities, describe_diff
+import linkahead as db
+import linkahead.common.models as models
+from linkahead.apiutils import compare_entities, describe_diff, merge_entities
CAOSDB_INTERNAL_PROPERTIES = [
@@ -60,7 +61,8 @@ class DataModel(dict):
different purpose (e.g. someone else's experiment).
DataModel inherits from dict. The keys are always the names of the
- entities. Thus you cannot have unnamed entities in your model.
+ entities. Thus you cannot have unnamed or ambiguously named entities in your
+ model.
Example:
@@ -141,7 +143,7 @@ class DataModel(dict):
# in via the extern keyword:
ref = db.Property(name=ent.name).retrieve()
else:
- query = db.Query(f"FIND * with id={ent.id}")
+ query = db.Query(f"FIND ENTITY with id={ent.id}")
ref = query.execute(unique=True)
diff = (describe_diff(*compare_entities(ent, ref
), name=ent.name))
@@ -261,3 +263,73 @@ class DataModel(dict):
all_ents[prop.name] = prop
return list(all_ents.values())
+
+ def get_deep(self, name: str, visited_props: dict = None, visited_parents: set = None):
+ """Attempt to resolve references for the given ``name``.
+
+ The returned entity has all the properties it inherits from its ancestry and all properties
+ have the correct descriptions and datatypes. This methods only uses data which is available
+ in this DataModel, which acts kind of like a cache pool.
+
+ Note that this may change this data model (subsequent "get" like calls may also return
+ deeper content.)
+
+ """
+ entity = self.get(name)
+ if not entity:
+ return entity
+ if not visited_props:
+ visited_props = {}
+ if not visited_parents:
+ visited_parents = set()
+
+ importances = {
+ models.OBLIGATORY: 0,
+ models.RECOMMENDED: 1,
+ models.SUGGESTED: 2,
+ }
+
+ for parent in list(entity.get_parents()): # Make a change-resistant list copy.
+ if parent.name in visited_parents:
+ continue
+ visited_parents.add(parent.name)
+ parent_importance = importances.get(parent._flags.get("inheritance"), 999)
+ if parent.name in self:
+ deep_parent = self.get_deep(parent.name, # visited_props=visited_props,
+ visited_parents=visited_parents
+ )
+
+ for prop in deep_parent.properties:
+ importance = importances[deep_parent.get_importance(prop.name)]
+ if (importance <= parent_importance
+ and prop.name not in [prop.name for prop in entity.properties]):
+ entity.add_property(prop)
+ else:
+ print(f"Referenced parent \"{parent.name}\" not found in data model.")
+
+ for prop in list(entity.get_properties()): # Make a change-resistant list copy.
+ if prop.name in visited_props:
+ if visited_props[prop.name]:
+ deep_prop = visited_props[prop.name]
+ merge_entities(prop, deep_prop)
+ prop.datatype = deep_prop.datatype
+ prop.value = deep_prop.value
+ prop.unit = deep_prop.unit
+ continue
+ visited_props[prop.name] = None
+ if prop.name in self:
+ deep_prop = self.get_deep(prop.name, visited_props=visited_props,
+ visited_parents=visited_parents)
+ linked_prop = entity.get_property(prop)
+ if not linked_prop.datatype:
+ if deep_prop.role == "Property":
+ linked_prop.datatype = deep_prop.datatype
+ elif deep_prop.role == "RecordType":
+ linked_prop.datatype = deep_prop
+ if deep_prop.description:
+ linked_prop.description = deep_prop.description
+ visited_props[prop.name] = deep_prop
+ else:
+ print(f"Referenced property \"{prop.name}\" not found in data model.")
+
+ return entity
diff --git a/src/caosadvancedtools/models/parser.py b/src/caosadvancedtools/models/parser.py
index ad149222b5b90671a50943dc00bc9de8074a42f1..37f34e7bcbae48188c96b9bea6434d59571020fd 100644
--- a/src/caosadvancedtools/models/parser.py
+++ b/src/caosadvancedtools/models/parser.py
@@ -1,8 +1,8 @@
# This file is a part of the CaosDB Project.
#
-# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2023 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -35,16 +35,20 @@ not defined, simply the name can be supplied with no value.
Parents can be provided under the 'inherit_from_xxxx' keywords. The value needs
to be a list with the names. Here, NO NEW entities can be defined.
"""
+import argparse
import json
+import jsonref
import re
import sys
import yaml
-from typing import List
+from typing import List, Optional
+from warnings import warn
import jsonschema
-import caosdb as db
+import linkahead as db
+from linkahead.common.datatype import get_list_datatype
from .data_model import CAOSDB_INTERNAL_PROPERTIES, DataModel
# Keywords which are allowed in data model descriptions.
@@ -74,27 +78,11 @@ JSON_SCHEMA_ATOMIC_TYPES = [
"string",
"boolean",
"integer",
- "number"
+ "number",
+ "null"
]
-def _get_listdatatype(dtype):
- """matches a string to check whether the type definition is a list
-
- returns the type within the list or None, if it cannot be matched with a
- list definition
- """
- # TODO: string representation should be the same as used by the server:
- # e.g. LIST<TEXT>
- # this should be changed in the module and the old behavour should be
- # marked as depricated
- match = re.match(r"^LIST[(<](?P<dt>.*)[)>]$", dtype)
-
- if match is None:
- return None
- else:
- return match.group("dt")
-
# Taken from https://stackoverflow.com/a/53647080, CC-BY-SA, 2018 by
# https://stackoverflow.com/users/2572431/augurar
@@ -136,30 +124,82 @@ class JsonSchemaDefinitionError(RuntimeError):
super().__init__(msg)
-def parse_model_from_yaml(filename):
- """Shortcut if the Parser object is not needed."""
- parser = Parser()
+def parse_model_from_yaml(filename, existing_model: Optional[dict] = None, debug: bool = False):
+ """Parse a data model from a YAML file.
+
+This is a convenience function if the Parser object is not needed, it calls
+``Parser.parse_model_from_yaml(...)`` internally.
- return parser.parse_model_from_yaml(filename)
+Parameters
+----------
+
+existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
+debug : bool, optional
+ If True, turn on miscellaneous debugging. Default is False.
+ """
+ parser = Parser(debug=debug)
+
+ return parser.parse_model_from_yaml(filename, existing_model=existing_model)
-def parse_model_from_string(string):
- """Shortcut if the Parser object is not needed."""
- parser = Parser()
- return parser.parse_model_from_string(string)
+def parse_model_from_string(string, existing_model: Optional[dict] = None, debug: bool = False):
+ """Parse a data model from a YAML string
+This is a convenience function if the Parser object is not needed, it calls
+``Parser.parse_model_from_string(...)`` internally.
-def parse_model_from_json_schema(filename: str):
+Parameters
+----------
+
+existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
+debug : bool, optional
+ If True, turn on miscellaneous debugging. Default is False.
+ """
+ parser = Parser(debug=debug)
+
+ return parser.parse_model_from_string(string, existing_model=existing_model)
+
+
+def parse_model_from_json_schema(
+ filename: str,
+ top_level_recordtype: bool = True,
+ types_for_missing_array_items: dict = {},
+ ignore_unspecified_array_items: bool = False,
+ existing_model: Optional[dict] = None
+):
"""Return a datamodel parsed from a json schema definition.
Parameters
----------
+
filename : str
The path of the json schema file that is to be parsed
+ top_level_recordtype : bool, optional
+ Whether there is a record type defined at the top level of the
+ schema. Default is true.
+
+ types_for_missing_array_items : dict, optional
+ dictionary containing fall-back types for json entries with `type:
+ array` but without `items` specification. Default is an empty dict.
+
+ ignore_unspecified_array_items : bool, optional
+ Whether to ignore `type: array` entries the type of which is not
+ specified by their `items` property or given in
+ `types_for_missing_array_items`. An error is raised if they are not
+ ignored. Default is False.
+
+ existing_model : dict, optional
+ An existing model to which the created model shall be added. Not implemented yet.
+
Returns
-------
+
out : Datamodel
The datamodel generated from the input schema which then can be used for
synchronizing with CaosDB.
@@ -170,24 +210,34 @@ def parse_model_from_json_schema(filename: str):
about the limitations of the current implementation.
"""
+ if existing_model is not None:
+ raise NotImplementedError("Adding to an existing model is not implemented yet.")
+
# @author Florian Spreckelsen
# @date 2022-02-17
- # @review Daniel Hornung 2022-02-18
- parser = JsonSchemaParser()
+ # @review Timm Fitschen 2023-05-25
+ parser = JsonSchemaParser(types_for_missing_array_items, ignore_unspecified_array_items)
- return parser.parse_model_from_json_schema(filename)
+ return parser.parse_model_from_json_schema(filename, top_level_recordtype)
class Parser(object):
- def __init__(self):
+ def __init__(self, debug: bool = False):
"""Initialize an empty parser object and initialize the dictionary of entities and the list of
treated elements.
+Parameters
+----------
+
+debug : bool, optional
+ If True, turn on miscellaneous debugging. Default is False.
+
"""
self.model = {}
self.treated = []
+ self.debug = debug
- def parse_model_from_yaml(self, filename):
+ def parse_model_from_yaml(self, filename, existing_model: Optional[dict] = None):
"""Create and return a data model from the given file.
Parameters
@@ -195,6 +245,9 @@ class Parser(object):
filename : str
The path to the YAML file.
+ existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
Returns
-------
out : DataModel
@@ -203,9 +256,9 @@ class Parser(object):
with open(filename, 'r') as outfile:
ymlmodel = yaml.load(outfile, Loader=SafeLineLoader)
- return self._create_model_from_dict(ymlmodel)
+ return self._create_model_from_dict(ymlmodel, existing_model=existing_model)
- def parse_model_from_string(self, string):
+ def parse_model_from_string(self, string, existing_model: Optional[dict] = None):
"""Create and return a data model from the given YAML string.
Parameters
@@ -213,6 +266,9 @@ class Parser(object):
string : str
The YAML string.
+ existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
Returns
-------
out : DataModel
@@ -220,9 +276,9 @@ class Parser(object):
"""
ymlmodel = yaml.load(string, Loader=SafeLineLoader)
- return self._create_model_from_dict(ymlmodel)
+ return self._create_model_from_dict(ymlmodel, existing_model=existing_model)
- def _create_model_from_dict(self, ymlmodel):
+ def _create_model_from_dict(self, ymlmodel, existing_model: Optional[dict] = None):
"""Create and return a data model out of the YAML dict `ymlmodel`.
Parameters
@@ -230,6 +286,9 @@ class Parser(object):
ymlmodel : dict
The dictionary parsed from a YAML file.
+ existing_model : dict, optional
+ An existing model to which the created model shall be added.
+
Returns
-------
out : DataModel
@@ -239,6 +298,9 @@ class Parser(object):
if not isinstance(ymlmodel, dict):
raise ValueError("Yaml file should only contain one dictionary!")
+ if existing_model is not None:
+ self.model.update(existing_model)
+
# Extern keyword:
# The extern keyword can be used to include Properties and RecordTypes
# from existing CaosDB datamodels into the current model.
@@ -256,9 +318,9 @@ class Parser(object):
self.model[name] = db.Property(name=name).retrieve()
continue
for role in ("Property", "RecordType", "Record", "File"):
- if db.execute_query("COUNT {} {}".format(role, name)) > 0:
+ if db.execute_query("COUNT {} \"{}\"".format(role, name)) > 0:
self.model[name] = db.execute_query(
- "FIND {} WITH name={}".format(role, name), unique=True)
+ f"FIND {role} WITH name=\"{name}\"", unique=True)
break
else:
raise Exception("Did not find {}".format(name))
@@ -274,7 +336,12 @@ class Parser(object):
self._check_and_convert_datatypes()
for name, entity in ymlmodel.items():
- self._treat_entity(name, entity, line=ymlmodel["__line__"])
+ try:
+ self._treat_entity(name, entity, line=ymlmodel["__line__"])
+ except ValueError as err:
+ err_str = err.args[0].replace("invalid keyword:",
+ f"invalid keyword in line {entity['__line__']}:", 1)
+ raise ValueError(err_str, *err.args[1:]) from err
return DataModel(self.model.values())
@@ -325,13 +392,12 @@ class Parser(object):
if definition is None:
return
- if (self.model[name] is None
- and isinstance(definition, dict)
+ if (self.model[name] is None and isinstance(definition, dict)
# is it a property
and "datatype" in definition
# but not simply an RT of the model
- and not (_get_listdatatype(definition["datatype"]) == name and
- _get_listdatatype(definition["datatype"]) in self.model)):
+ and not (get_list_datatype(definition["datatype"]) == name and
+ get_list_datatype(definition["datatype"]) in self.model)):
# and create the new property
self.model[name] = db.Property(name=name,
@@ -381,6 +447,9 @@ class Parser(object):
raise YamlDefinitionError(line) from None
raise
+ if self.debug and self.model[name] is not None:
+ self.model[name].__line__ = definition["__line__"]
+
def _add_to_recordtype(self, ent_name, props, importance):
"""Add properties to a RecordType.
@@ -414,9 +483,9 @@ class Parser(object):
n = self._stringify(n)
if isinstance(e, dict):
- if "datatype" in e and _get_listdatatype(e["datatype"]) is not None:
+ if "datatype" in e and get_list_datatype(e["datatype"]) is not None:
# Reuse the existing datatype for lists.
- datatype = db.LIST(_get_listdatatype(e["datatype"]))
+ datatype = db.LIST(get_list_datatype(e["datatype"]))
else:
# Ignore a possible e["datatype"] here if it's not a list
# since it has been treated in the definition of the
@@ -438,6 +507,9 @@ class Parser(object):
def _inherit(self, name, prop, inheritance):
if not isinstance(prop, list):
+ if isinstance(prop, str):
+ raise YamlDefinitionError(
+ f"Parents must be a list but is given as string: {name} > {prop}")
raise YamlDefinitionError("Parents must be a list, error in line {}".format(
prop["__line__"]))
@@ -461,9 +533,13 @@ class Parser(object):
if not isinstance(definition, dict):
return
- if ("datatype" in definition
- and definition["datatype"].startswith("LIST")):
+ # These definition items must be handled even for list props.
+ for prop_name, prop in definition.items():
+ if prop_name == "description":
+ self.model[name].description = prop
+ # For lists, everything else is not needed at this level.
+ if ("datatype" in definition and definition["datatype"].startswith("LIST")):
return
if name in self.treated:
@@ -481,7 +557,8 @@ class Parser(object):
self.model[name].value = prop
elif prop_name == "description":
- self.model[name].description = prop
+ # Handled above
+ continue
elif prop_name == "recommended_properties":
self._add_to_recordtype(
@@ -518,6 +595,16 @@ class Parser(object):
self._inherit(name, prop, db.RECOMMENDED)
elif prop_name == "inherit_from_suggested":
self._inherit(name, prop, db.SUGGESTED)
+ elif prop_name == "parent":
+ warn(
+ DeprecationWarning(
+ "The `parent` keyword is deprecated and will be "
+ "removed in a future version. Use "
+ "`inherit_from_{obligatory|recommended|suggested}` "
+ "instead."
+ )
+ )
+ self._inherit(name, prop, db.OBLIGATORY)
else:
raise ValueError("invalid keyword: {}".format(prop_name))
@@ -545,15 +632,19 @@ class Parser(object):
dtype = value.datatype
is_list = False
- if _get_listdatatype(value.datatype) is not None:
- dtype = _get_listdatatype(value.datatype)
+ if get_list_datatype(dtype) is not None:
+ dtype = get_list_datatype(dtype)
is_list = True
- if dtype in self.model:
+ dtype_name = dtype
+ if not isinstance(dtype_name, str):
+ dtype_name = dtype.name
+
+ if dtype_name in self.model:
if is_list:
- value.datatype = db.LIST(self.model[dtype])
+ value.datatype = db.LIST(self.model[dtype_name])
else:
- value.datatype = self.model[dtype]
+ value.datatype = self.model[dtype_name]
continue
@@ -575,7 +666,7 @@ class Parser(object):
continue
raise ValueError("Property {} has an unknown datatype: {}".format(
- value.name, value.datatype))
+ value.name, dtype_name))
def _set_recordtypes(self):
""" properties are defined in first iteration; set remaining as RTs """
@@ -588,14 +679,13 @@ class Parser(object):
class JsonSchemaParser(Parser):
"""Extends the yaml parser to read in datamodels defined in a json schema.
- **EXPERIMENTAL:** While this calss can already be used to create data models
+ **EXPERIMENTAL:** While this class can already be used to create data models
from basic json schemas, there are the following limitations and missing
features:
* Due to limitations of json-schema itself, we currently do not support
inheritance in the imported data models
* The same goes for suggested properties of RecordTypes
- * Currently, ``$defs`` and ``$ref`` in the input schema are not resolved.
* Already defined RecordTypes and (scalar) Properties can't be re-used as
list properties
* Reference properties that are different from the referenced RT. (Although
@@ -603,15 +693,18 @@ class JsonSchemaParser(Parser):
* Values
* Roles
* The extern keyword from the yaml parser
- * Currently, a json-schema cannot be transformed into a data model if its
- root element isn't a RecordType (or Property) with ``title`` and ``type``.
"""
# @author Florian Spreckelsen
# @date 2022-02-17
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
- def parse_model_from_json_schema(self, filename: str):
+ def __init__(self, types_for_missing_array_items={}, ignore_unspecified_array_items=False):
+ super().__init__()
+ self.types_for_missing_array_items = types_for_missing_array_items
+ self.ignore_unspecified_array_items = ignore_unspecified_array_items
+
+ def parse_model_from_json_schema(self, filename: str, top_level_recordtype: bool = True):
"""Return a datamodel created from the definition in the json schema in
`filename`.
@@ -619,6 +712,9 @@ class JsonSchemaParser(Parser):
----------
filename : str
The path to the json-schema file containing the datamodel definition
+ top_level_recordtype : bool, optional
+ Whether there is a record type defined at the top level of the
+ schema. Default is true.
Returns
-------
@@ -627,13 +723,13 @@ class JsonSchemaParser(Parser):
"""
# @author Florian Spreckelsen
# @date 2022-02-17
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
with open(filename, 'r') as schema_file:
- model_dict = json.load(schema_file)
+ model_dict = jsonref.load(schema_file)
- return self._create_model_from_dict(model_dict)
+ return self._create_model_from_dict(model_dict, top_level_recordtype=top_level_recordtype)
- def _create_model_from_dict(self, model_dict: [dict, List[dict]]):
+ def _create_model_from_dict(self, model_dict: [dict, List[dict]], top_level_recordtype: bool = True):
"""Parse a dictionary and return the Datamodel created from it.
The dictionary was typically created from the model definition in a json schema file.
@@ -642,36 +738,68 @@ class JsonSchemaParser(Parser):
----------
model_dict : dict or list[dict]
One or several dictionaries read in from a json-schema file
+ top_level_recordtype : bool, optional
+ Whether there is a record type defined at the top level of the
+ schema. Default is true.
Returns
-------
our : DataModel
The datamodel defined in `model_dict`
"""
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
if isinstance(model_dict, dict):
model_dict = [model_dict]
for ii, elt in enumerate(model_dict):
- if "title" not in elt:
- raise JsonSchemaDefinitionError(
- f"Object {ii+1} is lacking the `title` key word")
- if "type" not in elt:
- raise JsonSchemaDefinitionError(
- f"Object {ii+1} is lacking the `type` key word")
- # Check if this is a valid Json Schema
try:
jsonschema.Draft202012Validator.check_schema(elt)
except jsonschema.SchemaError as err:
+ key = elt["title"] if "title" in elt else f"element {ii}"
raise JsonSchemaDefinitionError(
- f"Json Schema error in {elt['title']}:\n{str(err)}") from err
- name = self._stringify(elt["title"], context=elt)
- self._treat_element(elt, name)
+ f"Json Schema error in {key}:\n{str(err)}") from err
+
+ if top_level_recordtype:
+ if "title" not in elt:
+ raise JsonSchemaDefinitionError(
+ f"Object {ii+1} is lacking the `title` key word")
+ if "type" not in elt:
+ raise JsonSchemaDefinitionError(
+ f"Object {ii+1} is lacking the `type` key word")
+ # Check if this is a valid Json Schema
+ name = self._stringify(elt["title"], context=elt)
+ self._treat_element(elt, name)
+ elif "properties" in elt or "patternProperties" in elt:
+ # No top-level type but there are entities
+ if "properties" in elt:
+ for key, prop in elt["properties"].items():
+ name = self._get_name_from_property(key, prop)
+ self._treat_element(prop, name)
+ if "patternProperties" in elt:
+ # See also treatment in ``_treat_record_type``. Since here,
+ # there is no top-level RT we use the prefix `__Pattern`,
+ # i.e., the resulting Record Types will be called
+ # `__PatternElement`.
+ self._treat_pattern_properties(
+ elt["patternProperties"], name_prefix="__Pattern")
+ else:
+ # Neither RecordType itself, nor further properties in schema,
+ # so nothing to do here. Maybe add something in the future.
+ continue
return DataModel(self.model.values())
+ def _get_name_from_property(self, key: str, prop: dict):
+ # @review Timm Fitschen 2023-05-25
+ if "title" in prop:
+ name = self._stringify(prop["title"])
+ else:
+ name = self._stringify(key)
+
+ return name
+
def _get_atomic_datatype(self, elt):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
if elt["type"] == "string":
if "format" in elt and elt["format"] in ["date", "date-time"]:
return db.DATETIME
@@ -683,11 +811,15 @@ class JsonSchemaParser(Parser):
return db.DOUBLE
elif elt["type"] == "boolean":
return db.BOOLEAN
+ elif elt["type"] == "null":
+ # This could be any datatype since a valid json will never have a
+ # value in a null property. We use TEXT for convenience.
+ return db.TEXT
else:
raise JsonSchemaDefinitionError(f"Unkown atomic type in {elt}.")
def _treat_element(self, elt: dict, name: str):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
force_list = False
if name in self.model:
return self.model[name], force_list
@@ -698,12 +830,17 @@ class JsonSchemaParser(Parser):
if name == "name":
# This is identified with the CaosDB name property as long as the
# type is correct.
- if not elt["type"] == "string":
+ if not elt["type"] == "string" and "string" not in elt["type"]:
raise JsonSchemaDefinitionError(
"The 'name' property must be string-typed, otherwise it cannot "
"be identified with CaosDB's name property."
)
return None, force_list
+ # LinkAhead suports null for all types, so in the very special case of
+ # `"type": ["null", "<other_type>"]`, only consider the other type:
+ if isinstance(elt["type"], list) and len(elt["type"]) == 2 and "null" in elt["type"]:
+ elt["type"].remove("null")
+ elt["type"] = elt["type"][0]
if "enum" in elt:
ent = self._treat_enum(elt, name)
elif elt["type"] in JSON_SCHEMA_ATOMIC_TYPES:
@@ -721,11 +858,12 @@ class JsonSchemaParser(Parser):
# treat_something function
ent.description = elt["description"]
- self.model[name] = ent
+ if ent is not None:
+ self.model[name] = ent
return ent, force_list
def _treat_record_type(self, elt: dict, name: str):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
rt = db.RecordType(name=name)
if "required" in elt:
required = elt["required"]
@@ -733,10 +871,7 @@ class JsonSchemaParser(Parser):
required = []
if "properties" in elt:
for key, prop in elt["properties"].items():
- if "title" in prop:
- name = self._stringify(prop["title"])
- else:
- name = self._stringify(key)
+ name = self._get_name_from_property(key, prop)
prop_ent, force_list = self._treat_element(prop, name)
if prop_ent is None:
# Nothing to be appended since the property has to be
@@ -750,6 +885,17 @@ class JsonSchemaParser(Parser):
rt.add_property(prop_ent, importance=importance,
datatype=db.LIST(prop_ent))
+ if "patternProperties" in elt:
+
+ pattern_property_rts = self._treat_pattern_properties(
+ elt["patternProperties"], name_prefix=name)
+ for ppr in pattern_property_rts:
+ # add reference to pattern property type. These can never be
+ # obligatory since pattern properties cannot be required in the
+ # original schema (since their actual names are not known a
+ # priori).
+ rt.add_property(ppr)
+
if "description" in elt:
rt.description = elt["description"]
return rt
@@ -771,30 +917,118 @@ class JsonSchemaParser(Parser):
return rt
def _treat_list(self, elt: dict, name: str):
- # @review Timm Fitschen 2022-02-30
+ # @review Timm Fitschen 2023-05-25
- if "items" not in elt:
+ if "items" not in elt and name not in self.types_for_missing_array_items:
+ if self.ignore_unspecified_array_items:
+ return None, False
raise JsonSchemaDefinitionError(
f"The definition of the list items is missing in {elt}.")
- items = elt["items"]
- if "enum" in items:
- return self._treat_enum(items, name), True
- if items["type"] in JSON_SCHEMA_ATOMIC_TYPES:
- datatype = db.LIST(self._get_atomic_datatype(items))
+ if "items" in elt:
+ items = elt["items"]
+ if "enum" in items:
+ return self._treat_enum(items, name), True
+ if items["type"] in JSON_SCHEMA_ATOMIC_TYPES:
+ datatype = db.LIST(self._get_atomic_datatype(items))
+ return db.Property(name=name, datatype=datatype), False
+ if items["type"] == "object":
+ if "title" not in items or self._stringify(items["title"]) == name:
+ # Property is RecordType
+ return self._treat_record_type(items, name), True
+ else:
+ # List property will be an entity of its own with a name
+ # different from the referenced RT
+ ref_rt = self._treat_record_type(
+ items, self._stringify(items["title"]))
+ self.model[ref_rt.name] = ref_rt
+ return db.Property(name=name, datatype=db.LIST(ref_rt)), False
+ else:
+ # Use predefined type:
+ datatype = db.LIST(self.types_for_missing_array_items[name])
return db.Property(name=name, datatype=datatype), False
- if items["type"] == "object":
- if "title" not in items or self._stringify(items["title"]) == name:
- # Property is RecordType
- return self._treat_record_type(items, name), True
+
+ def _get_pattern_prop(self):
+ # @review Timm Fitschen 2023-05-25
+ if "__pattern_property_pattern_property" in self.model:
+ return self.model["__pattern_property_pattern_property"]
+ pp = db.Property(name="__matched_pattern", datatype=db.TEXT)
+ self.model["__pattern_property_pattern_property"] = pp
+ return pp
+
+ def _treat_pattern_properties(self, pattern_elements, name_prefix=""):
+ """Special Treatment for pattern properties: A RecordType is created for
+ each pattern property. In case of a `type: object` PatternProperty, the
+ remaining properties of the JSON entry are appended to the new
+ RecordType; in case of an atomic type PatternProperty, a single value
+ Property is added to the RecordType.
+
+ Raises
+ ------
+ NotImplementedError
+ In case of patternProperties with non-object, non-atomic type, e.g.,
+ array.
+
+ """
+ # @review Timm Fitschen 2023-05-25
+ num_patterns = len(pattern_elements)
+ pattern_prop = self._get_pattern_prop()
+ returns = []
+ for ii, (key, element) in enumerate(pattern_elements.items()):
+ if "title" not in element:
+ name_suffix = f"_{ii+1}" if num_patterns > 1 else ""
+ name = name_prefix + "Entry" + name_suffix
+ else:
+ name = element["title"]
+ if element["type"] == "object":
+ # simple, is already an object, so can be treated like any other
+ # record type.
+ pattern_type = self._treat_record_type(element, name)
+ elif element["type"] in JSON_SCHEMA_ATOMIC_TYPES:
+ # create a property that stores the actual value of the pattern
+ # property.
+ propname = f"{name}_value"
+ prop = db.Property(name=propname, datatype=self._get_atomic_datatype(element))
+ self.model[propname] = prop
+ pattern_type = db.RecordType(name=name)
+ pattern_type.add_property(prop)
+ else:
+ raise NotImplementedError(
+ "Pattern properties are currently only supported for types " +
+ ", ".join(JSON_SCHEMA_ATOMIC_TYPES) + ", and object.")
+
+ # Add pattern property and description
+ pattern_type.add_property(pattern_prop, importance=db.OBLIGATORY)
+ if pattern_type.description:
+ pattern_type.description += f"\n\npattern: {key}"
else:
- # List property will be an entity of its own with a name
- # different from the referenced RT
- ref_rt = self._treat_record_type(
- items, self._stringify(items["title"]))
- self.model[ref_rt.name] = ref_rt
- return db.Property(name=name, datatype=db.LIST(ref_rt)), False
+ pattern_type.description = f"pattern: {key}"
+
+ self.model[name] = pattern_type
+ returns.append(pattern_type)
+
+ return returns
if __name__ == "__main__":
- model = parse_model_from_yaml('data_model.yml')
- print(model)
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawTextHelpFormatter)
+ parser.add_argument("data_model",
+ help="Path name of the data model file (yaml or json) to be used.")
+ parser.add_argument("--sync", action="store_true",
+ help="Whether or not to sync the data model with the server.")
+ parser.add_argument("--noquestion", action="store_true",
+ help="Whether or not to ask questions during synchronization.")
+ parser.add_argument("--print", action="store_true",
+ help="Whether or not to print the data model.")
+
+ args = parser.parse_args()
+ if args.data_model.endswith(".json"):
+ model = parse_model_from_json_schema(args.data_model)
+ elif args.data_model.endswith(".yml") or args.data_model.endswith(".yaml"):
+ model = parse_model_from_yaml(args.data_model)
+ else:
+ RuntimeError("did not recognize file ending")
+ if args.print:
+ print(model)
+ if args.sync:
+ model.sync_data_model(noquestion=args.noquestion)
diff --git a/src/caosadvancedtools/scifolder/utils.py b/src/caosadvancedtools/scifolder/utils.py
index afa671af85506a57a06ad5198bec4495823c76f1..50e897c7d2f19c6269ec622489c5a2c6ce1a28e0 100644
--- a/src/caosadvancedtools/scifolder/utils.py
+++ b/src/caosadvancedtools/scifolder/utils.py
@@ -154,7 +154,7 @@ def create_files_list(df, ftype):
files = []
for indx, src in df.loc[ftype,
- pd.notnull(df.loc[ftype])].iteritems():
+ pd.notnull(df.loc[ftype])].items():
desc = df.loc[ftype+" description", indx]
if pd.notnull(desc):
diff --git a/src/caosadvancedtools/table_converter.py b/src/caosadvancedtools/table_converter.py
index 76f4dfcdb5f040d81d923289a7a730806ad8681b..4b8591ed009ee8e63b328ad43e0d458b3e805ce7 100644
--- a/src/caosadvancedtools/table_converter.py
+++ b/src/caosadvancedtools/table_converter.py
@@ -79,7 +79,7 @@ def from_table(spreadsheet, recordtype):
rec = db.Record()
rec.add_parent(name=recordtype)
- for key, value in row.iteritems():
+ for key, value in row.items():
if key.lower() == "description":
rec.description = value
continue
diff --git a/src/caosadvancedtools/table_export.py b/src/caosadvancedtools/table_export.py
index 056207a76fa01357e2269cd4cb8e9a09905d5d90..eabb10754bdb93859dcc6ef3d3ff0838fa6ff6d4 100644
--- a/src/caosadvancedtools/table_export.py
+++ b/src/caosadvancedtools/table_export.py
@@ -27,6 +27,7 @@ them for an export as a table, e.g., for the export to metadata
repositories.
"""
+from inspect import signature
import json
import logging
@@ -83,7 +84,7 @@ class BaseTableExporter(object):
```
{"entry_to_be_exported: {
"optional": True/False
- "find_func": name of member function
+ "find_func": callable or name of member function
"query": query string
"selector": selector for the query
"error": error explanation
@@ -97,8 +98,8 @@ class BaseTableExporter(object):
- optional: True or False, if not present, the entry is
assumed to be mandatory.
- find_func: name of the member function that returns the
- value for this entry. Must not exist together with
- `query`
+ value for this entry or callable object. Must not exist
+ together with `query`
- query: Query string for finding the value for this
entry. If this is given, a record must be given to the
constructor of this class. The query is then executed as
@@ -132,6 +133,7 @@ class BaseTableExporter(object):
self._check_sanity_of_export_dict()
self.raise_error_if_missing = raise_error_if_missing
self.info = {}
+ self.all_keys = [key for key in self.export_dict]
def collect_information(self):
"""Use the items of `export_dict` to collect the information for the
@@ -139,7 +141,8 @@ class BaseTableExporter(object):
"""
- for e, d in self.export_dict.items():
+ for e in self.all_keys:
+ d = self.export_dict[e]
if QUERY in d:
# TODO: How do we make this more general? There might
# be queries that don't need the record or work with
@@ -163,12 +166,15 @@ class BaseTableExporter(object):
else:
self._append_missing(e, d)
elif FIND_FUNCTION in d:
- find_fun = getattr(self, d[FIND_FUNCTION])
try:
- self.info[e] = find_fun()
+ val = self._call_find_function(d[FIND_FUNCTION], e)
+ if val is not None:
+ self.info[e] = val
+ else:
+ self._append_missing(e, d)
except Exception as exc:
self._append_missing(e, d)
- logger.debug(exc)
+ logger.error(exc)
# last resort: check if record has e as property:
else:
try:
@@ -200,6 +206,20 @@ class BaseTableExporter(object):
else:
logger.error(errmssg)
+ def _call_find_function(self, find_function, e):
+ if callable(find_function):
+ find_fun = find_function
+ else:
+ find_fun = getattr(self, find_function)
+
+ sig = signature(find_fun)
+ params = sig.parameters
+ if len(params) > 1:
+ return find_fun(self.record, e)
+ elif len(params) > 0:
+ return find_fun(self.record)
+ return find_fun()
+
def prepare_csv_export(self, delimiter=',', print_header=False,
skip_empty_optionals=False):
"""Return the values in self.info as a single-line string, separated
@@ -238,7 +258,8 @@ class BaseTableExporter(object):
if print_header:
header = ""
- for e, d in self.export_dict.items():
+ for e in self.all_keys:
+ d = self.export_dict[e]
if e in self.info:
body += str(self.info[e]) + delimiter
@@ -287,7 +308,9 @@ class BaseTableExporter(object):
# check find function if present
if FIND_FUNCTION in d:
- if not hasattr(self, d[FIND_FUNCTION]):
+ if callable(d[FIND_FUNCTION]):
+ pass
+ elif not hasattr(self, d[FIND_FUNCTION]):
raise TableExportError(
"Find function " + d[FIND_FUNCTION] +
" was specified for entry " + e +
diff --git a/src/caosadvancedtools/table_importer.py b/src/caosadvancedtools/table_importer.py
index 1f515e78e3ddbd198fa0336589a359ba9154f038..bae813b23195c93ccfd369a626424dd069164fb0 100755
--- a/src/caosadvancedtools/table_importer.py
+++ b/src/caosadvancedtools/table_importer.py
@@ -210,7 +210,7 @@ class TableImporter():
"""
def __init__(self, converters, obligatory_columns=None, unique_keys=None,
- datatypes=None):
+ datatypes=None, existing_columns=None):
"""
Parameters
----------
@@ -221,7 +221,7 @@ class TableImporter():
value check is not necessary.
obligatory_columns : list, optional
- List of column names, each listed column must not have missing values.
+ List of column names that (if they exist) must not have missing values.
unique_keys : list, optional
List of column names that in combination must be unique: each row has a unique
@@ -232,22 +232,31 @@ class TableImporter():
checked whether they have the provided datatype. This dict also defines what columns are
required to exist throught the existing keys.
+ existing_columns : list, optional
+ List of column names that must exist but may have missing (NULL) values
"""
if converters is None:
converters = {}
+ self.converters = converters
+
+ if obligatory_columns is None:
+ obligatory_columns = []
+ self.obligatory_columns = obligatory_columns
+
+ if unique_keys is None:
+ unique_keys = []
+ self.unique_keys = unique_keys
if datatypes is None:
datatypes = {}
+ self.datatypes = datatypes
+
+ if existing_columns is None:
+ existing_columns = []
+ self.existing_columns = existing_columns
self.sup = SuppressKnown()
- self.required_columns = list(converters.keys())+list(datatypes.keys())
- self.obligatory_columns = ([]
- if obligatory_columns is None
- else obligatory_columns)
- self.unique_keys = [] if unique_keys is None else unique_keys
- self.converters = converters
- self.datatypes = datatypes
def read_file(self, filename, **kwargs):
raise NotImplementedError()
@@ -263,7 +272,7 @@ class TableImporter():
"""
- for col in self.required_columns:
+ for col in self.existing_columns:
if col not in df.columns:
errmsg = "Column '{}' missing in ".format(col)
errmsg += ("\n{}.\n".format(filename) if filename
@@ -313,7 +322,7 @@ class TableImporter():
.. note::
If columns are integer, but should be float, this method converts the respective columns
- in place.
+ in place. The same for columns that should have string value but have numeric value.
Parameters
----------
@@ -323,18 +332,21 @@ class TableImporter():
"""
for key, datatype in self.datatypes.items():
+ if key not in df.columns:
+ continue
# Check for castable numeric types first: We unconditionally cast int to the default
# float, because CaosDB does not have different sizes anyway.
col_dtype = df.dtypes[key]
if not strict and not np.issubdtype(col_dtype, datatype):
- issub = np.issubdtype
# These special cases should be fine.
- if issub(col_dtype, np.integer) and issub(datatype, np.floating):
+ if ((datatype == str)
+ or (np.issubdtype(col_dtype, np.integer)
+ and np.issubdtype(datatype, np.floating))
+ ): # NOQA
df[key] = df[key].astype(datatype)
# Now check each element
- for idx, val in df.loc[
- pd.notnull(df.loc[:, key]), key].iteritems():
+ for idx, val in df.loc[pd.notnull(df.loc[:, key]), key].items():
if not isinstance(val, datatype):
msg = (
@@ -363,24 +375,23 @@ class TableImporter():
for index, row in df.iterrows():
# if none of the relevant information is given, skip
-
- if np.array([pd.isnull(row.loc[key]) for key in
- self.obligatory_columns]).all():
-
+ if pd.isnull(row.loc[[key for key in self.obligatory_columns if key in df.columns]]).all():
df = df.drop(index)
continue
# if any of the relevant information is missing, report it
-
i = 0
okay = True
while okay and i < len(self.obligatory_columns):
key = self.obligatory_columns[i]
i += 1
+ if key not in df.columns:
+ continue
- if pd.isnull(row.loc[key]):
+ null_check = pd.isnull(row.loc[key])
+ if (isinstance(null_check, np.ndarray) and null_check.any()) or (not isinstance(null_check, np.ndarray) and null_check):
errmsg = (
"Required information is missing ({}) in {}. row"
" (without header) of "
@@ -449,7 +460,10 @@ class XLSImporter(TableImporter):
"All but the first are being ignored.".format(filename))
try:
- df = xls_file.parse(converters=self.converters, **kwargs)
+ tmpdf = xls_file.parse(**kwargs)
+ applicable_converters = {k: v for k, v in self.converters.items()
+ if k in tmpdf.columns}
+ df = xls_file.parse(converters=applicable_converters, **kwargs)
except Exception as e:
logger.warning(
"Cannot parse {}.\n{}".format(filename, e),
@@ -465,7 +479,11 @@ class XLSImporter(TableImporter):
class CSVImporter(TableImporter):
def read_file(self, filename, sep=",", **kwargs):
try:
- df = pd.read_csv(filename, sep=sep, converters=self.converters,
+ tmpdf = pd.read_csv(filename, sep=sep, converters=self.converters,
+ **kwargs)
+ applicable_converters = {k: v for k, v in self.converters.items()
+ if k in tmpdf.columns}
+ df = pd.read_csv(filename, sep=sep, converters=applicable_converters,
**kwargs)
except ValueError as ve:
logger.warning(
@@ -482,6 +500,10 @@ class CSVImporter(TableImporter):
class TSVImporter(TableImporter):
def read_file(self, filename, **kwargs):
try:
+ tmpdf = pd.read_csv(filename, sep="\t", converters=self.converters,
+ **kwargs)
+ applicable_converters = {k: v for k, v in self.converters.items()
+ if k in tmpdf.columns}
df = pd.read_csv(filename, sep="\t", converters=self.converters,
**kwargs)
except ValueError as ve:
diff --git a/src/caosadvancedtools/utils.py b/src/caosadvancedtools/utils.py
index 2504f56976e2f6122f3e3468db1c7ae807bbb8cd..4d6a4b36bcb5dbdcd5bfe9357c53d4e9aa3501ca 100644
--- a/src/caosadvancedtools/utils.py
+++ b/src/caosadvancedtools/utils.py
@@ -55,6 +55,26 @@ def replace_path_prefix(path, old_prefix, new_prefix):
return os.path.join(new_prefix, path)
+def create_entity_link(entity: db.Entity, base_url: str = ""):
+ """
+ creates a string that contains the code for an html link to the provided entity.
+
+ The text of the link is the entity name if one exists and the id otherwise.
+
+ Args:
+ entity (db.Entity): the entity object to which the link will point
+ base_url (str): optional, by default, the url starts with '/Entity' and thus is relative.
+ You can provide a base url that will be prefixed.
+ Returns:
+ str: the string containing the html code
+
+ """
+ return "<a href='{}/Entity/{}'>{}</a>".format(
+ base_url,
+ entity.id,
+ entity.name if entity.name is not None else entity.id)
+
+
def string_to_person(person):
"""
Creates a Person Record from a string.
diff --git a/src/doc/conf.py b/src/doc/conf.py
index c7f82a99d3b287ca72ca57430b2d4b868539d39e..590feca405444032b0cdaa5fefc3aff09538bff0 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -23,13 +23,13 @@ import sphinx_rtd_theme
# -- Project information -----------------------------------------------------
project = 'caosadvancedtools'
-copyright = '2021, IndiScale GmbH'
+copyright = '2023, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.4.1'
+version = '0.9.1'
# The full version, including alpha/beta/rc tags
-release = '0.4.1'
+release = '0.9.1-dev'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/crawler.rst b/src/doc/crawler.rst
index 4b99c97e6db16e5691f373fa5fb4903e4d078155..aea023192c0b95c784d5ac91ade12d0c30591d42 100644
--- a/src/doc/crawler.rst
+++ b/src/doc/crawler.rst
@@ -121,6 +121,14 @@ as seen by the CaosDB server (The actual path may vary. This is the used
in the LinkAhead distribution of CaosDB). In this case the root file
system as seen from within the CaosDB docker process is used.
+
+You can provide a ``.caosdbignore`` file as a commandline option to the above
+loadFiles command. The syntax of that file is the same as for `gitignore
+<https://git-scm.com/docs/gitignore>`_ files. Note, that you can have additional
+``.caosdbignore`` files at lower levels which are appended to the current ignore
+file and have an effect of the respective subtree.
+
+
Extending the Crawlers
======================
diff --git a/src/doc/index.rst b/src/doc/index.rst
index 9aa045349ab05d3f5130a7f33b38c7eca0c4f32e..6c2c5f9894ad5c0f5dc3f124de726d264f46d452 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -13,9 +13,9 @@ This documentation helps you to :doc:`get started<getting_started>`, explains th
Getting started <README_SETUP>
Concepts <concepts>
- tutorials
- Caosdb-Crawler <crawler>
- YAML Interface <yaml_interface>
+ The Caosdb Crawler <crawler>
+ YAML data model specification <yaml_interface>
+ Specifying a datamodel with JSON schema <json_schema_interface>
_apidoc/modules
diff --git a/src/doc/json_schema_interface.rst b/src/doc/json_schema_interface.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0e8aebd3a4204f29608212f7ed0c115fd1d4a134
--- /dev/null
+++ b/src/doc/json_schema_interface.rst
@@ -0,0 +1,75 @@
+Defining datamodels with a JSON schema specification
+====================================================
+
+TODO, see https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/42
+
+Further information
+###################
+
+Pattern Properties
+%%%%%%%%%%%%%%%%%%
+
+The JSON-schema parser has rudimentary support for ``patternProperties``. Since
+their names (only the pattern that their names will suffice) are not known a
+priori, we create RecordTypes for all pattern properties. The names of these
+RecordTypes are created from their parent element's name by appending the string
+``"Entry"`` and possibly a number if there are more than one pattern properties
+for one parent.
+
+All the RecordTypes created for pattern properties have at least an obligatory
+``__matched_pattern`` property which will -- as the name suggests -- store the
+matched pattern of an actual data entry.
+
+.. note::
+
+ The ``__matched_pattern`` property is added automatically to your datamodel
+ as soon as there is at least one pattern property in your JSON schema. So be
+ sure that you don't happen to have an entity with exactly this name in your
+ database.
+
+E.g., a json schema with
+
+.. code-block:: json
+
+ "dataset": {
+ "patternProperties": {
+ "^[0-9]{4,4}": {
+ "type": "boolean"
+ },
+ "^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}": {
+ "type": "object",
+ "properties": {
+ "date_id": {
+ "$ref": "#/definitions/uuid"
+ }
+ }
+ }
+ }
+ }
+
+Would result in a ``Dataset`` RecordType that has the two properties
+``DatasetEntry_1`` and ``DatasetEntry_2`` (as always, name can be overwritten
+explicitly by specifying the ``title`` property), referencing corresponding
+``DatasetEntry_1`` and ``DatasetEntry_2`` Records.
+
+Apart from the aforementioned ``__matched_pattern`` property, ``DatasetEntry_1``
+also has the ``DatasetEntry_1_value`` property with datatype ``BOOLEAN``, that
+stores the actual value. In turn, ``DatasetEntry_2`` is of ``type: object`` and
+is treated like any other RecordType. Consequently, it has, appart from the
+``__matched_pattern`` property, a ``date_id`` property as specified in its
+``properties``.
+
+Array entries without ``items`` specification
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+JSON schema allows for properties of ``type: array`` without the ``items``
+specification that consequently can be arrays of any (and of mixed) types. While
+this is in general problematic when specifying a data model, sometimes these
+properties cannot be specified further, e.g., when you're using an external
+schema that you cannot change.
+
+These properties can still be added to your datamodel by specifying their types
+explicitly in a dictionary or, alternatively, they can be ignored. See the
+``types_for_missing_array_items`` and ``ignore_unspecified_array_items``
+parameters of ``models.parser.JsonSchemaParser``, respectively, for more
+information.
diff --git a/src/doc/yaml_interface.rst b/src/doc/yaml_interface.rst
index 476e92829238a0fc9dac851c61790c022e9fcde9..ac3914385d31df5a306b3f8400fbcb6b005f17fa 100644
--- a/src/doc/yaml_interface.rst
+++ b/src/doc/yaml_interface.rst
@@ -1,10 +1,14 @@
-YAML-Interface
---------------
-The yaml interface is a module in caosdb-pylib that can be used to create and update
+===============================
+ YAML data model specification
+===============================
+
+The ``caosadvancedtools`` library features the possibility to create and update
CaosDB models using a simplified definition in YAML format.
-Let's start with an example taken from https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/dev/unittests/model.yml.
+Let's start with an example taken from `model.yml
+<https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/dev/unittests/model.yml>`__
+in the library sources.
.. code-block:: yaml
@@ -44,7 +48,7 @@ Let's start with an example taken from https://gitlab.indiscale.com/caosdb/src/c
-This example defines 3 ``RecordType``s:
+This example defines 3 ``RecordTypes``:
- A ``Project`` with one obligatory property ``datatype``
- A Person with a ``firstName`` and a ``lastName`` (as recommended properties)
@@ -69,8 +73,10 @@ Note the difference between the three property declarations of ``LabbookEntry``:
If the data model depends on record types or properties which already exist in CaosDB, those can be
added using the ``extern`` keyword: ``extern`` takes a list of previously defined names.
+
+
Datatypes
----------
+=========
You can use any data type understood by CaosDB as datatype attribute in the yaml model.
@@ -90,9 +96,8 @@ would declare a list of elements with datatype Project.
Keywords
---------
+========
-- **parent**: Parent of this entity.
- **importance**: Importance of this entity. Possible values: "recommended", "obligatory", "suggested"
- **datatype**: The datatype of this property, e.g. TEXT, INTEGER or Project.
- **unit**: The unit of the property, e.g. "m/s".
@@ -100,12 +105,14 @@ Keywords
- **recommended_properties**: Add properties to this entity with importance "recommended".
- **obligatory_properties**: Add properties to this entity with importance "obligatory".
- **suggested_properties**: Add properties to this entity with importance "suggested".
-- **inherit_from_recommended**: Inherit from another entity using the specified importance level including the higher importance level "obligatory". This would add a corresponding parent and add all obligatory and recommended properties from the parent.
-- **inherit_from_suggested including higher importance levels**: Inherit from another entity using the specified importance level. This would add a corresponding parent and add all obligatory, recommended and suggested properties from the parent.
-- **inherit_from_obligatory**: Inherit from another entity using the specified importance level. This would add a corresponding parent and add all obligatory properties from the parent.
+- **inherit_from_XXX**: This keyword accepts a list of other RecordTypes. Those RecordTypes are
+ added as parents, and all Properties with at least the importance ``XXX`` are inherited. For
+ example, ``inherited_from_recommended`` will inherit all Properties of importance ``recommended``
+ and ``obligatory``, but not ``suggested``.
+- **parent**: Parent of this entity. Same as ``inherit_from_obligatory``. (*Deprecated*)
Usage
------
+=====
You can use the yaml parser directly in python as follows:
@@ -118,9 +125,19 @@ You can use the yaml parser directly in python as follows:
This creates a DataModel object containing all entities defined in the yaml file.
-You can then use the functions from caosadvancedtools.models.data_model.DataModel to synchronize
+If the parsed data model shall be appended to a pre-exsting data model, the optional
+``extisting_model`` can be used:
+
+.. code-block:: python
+
+ new_model = parser.parse_model_from_yaml("model.yml", existing_model=old_model)
+
+You can now use the functions from ``DataModel`` to synchronize
the model with a CaosDB instance, e.g.:
.. code-block:: python
model.sync_data_model()
+
+.. LocalWords: yml projectId UID firstName lastName LabbookEntry entryId textElement labbook
+.. LocalWords: associatedFile extern Textfile DataModel
diff --git a/tox.ini b/tox.ini
index dde34b987b9b08bfdfc51a06dd46a9a0e0494f28..00548dea25c5017f1d0301a00a629c62d16631ef 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,16 +1,23 @@
[tox]
-envlist=py36, py37, py38, py39, py310
+envlist=py37, py38, py39, py310, py311
skip_missing_interpreters = true
+
[testenv]
deps=nose
pandas
git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
pytest
pytest-cov
- openpyxl
+ gitignore-parser
+ openpyxl >= 3.0.7
xlrd == 1.2
h5py
-commands=py.test --cov=caosadvancedtools -vv {posargs}
+commands=py.test --cov=caosadvancedtools --cov-report=html:.tox/cov_html -vv {posargs}
[flake8]
max-line-length=100
+
+[pytest]
+testpaths = unittests
+addopts = -vv
+xfail_strict = True
diff --git a/unittests/caosdbignore-example b/unittests/caosdbignore-example
new file mode 100644
index 0000000000000000000000000000000000000000..04500046def6f8d1a7efb1cdceeff7e85f8e1f91
--- /dev/null
+++ b/unittests/caosdbignore-example
@@ -0,0 +1,2 @@
+README.md
+.caosdbignore*
diff --git a/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore b/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore
new file mode 100644
index 0000000000000000000000000000000000000000..43b9cb201ed2ae7e193a1755d4e935a8d7b17a23
--- /dev/null
+++ b/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore
@@ -0,0 +1 @@
+!README.md
diff --git a/unittests/json-schema-models/datamodel_atomic_properties.schema.json b/unittests/json-schema-models/datamodel_atomic_properties.schema.json
index 3828f131180a839d5c9b8bc5aa1a1285717da723..7b4a23e5bb48b995d07a261bcae0a8a486b7969a 100644
--- a/unittests/json-schema-models/datamodel_atomic_properties.schema.json
+++ b/unittests/json-schema-models/datamodel_atomic_properties.schema.json
@@ -18,7 +18,8 @@
"date": { "type": "string", "format": "date" },
"integer": { "type": "integer", "description": "Some integer property" },
"boolean": { "type": "boolean" },
- "number_prop": { "type": "number", "description": "Some float property" }
+ "number_prop": { "type": "number", "description": "Some float property" },
+ "null_prop": { "type": "null", "description": "This property will never have a value." }
}
}
]
diff --git a/unittests/json-schema-models/datamodel_missing_array_items.schema.json b/unittests/json-schema-models/datamodel_missing_array_items.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..8ac17ac3162def3dbf070d7027fd318366bb4682
--- /dev/null
+++ b/unittests/json-schema-models/datamodel_missing_array_items.schema.json
@@ -0,0 +1,9 @@
+{
+ "title": "something_with_missing_array_items",
+ "type": "object",
+ "properties": {
+ "missing": {
+ "type": "array"
+ }
+ }
+}
diff --git a/unittests/json-schema-models/datamodel_no_toplevel_entity.schema.json b/unittests/json-schema-models/datamodel_no_toplevel_entity.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..35240d765479b719576e6ee67e387790d3d6d160
--- /dev/null
+++ b/unittests/json-schema-models/datamodel_no_toplevel_entity.schema.json
@@ -0,0 +1,56 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://my-schema-id.net",
+ "type": "object",
+ "definitions": {
+ "uuid": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+ },
+ "datetime": {
+ "type": "string",
+ "format": "date-time"
+ }
+ },
+ "properties": {
+ "Dataset1": {
+ "title": "Dataset1",
+ "description": "Some description",
+ "type": "object",
+ "properties": {
+ "title": {
+ "type": "string",
+ "description": "full dataset title"
+ },
+ "campaign": {
+ "type": "string",
+ "description": "FIXME"
+ },
+ "number_prop": {
+ "type": "number",
+ "description": "Some float property"
+ },
+ "user_id": {
+ "$ref": "#/definitions/uuid"
+ }
+ },
+ "required": ["title", "number_prop"]
+ }
+ },
+ "patternProperties": {
+ "^[0-9]{4,4}": {
+ "type": "boolean"
+ },
+ "^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}": {
+ "type": "object",
+ "properties": {
+ "date_id": {
+ "$ref": "#/definitions/uuid"
+ }
+ }
+ }
+ }
+}
diff --git a/unittests/json-schema-models/datamodel_pattern_properties.schema.json b/unittests/json-schema-models/datamodel_pattern_properties.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..9b85c7b80cf0990713f8f130050c21751e311b42
--- /dev/null
+++ b/unittests/json-schema-models/datamodel_pattern_properties.schema.json
@@ -0,0 +1,39 @@
+[
+ {
+ "title": "Dataset",
+ "type": "object",
+ "patternProperties": {
+ "^[0-9]{4,4}": {
+ "type": "boolean"
+ },
+ "^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}": {
+ "type": "object",
+ "properties": {
+ "date_id": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+ }
+ }
+ }
+ }
+ },
+ {
+ "title": "Dataset2",
+ "type": "object",
+ "properties": {
+ "datetime": {
+ "type": "string",
+ "format": "date-time"
+ }
+ },
+ "patternProperties": {
+ ".*": {
+ "title": "Literally anything",
+ "type": "object"
+ }
+ }
+ }
+]
diff --git a/unittests/model.yml b/unittests/models/model.yml
similarity index 100%
rename from unittests/model.yml
rename to unittests/models/model.yml
diff --git a/unittests/models/model_invalid.yml b/unittests/models/model_invalid.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c8368b9701db9b3461b7e0f1f3514c2411f56b56
--- /dev/null
+++ b/unittests/models/model_invalid.yml
@@ -0,0 +1,2 @@
+Project:
+ ObligatoryProperties:
diff --git a/unittests/test_base_table_exporter.py b/unittests/test_base_table_exporter.py
index 3b8276cdf947c5b22e829e050295dd47f3cfe9ea..8a65b71aa489f8fca457c0e700452a6dc5956eed 100644
--- a/unittests/test_base_table_exporter.py
+++ b/unittests/test_base_table_exporter.py
@@ -82,13 +82,13 @@ def test_simple_record():
assert my_exporter.prepare_csv_export(
delimiter='\t', print_header=True) == "Test_Prop_1\tTest_Prop_2\nbla\tblabla"
# remove optional entry from info
- del(my_exporter.info["Test_Prop_2"])
+ del my_exporter.info["Test_Prop_2"]
assert my_exporter.prepare_csv_export(skip_empty_optionals=True) == "bla"
assert my_exporter.prepare_csv_export(
delimiter='\t', print_header=True) == "Test_Prop_1\tTest_Prop_2\nbla\t"
# reload info, and delete mandatory entry
my_exporter.collect_information()
- del(my_exporter.info["Test_Prop_1"])
+ del my_exporter.info["Test_Prop_1"]
with raises(te.TableExportError) as exc:
my_exporter.prepare_csv_export()
assert "Test_Prop_1" in exc.value.msg
@@ -184,7 +184,7 @@ def test_info_collection():
assert "optional_value" not in my_exporter.info
# now error in mandatory value
- del(export_dict["optional_value"])
+ del export_dict["optional_value"]
export_dict["mandatory_value"] = {
"find_func": "find_function_with_error"
}
diff --git a/unittests/test_cache.py b/unittests/test_cache.py
index 2d7b863fe971dd61a575e24f52853de4f5c4e204..de3430bf2f28a6b05ea36b1047ac11937809ff44 100644
--- a/unittests/test_cache.py
+++ b/unittests/test_cache.py
@@ -24,31 +24,35 @@ import os
import unittest
from copy import deepcopy
from tempfile import NamedTemporaryFile
+import sqlite3
import caosdb as db
-from caosadvancedtools.cache import Cache, cleanXML
+from caosadvancedtools.cache import IdentifiableCache, cleanXML
from lxml import etree
+import pytest
+
class CacheTest(unittest.TestCase):
def setUp(self):
- self.cache = Cache(db_file=NamedTemporaryFile(delete=False).name)
- self.cache.create_cache()
+ self.cache = IdentifiableCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
def test_hash(self):
ent = db.Record()
- assert isinstance(Cache.hash_entity(ent), str)
- assert (Cache.hash_entity(ent) !=
- Cache.hash_entity(db.Record().add_parent("lol")))
+ assert isinstance(IdentifiableCache.hash_entity(ent), str)
+ assert (IdentifiableCache.hash_entity(ent) !=
+ IdentifiableCache.hash_entity(db.Record().add_parent("lol")))
def test_insert(self):
ent = db.Record()
ent2 = db.Record()
ent2.add_parent(name="Experiment")
- ent_hash = Cache.hash_entity(ent)
- ent2_hash = Cache.hash_entity(ent2)
- self.cache.insert(ent2_hash, 1235)
- assert isinstance(self.cache.check_existing(ent2_hash), int)
+ ent_hash = IdentifiableCache.hash_entity(ent)
+ ent2_hash = IdentifiableCache.hash_entity(ent2)
+ self.cache.insert(ent2_hash, 1235, "ajkfljadsklf")
+ assert self.cache.check_existing(ent2_hash)[0] == 1235
+ assert self.cache.check_existing(ent2_hash)[1] == "ajkfljadsklf"
assert self.cache.check_existing(ent_hash) is None
def test_hirarchy(self):
@@ -64,17 +68,29 @@ class CacheTest(unittest.TestCase):
ent3 = db.Record()
ent3.add_parent(name="Analysis")
test_id = 2353243
- self.cache.insert(Cache.hash_entity(ent2), test_id)
+ self.cache.insert(IdentifiableCache.hash_entity(ent2), test_id, "ajdsklfjadslf")
entities = [ent, ent2, ent3]
hashes = self.cache.update_ids_from_cache(entities)
+ self.assertEqual(ent.id, None)
self.assertEqual(ent2.id, test_id)
+ self.assertEqual(ent3.id, None)
+
+ with pytest.raises(RuntimeError, match=r".*no ID.*"):
+ self.cache.insert_list(hashes, entities)
# test
ent.id = 1001
ent3.id = 1003
+ with pytest.raises(RuntimeError, match=r".*no version ID.*"):
+ self.cache.insert_list(hashes, entities)
+
+ ent.version = db.common.versioning.Version("jkadsjfldf")
+ ent2.version = db.common.versioning.Version("jkadsjfldf")
+ ent3.version = db.common.versioning.Version("jkadsjfldf")
+
self.cache.insert_list(hashes, entities)
- self.assertEqual(self.cache.check_existing(hashes[0]), 1001)
- self.assertEqual(self.cache.check_existing(hashes[2]), 1003)
+ self.assertEqual(self.cache.check_existing(hashes[0])[0], 1001)
+ self.assertEqual(self.cache.check_existing(hashes[2])[0], 1003)
def test_clean(self):
xml = etree.XML(
@@ -91,3 +107,138 @@ class CacheTest(unittest.TestCase):
""")
cleanXML(xml)
assert len(xml.findall('TransactionBenchmark')) == 0
+
+
+def create_sqlite_file(commands):
+ """
+ A temporary file will be used
+ commands: list of sql commands (tuples) to execute after creation
+ Name of the file is returned
+ """
+ db_file = NamedTemporaryFile(delete=False).name
+ conn = sqlite3.connect(db_file)
+ c = conn.cursor()
+ for sql in commands:
+ c.execute(*sql)
+ conn.commit()
+ conn.close()
+ return db_file
+
+
+class CacheTest2(unittest.TestCase):
+ """
+ Test the schema version.
+ """
+
+ def setUp(self):
+ # Correct version:
+ self.cache = IdentifiableCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
+
+ self.db_file_defect = []
+ self.db_file_defect.extend([
+ # Version without version table (old version):
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER)''',)]),
+ # Version with version table with wrong version:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (1,))]),
+ # Version with version table with wrong version:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (3,))]),
+ # Version with version table with missing version:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',)]),
+ # Version with version table with too many versions:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (1,)),
+ ("INSERT INTO version VALUES (?)", (3,))])])
+
+ def test_schema(self):
+ # Test whether new cache is created correctly:
+ assert os.path.exists(self.cache.db_file)
+ # Test whether it can be opened
+ test_cache_2 = IdentifiableCache(db_file=self.cache.db_file)
+ assert test_cache_2.get_cache_version() == 2
+
+ with pytest.raises(RuntimeError, match="Cache version too old.") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[0])
+
+ with pytest.raises(RuntimeError, match="Cache version too old.") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[1])
+
+ with pytest.raises(RuntimeError, match=r".*future version.*") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[2])
+
+ with pytest.raises(RuntimeError, match=r".*table broken.*") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[3])
+
+ with pytest.raises(RuntimeError, match=r".*table broken.*") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[4])
+
+ def tearDown(self):
+ os.remove(self.cache.db_file)
+
+ for db_fn_defect in self.db_file_defect:
+ os.remove(db_fn_defect)
+
+
+class InvalidationTest(unittest.TestCase):
+ """
+ Test invalidation of cache entries.
+ """
+
+ def setUp(self):
+ # Correct version:
+ self.cache = IdentifiableCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
+
+ def tearDown(self):
+ os.remove(self.cache.db_file)
+
+ def test_invalid(self):
+ ent = db.Record()
+ ent2 = db.Record()
+ ent2.add_parent(name="Experiment")
+ ent3 = db.Record()
+ ent3.add_parent(name="Analysis")
+ ent.id = 117
+ ent2.id = 328
+ ent3.id = 224
+
+ ent.version = db.common.versioning.Version("a")
+ ent2.version = db.common.versioning.Version("b")
+ ent3.version = db.common.versioning.Version("a")
+
+ el = [ent, ent2, ent3]
+
+ for e in el:
+ self.cache.insert(IdentifiableCache.hash_entity(e), e.id, e.version.id)
+
+ for e in el:
+ res = self.cache.check_existing(IdentifiableCache.hash_entity(e))
+ assert e.id == res[0]
+ assert e.version.id == res[1]
+
+ ent2.version.id = "c"
+ ent3.version.id = "b"
+
+ for e in el[1:]:
+ res = self.cache.check_existing(IdentifiableCache.hash_entity(e))
+ assert res is None
+
+ invalidated_entries = self.cache.validate_cache(el)
+ assert 328 in invalidated_entries
+ assert 224 in invalidated_entries
+ assert 117 not in invalidated_entries
+
+ res = self.cache.run_sql_commands([
+ ("SELECT * FROM identifiables", ())], fetchall=True)
+ assert len(res) == 1
diff --git a/unittests/test_caosdbignore.py b/unittests/test_caosdbignore.py
new file mode 100644
index 0000000000000000000000000000000000000000..9394bf0c8b177bc6df132ee0b43dcb2753d70f28
--- /dev/null
+++ b/unittests/test_caosdbignore.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+
+import os
+import re
+from tempfile import NamedTemporaryFile
+import unittest
+
+from caosadvancedtools.loadFiles import compile_file_list, create_re_for_file_list, combine_ignore_files
+
+BASEDIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class Caosdbignore(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def test_compile(self):
+ files = compile_file_list(os.path.join(BASEDIR, "caosdbignore-example"),
+ os.path.join(BASEDIR, "data"))
+ assert len(files) == 3
+ assert os.path.join(BASEDIR, "data", "datatypes.xlsx") in files
+ assert os.path.join(BASEDIR, "data", "README.xlsx") in files
+ assert os.path.join(BASEDIR, "data", "Publications/Posters/2019-02-03_something/README.md") in files
+
+ def test_regex(self):
+ files = [r"/dies/ist/simple", r"/dies/eh(er)/nich?t"]
+ regex = create_re_for_file_list(files, "/dies", "/dies")
+ assert re.match(regex, files[0]) is not None
+ assert re.match(regex, files[1]) is not None
+ assert re.match(regex, "/dies/ist") is not None
+ assert re.match(regex, "/die") is None
+ assert re.match(regex, files[0]+files[1]) is None
+ assert re.match(regex, "d") is None
+
+ def test_combine(self):
+ fi1 = NamedTemporaryFile(delete=False, mode="w")
+ fi1.write("ha")
+ fi1.close()
+ fi2 = NamedTemporaryFile(delete=False, mode="w")
+ fi2.write("ha")
+ fi2.close()
+ fi_new = combine_ignore_files(fi1.name, fi2.name)
+ with open(fi_new, "r") as fi:
+ assert "haha" == fi.read()
diff --git a/unittests/test_cfood.py b/unittests/test_cfood.py
index 7055bc7c51962c0cbc487f29bcdacb391218a7d3..e2f15ffdc7929fbd67aee37bccdb0f44cacef104 100644
--- a/unittests/test_cfood.py
+++ b/unittests/test_cfood.py
@@ -32,7 +32,7 @@ from caosadvancedtools.cfood import (AbstractCFood, AbstractFileCFood, CMeal,
get_entity_for_path)
from caosadvancedtools.crawler import FileCrawler
from caosadvancedtools.example_cfood import ExampleCFood
-from caosdb.common.models import _parse_single_xml_element
+from linkahead.common.models import _parse_single_xml_element
from lxml import etree
from datetime import datetime, timezone
@@ -336,6 +336,7 @@ class MealTest(unittest.TestCase):
class FileCacheTest(unittest.TestCase):
def test(self):
+ # if there is no connection to a server an exception should be raised.
self.assertRaises(Exception, get_entity_for_path, "/lol")
FileCrawler(cfood_types=[], files=[db.File(path="/lol")])
get_entity_for_path("/lol")
diff --git a/unittests/test_data_model.py b/unittests/test_data_model.py
index 159adfca1d589bb092b6f59110828b5868401e25..cafeb6ca6a43d7e0409aee3352b43f26d5208732 100644
--- a/unittests/test_data_model.py
+++ b/unittests/test_data_model.py
@@ -2,6 +2,7 @@ import unittest
import caosdb as db
from caosadvancedtools.models.data_model import DataModel
+from caosadvancedtools.models.parser import parse_model_from_string
class DataModelTest(unittest.TestCase):
@@ -33,3 +34,49 @@ class DataModelTest(unittest.TestCase):
DataModel.sync_ids_by_name(l1, l2)
assert l1["TestRecord"].id == rt.id
assert l1["TestRecord2"].id < 0
+
+ def test_get_deep(self):
+ model_recursive_str = """
+RT1:
+ description: some description
+ obligatory_properties:
+ RT1:
+ """
+ model_recursive = parse_model_from_string(model_recursive_str)
+ prop1 = model_recursive["RT1"].get_property("RT1")
+ assert prop1.datatype is None
+ # TODO The next line actually changes model_recursive in place, is this OK?
+ RT1 = model_recursive.get_deep("RT1")
+ assert model_recursive["RT1"] == RT1
+
+ model_unresolved_str = """
+RT1:
+ description: some description
+ obligatory_properties:
+ unresolved:
+ """
+ model_unresolved = parse_model_from_string(model_unresolved_str)
+ rt1_unresolved = model_unresolved["RT1"]
+ prop_unresolved = model_unresolved.get_deep("unresolved")
+ assert prop_unresolved.datatype is None
+ rt1_deep = model_unresolved.get_deep("RT1")
+ assert rt1_deep == rt1_unresolved
+ assert rt1_deep is rt1_unresolved
+
+ model_double_property = """
+p1:
+ description: Hello world
+ datatype: TEXT
+RT1:
+ recommended_properties:
+ p1:
+RT2:
+ recommended_properties:
+ RT1:
+ p1:
+"""
+ model_unresolved = parse_model_from_string(model_double_property)
+ rt2_deep = model_unresolved.get_deep("RT2")
+ p1 = rt2_deep.get_property("p1")
+ assert p1.datatype == "TEXT"
+ assert p1.description == "Hello world"
diff --git a/unittests/test_h5.py b/unittests/test_h5.py
index 360d4b28938492d0f2af6d696e39dffb1cc3fead..961dd4246ef4b02208226ada5d3e1389133ddbcc 100644
--- a/unittests/test_h5.py
+++ b/unittests/test_h5.py
@@ -1,8 +1,8 @@
import unittest
from tempfile import NamedTemporaryFile
-import caosdb as db
-import caosdb.apiutils
+import linkahead as db
+import linkahead.apiutils
import h5py
import numpy as np
from caosadvancedtools.cfoods import h5
@@ -77,8 +77,8 @@ class H5CFoodTest(unittest.TestCase):
# TODO this does probably break the code: The function will not be
# restored correctly.
# Change it to use the BaseMockUpTest
- real_retrieve = caosdb.apiutils.retrieve_entity_with_id
- caosdb.apiutils.retrieve_entity_with_id = dummy_get
+ real_retrieve = linkahead.apiutils.retrieve_entity_with_id
+ linkahead.apiutils.retrieve_entity_with_id = dummy_get
# should run without problem
h5.collect_existing_structure(db.Record(), db.Record(id=234), h5.EntityMapping())
@@ -151,7 +151,7 @@ class H5CFoodTest(unittest.TestCase):
self.assertEqual(em.to_existing[r_child2._cuid], ENTS[101])
self.assertEqual(em.to_target[101], r_child2)
- caosdb.apiutils.retrieve_entity_with_id = real_retrieve
+ linkahead.apiutils.retrieve_entity_with_id = real_retrieve
def test_h5_attr_to_property(self):
@@ -160,7 +160,8 @@ class H5CFoodTest(unittest.TestCase):
test_float = np.float_(1.0)
test_str = "Test"
test_complex: complex = 2+3j
- self.assertRaises(NotImplementedError, h5_attr_to_property, test_int) # only numpy-integers processed?
+ self.assertRaises(NotImplementedError, h5_attr_to_property,
+ test_int) # only numpy-integers processed?
self.assertTupleEqual((1, db.INTEGER), h5_attr_to_property(test_integer))
self.assertTupleEqual((1.0, db.DOUBLE), h5_attr_to_property(test_float))
self.assertTupleEqual(("Test", db.TEXT), h5_attr_to_property(test_str))
@@ -187,4 +188,5 @@ class H5CFoodTest(unittest.TestCase):
# Test scalar values given as np.array
self.assertTupleEqual((1, db.INTEGER), h5_attr_to_property(np.array(1)))
self.assertTupleEqual((1.123, db.DOUBLE), h5_attr_to_property(np.array(1.123)))
- self.assertTupleEqual(('Hello World', db.TEXT), h5_attr_to_property(np.array("Hello World")))
+ self.assertTupleEqual(('Hello World', db.TEXT),
+ h5_attr_to_property(np.array("Hello World")))
diff --git a/unittests/test_json_schema_exporter.py b/unittests/test_json_schema_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0503385a25eb89e66dd3518d71a32b91d07bf88
--- /dev/null
+++ b/unittests/test_json_schema_exporter.py
@@ -0,0 +1,1049 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Tests the Json schema exporter."""
+
+import json
+
+import linkahead as db
+import caosadvancedtools.json_schema_exporter as jsex
+
+from collections import OrderedDict
+
+from jsonschema import FormatChecker, validate, ValidationError
+from pytest import raises
+from unittest.mock import Mock, patch
+
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema as rtjs
+from caosadvancedtools.models.parser import parse_model_from_string
+
+GLOBAL_MODEL = parse_model_from_string("""
+RT1:
+ description: some description
+ obligatory_properties:
+ some_date:
+ datatype: DATETIME
+ description: Just some date
+RT21:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+RT31:
+ obligatory_properties:
+ RT1:
+
+""")
+
+RT1 = GLOBAL_MODEL.get_deep("RT1")
+RT21 = GLOBAL_MODEL.get_deep("RT21")
+RT31 = GLOBAL_MODEL.get_deep("RT31")
+
+
+def _mock_execute_query(query_string, unique=False, **kwargs):
+ """Mock the response to queries for references."""
+ all_records = db.Container()
+ all_files = db.Container()
+ other_type_rt = db.RecordType(name="OtherType")
+ other_type_rt.add_property(name="IntegerProp", datatype=db.INTEGER, importance=db.OBLIGATORY)
+ other_type_records = db.Container().extend([
+ db.Record(id=100, name="otherA").add_parent(other_type_rt),
+ db.Record(id=101, name="otherB").add_parent(other_type_rt),
+ db.Record(id=102).add_parent(other_type_rt)
+ ])
+ all_records.extend(other_type_records)
+
+ referencing_type_rt = db.RecordType(name="ReferencingType")
+ referencing_type_rt.add_property(name=other_type_rt.name, datatype=db.LIST(other_type_rt.name))
+ referencing_type_records = db.Container().extend([
+ db.Record(id=103).add_parent(referencing_type_rt),
+ db.Record(id=104, name="referencing").add_parent(referencing_type_rt)
+ ])
+ all_records.extend(referencing_type_records)
+
+ all_files.append(db.File(id=105, name="GenericFile.txt"))
+
+ if query_string == "SELECT name, id FROM RECORD 'OtherType'":
+ return other_type_records
+ elif query_string == "FIND RECORDTYPE WITH name='OtherType'" and unique is True:
+ return other_type_rt
+ elif query_string == "SELECT name, id FROM RECORD 'ReferencingType'":
+ return referencing_type_records
+ elif query_string == "FIND RECORDTYPE WITH name='ReferencingType'" and unique is True:
+ return referencing_type_rt
+ elif query_string == "SELECT name, id FROM RECORD 'RT1'":
+ return referencing_type_records # wrong types, but who cares for the test?
+ elif query_string == "FIND RECORDTYPE WITH name='RT1'" and unique is True:
+ return RT1
+ elif query_string == "FIND RECORDTYPE WITH name='RT21'" and unique is True:
+ return RT21
+ elif query_string == "FIND RECORDTYPE WITH name='RT31'" and unique is True:
+ return RT31
+ elif query_string == "SELECT name, id FROM RECORD":
+ return all_records
+ elif query_string == "SELECT name, id FROM FILE":
+ return all_files
+ else:
+ print(f"Query string: {query_string}")
+ if unique is True:
+ return db.Entity()
+ return db.Container()
+
+
+def test_empty_rt():
+
+ rt = db.RecordType(name="Test", description="descr")
+
+ schema = rtjs(rt)
+
+ assert schema["title"] == rt.name
+ assert schema["description"] == rt.description
+ assert len(schema["properties"]) == 0
+ assert len(schema["required"]) == 0
+ assert schema["additionalProperties"] is True
+
+ schema = rtjs(rt, additional_properties=False)
+
+ assert schema["title"] == rt.name
+ assert schema["description"] == rt.description
+ assert len(schema["properties"]) == 0
+ assert len(schema["required"]) == 0
+ assert schema["additionalProperties"] is False
+
+ schema = rtjs(rt, name_property_for_new_records=True,
+ description_property_for_new_records=True)
+
+ assert len(schema["properties"]) == 2
+ assert "name" in schema["properties"]
+ assert "description" in schema["properties"]
+ assert schema["properties"]["name"]["type"] == "string"
+ assert schema["properties"]["description"]["type"] == "string"
+
+
+def test_rt_with_scalar_props():
+
+ rt = db.RecordType(name="Test")
+ rt.add_property(name="SimpleText", datatype=db.TEXT, description="This is a simple text")
+ rt.add_property(name="ObligatoryDatetime", datatype=db.DATETIME, importance=db.OBLIGATORY)
+ rt.add_property(name="JustDateNoTime", datatype=db.DATETIME, description="Only dates, no times")
+ rt.add_property(name="ObligatoryInteger", datatype=db.INTEGER, importance=db.OBLIGATORY)
+ rt.add_property(name="Double", datatype=db.DOUBLE)
+ # Suggested shouldn't influence the result in any way.
+ rt.add_property(name="Boolean", datatype=db.BOOLEAN, importance=db.SUGGESTED)
+
+ schema = rtjs(rt, additional_options_for_text_props={"JustDateNoTime": {"format": "date"}})
+
+ assert "properties" in schema
+ props = schema["properties"]
+ assert len(props) == 6
+ assert "required" in schema
+ assert len(schema["required"]) == 2
+ assert "ObligatoryDatetime" in schema["required"]
+ assert "ObligatoryInteger" in schema["required"]
+
+ assert "SimpleText" in props
+ assert props["SimpleText"]["type"] == "string"
+ assert "format" not in props["SimpleText"]
+ assert "description" in props["SimpleText"]
+ assert props["SimpleText"]["description"] == "This is a simple text"
+
+ assert "ObligatoryDatetime" in props
+ assert "type" not in props["ObligatoryDatetime"]
+ assert "anyOf" in props["ObligatoryDatetime"]
+ assert len(props["ObligatoryDatetime"]["anyOf"]) == 2
+ date_found = 0
+ datetime_found = 0
+ for option in props["ObligatoryDatetime"]["anyOf"]:
+ assert option["type"] == "string"
+ fmt = option["format"]
+ if fmt == "date":
+ date_found += 1
+ if fmt == "date-time":
+ datetime_found += 1
+ assert date_found == 1
+ assert datetime_found == 1
+
+ assert "JustDateNoTime" in props
+ assert props["JustDateNoTime"]["type"] == "string"
+ assert "anyOf" not in props["JustDateNoTime"]
+ assert "pattern" not in props["JustDateNoTime"]
+ assert props["JustDateNoTime"]["format"] == "date"
+ assert props["JustDateNoTime"]["description"] == "Only dates, no times"
+
+ assert "ObligatoryInteger" in props
+ assert props["ObligatoryInteger"]["type"] == "integer"
+
+ assert "Double" in props
+ assert props["Double"]["type"] == "number"
+
+ assert "Boolean" in props
+ assert props["Boolean"]["type"] == "boolean"
+
+ # test validation (we turst the jsonschema.validat function, so only test
+ # some more or less tricky cases with format or required).
+ example = {
+ "SimpleText": "something",
+ "ObligatoryInteger": 23,
+ "ObligatoryDatetime": "1900-01-01T12:34:56.0Z",
+ "JustDateNoTime": "2023-10-13"
+ }
+
+ # We need to explicitly enable the FormatChecker, otherwise format will be
+ # ignored
+ # (https://python-jsonschema.readthedocs.io/en/latest/validate/#validating-formats)
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "SimpleText": "something",
+ "ObligatoryInteger": 23,
+ "ObligatoryDatetime": "1900-01-01",
+ "JustDateNoTime": "2023-10-13"
+ }
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "SimpleText": "something",
+ "ObligatoryDatetime": "1900-01-01T12:34:56.0Z",
+ "JustDateNoTime": "2023-10-13"
+ }
+
+ with raises(ValidationError):
+ # required missing
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "SimpleText": "something",
+ "ObligatoryInteger": 23,
+ "ObligatoryDatetime": "1900-01-01T12:34:56.0Z",
+ "JustDateNoTime": "2023-10-13T23:59:59.123Z"
+ }
+
+ with raises(ValidationError):
+ # date expected in JustDateNoTime, but datetime given
+ validate(example, schema, format_checker=FormatChecker())
+
+
+def test_units():
+
+ rt = db.RecordType()
+ rt.add_property(name="ScalarWithUnit", datatype=db.DOUBLE, unit="m")
+ rt.add_property(name="ListWithUnit", description="This is a list.",
+ datatype=db.LIST(db.DOUBLE), unit="m")
+
+ schema = rtjs(rt, units_in_description=True)
+
+ props = schema["properties"]
+ assert "ScalarWithUnit" in props
+ assert props["ScalarWithUnit"]["type"] == "number"
+ assert "description" in props["ScalarWithUnit"]
+ assert props["ScalarWithUnit"]["description"] == "Unit is m."
+ assert "unit" not in props["ScalarWithUnit"]
+
+ assert "ListWithUnit" in props
+ assert props["ListWithUnit"]["type"] == "array"
+ assert "items" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["items"]["type"] == "number"
+ assert "description" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["description"] == "This is a list. Unit is m."
+ assert "unit" not in props["ListWithUnit"]
+
+ schema = rtjs(rt, units_in_description=False)
+
+ props = schema["properties"]
+ assert "ScalarWithUnit" in props
+ assert props["ScalarWithUnit"]["type"] == "number"
+ assert "description" not in props["ScalarWithUnit"]
+ assert "unit" in props["ScalarWithUnit"]
+ assert props["ScalarWithUnit"]["unit"] == "m"
+
+ assert "ListWithUnit" in props
+ assert props["ListWithUnit"]["type"] == "array"
+ assert "items" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["items"]["type"] == "number"
+ assert "description" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["description"] == "This is a list."
+ assert "unit" in props["ListWithUnit"]
+ assert props["ListWithUnit"]["unit"] == "m"
+
+
+def test_rt_with_list_props():
+
+ rt = db.RecordType()
+ rt.add_property(name="ListOfIntegers", datatype=db.LIST(
+ db.INTEGER), description="List of integers")
+ rt.add_property(name="ListOfPatterns", datatype=db.LIST(db.TEXT))
+
+ schema = rtjs(rt, additional_options_for_text_props={"ListOfPatterns": {"pattern": "[A-Z]+"}})
+
+ props = schema["properties"]
+
+ assert "ListOfIntegers" in props
+ assert props["ListOfIntegers"]["type"] == "array"
+ assert "items" in props["ListOfIntegers"]
+ assert props["ListOfIntegers"]["items"]["type"] == "integer"
+ assert "description" not in props["ListOfIntegers"]["items"]
+ assert props["ListOfIntegers"]["description"] == "List of integers"
+
+ assert "ListOfPatterns" in props
+ assert props["ListOfPatterns"]["type"] == "array"
+ assert "items" in props["ListOfPatterns"]
+ assert props["ListOfPatterns"]["items"]["type"] == "string"
+ assert props["ListOfPatterns"]["items"]["pattern"] == "[A-Z]+"
+
+ # Validation
+ example = {
+ "ListOfIntegers": [1, 2, 3],
+ "ListOfPatterns": ["A", "BB", "CCC"]
+ }
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "ListOfIntegers": 1,
+ "ListOfPatterns": ["A", "BB", "CCC"]
+ }
+ with raises(ValidationError):
+ # No list
+ validate(example, schema, format_checker=FormatChecker())
+
+ example = {
+ "ListOfIntegers": [1, 2, 3],
+ "ListOfPatterns": ["A", "bb", "CCC"]
+ }
+ with raises(ValidationError):
+ # Pattern doesn't match
+ validate(example, schema, format_checker=FormatChecker())
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_rt_with_references():
+
+ rt = db.RecordType()
+ rt.add_property(name="RefProp", datatype=db.REFERENCE)
+
+ schema = rtjs(rt)
+ props = schema["properties"]
+ assert "RefProp" in props
+ assert "enum" in props["RefProp"]
+ assert isinstance(props["RefProp"]["enum"], list)
+ assert len(props["RefProp"]["enum"]) == len(
+ db.execute_query("SELECT name, id FROM RECORD")) + len(
+ db.execute_query("SELECT name, id FROM FILE"))
+ assert "oneOf" not in props["RefProp"]
+
+ example = {
+ "RefProp": "otherB"
+ }
+ validate(example, schema)
+ example = {
+ "RefProp": "I don't exist"
+ }
+ with raises(ValidationError):
+ # Wrong enum value
+ validate(example, schema)
+ example = {
+ "RefProp": {
+ "IntegerProp": 12
+ }
+ }
+ with raises(ValidationError):
+ # Can't have objects in generic references
+ validate(example, schema)
+
+ rt = db.RecordType()
+ rt.add_property(name="RefProp", datatype="OtherType")
+ rt.add_property(name="OtherTextProp", datatype=db.TEXT)
+
+ schema = rtjs(rt)
+ props = schema["properties"]
+ assert "RefProp" in props
+ assert "oneOf" in props["RefProp"]
+ assert len(props["RefProp"]["oneOf"]) == 2
+ enum_index = 0
+ if "enum" not in props["RefProp"]["oneOf"][enum_index]:
+ # We can't really require the order here, so we just know that one of
+ # the two elements must be the enum, the other the object.
+ enum_index = 1 - enum_index
+ assert "enum" in props["RefProp"]["oneOf"][enum_index]
+ assert isinstance(props["RefProp"]["oneOf"][enum_index]["enum"], list)
+ assert len(props["RefProp"]["oneOf"][enum_index]["enum"]) == 3
+ assert "otherA" in props["RefProp"]["oneOf"][enum_index]["enum"]
+ assert "otherB" in props["RefProp"]["oneOf"][enum_index]["enum"]
+ assert "102" in props["RefProp"]["oneOf"][enum_index]["enum"]
+ # the other element of oneOf is the OtherType object
+ assert props["RefProp"]["oneOf"][1 - enum_index]["type"] == "object"
+ other_props = props["RefProp"]["oneOf"][1 - enum_index]["properties"]
+ assert "IntegerProp" in other_props
+ assert other_props["IntegerProp"]["type"] == "integer"
+ assert "required" in props["RefProp"]["oneOf"][1 - enum_index]
+ assert len(props["RefProp"]["oneOf"][1 - enum_index]["required"]) == 1
+ assert "IntegerProp" in props["RefProp"]["oneOf"][1 - enum_index]["required"]
+ # The other prop also works as before
+ assert "OtherTextProp" in props
+ assert props["OtherTextProp"]["type"] == "string"
+
+ example = {
+ "RefProp": {
+ "IntegerProp": 12
+ }
+ }
+ validate(example, schema)
+
+ example = {
+ "RefProp": "otherB",
+ "OtherTextProp": "something"
+ }
+ validate(example, schema)
+
+ rt = db.RecordType(name="TestType", description="Some description")
+ rt.add_property(name="RefProp", datatype=db.LIST(db.REFERENCE),
+ description="I'm a list of references.")
+
+ schema = rtjs(rt)
+ assert schema["title"] == rt.name
+ assert schema["description"] == rt.description
+ assert "RefProp" in schema["properties"]
+ ref_prop = schema["properties"]["RefProp"]
+ assert ref_prop["type"] == "array"
+ assert "description" in ref_prop
+ assert ref_prop["description"] == "I'm a list of references."
+ assert "items" in ref_prop
+ items = ref_prop["items"]
+ assert "enum" in items
+ assert isinstance(items["enum"], list)
+ assert len(items["enum"]) == len(
+ db.execute_query("SELECT name, id FROM RECORD")) + len(
+ db.execute_query("SELECT name, id FROM FILE"))
+ assert "oneOf" not in items
+ assert "description" not in items
+
+ example = {
+ "RefProp": "otherB"
+ }
+ with raises(ValidationError):
+ # Should be list but isn't
+ validate(example, schema)
+ example = {
+ "RefProp": ["otherB"]
+ }
+ validate(example, schema)
+ example = {
+ "RefProp": ["otherB", "102", "referencing"]
+ }
+ validate(example, schema)
+
+ rt = db.RecordType()
+ rt.add_property(name="RefProp", datatype=db.LIST("OtherType"))
+
+ schema = rtjs(rt, additional_properties=False, name_property_for_new_records=True,
+ description_property_for_new_records=True)
+ assert schema["additionalProperties"] is False
+ assert "name" in schema["properties"]
+ assert schema["properties"]["name"]["type"] == "string"
+ assert "description" in schema["properties"]
+ assert schema["properties"]["description"]["type"] == "string"
+ assert "RefProp" in schema["properties"]
+ assert schema["properties"]["RefProp"]["type"] == "array"
+ assert "additionalProperties" not in schema["properties"]["RefProp"]
+ assert "items" in schema["properties"]["RefProp"]
+ items = schema["properties"]["RefProp"]["items"]
+ assert "oneOf" in items
+ assert len(items["oneOf"]) == 2
+ # same as above, we can't rely on the order
+ enum_index = 0
+ if "enum" not in items["oneOf"][enum_index]:
+ enum_index = 1 - enum_index
+ assert "enum" in items["oneOf"][enum_index]
+ assert isinstance(items["oneOf"][enum_index]["enum"], list)
+ assert len(items["oneOf"][enum_index]["enum"]) == 3
+ assert "otherA" in items["oneOf"][enum_index]["enum"]
+ assert "otherB" in items["oneOf"][enum_index]["enum"]
+ assert "102" in items["oneOf"][enum_index]["enum"]
+ other_type = items["oneOf"][1 - enum_index]
+ assert other_type["type"] == "object"
+ assert other_type["additionalProperties"] is False
+ assert "IntegerProp" in other_type["properties"]
+ assert len(other_type["required"]) == 1
+ assert "IntegerProp" in other_type["required"]
+
+ example = {
+ "RefProp": ["otherB", "102", "referencing"]
+ }
+ with raises(ValidationError):
+ # Wrong value in enum
+ validate(example, schema)
+ example = {
+ "RefProp": [{"IntegerProp": 12}]
+ }
+ validate(example, schema)
+ example = {
+ "RefProp": [{"IntegerProp": 12, "additionalProperty": "something"}]
+ }
+ with raises(ValidationError):
+ # we have additional_properties=False which propagates to subschemas
+ validate(example, schema)
+ example = {
+ "RefProp": [{"IntegerProp": 12}, "otherB"]
+ }
+ validate(example, schema)
+
+ rt = db.RecordType(name="ReferenceofReferencesType")
+ rt.add_property(name="RefRefProp", datatype="ReferencingType")
+
+ schema = rtjs(rt)
+
+ assert "RefRefProp" in schema["properties"]
+ ref_ref = schema["properties"]["RefRefProp"]
+ assert "oneOf" in ref_ref
+ assert len(ref_ref["oneOf"]) == 2
+ enum_index = 0
+ if "enum" not in ref_ref["oneOf"][enum_index]:
+ enum_index = 1 - enum_index
+ assert len(ref_ref["oneOf"][enum_index]["enum"]) == 2
+ assert "103" in ref_ref["oneOf"][enum_index]["enum"]
+ assert "referencing" in ref_ref["oneOf"][enum_index]["enum"]
+ assert ref_ref["oneOf"][1 - enum_index]["type"] == "object"
+ assert "OtherType" in ref_ref["oneOf"][1 - enum_index]["properties"]
+ assert ref_ref["oneOf"][1 - enum_index]["properties"]["OtherType"]["type"] == "array"
+ items = ref_ref["oneOf"][1 - enum_index]["properties"]["OtherType"]["items"]
+ assert "oneOf" in items
+ assert len(items["oneOf"]) == 2
+ # same as above, we can't rely on the order
+ enum_index = 0
+ if "enum" not in items["oneOf"][enum_index]:
+ enum_index = 1 - enum_index
+ assert "enum" in items["oneOf"][enum_index]
+ assert isinstance(items["oneOf"][enum_index]["enum"], list)
+ assert len(items["oneOf"][enum_index]["enum"]) == 3
+ assert "otherA" in items["oneOf"][enum_index]["enum"]
+ assert "otherB" in items["oneOf"][enum_index]["enum"]
+ assert "102" in items["oneOf"][enum_index]["enum"]
+ other_type = items["oneOf"][1 - enum_index]
+ assert other_type["type"] == "object"
+ assert "IntegerProp" in other_type["properties"]
+ assert len(other_type["required"]) == 1
+ assert "IntegerProp" in other_type["required"]
+
+ example = {
+ "RefRefProp": {
+ "OtherType": [
+ "otherA",
+ {"IntegerProp": 12}
+ ]
+ }
+ }
+ validate(example, schema)
+
+ # Single file and multiple files
+ rt = db.RecordType()
+ rt.add_property(name="FileProp", datatype=db.FILE)
+
+ schema = rtjs(rt)
+ assert schema["properties"]["FileProp"]["type"] == "string"
+ assert schema["properties"]["FileProp"]["format"] == "data-url"
+
+ # wrap in array (cf. https://github.com/rjsf-team/react-jsonschema-form/issues/3957)
+ schema = rtjs(rt, wrap_files_in_objects=True)
+ assert schema["properties"]["FileProp"]["type"] == "array"
+ assert schema["properties"]["FileProp"]["maxItems"] == 1
+ assert "items" in schema["properties"]["FileProp"]
+ items = schema["properties"]["FileProp"]["items"]
+ assert items["type"] == "object"
+ assert len(items["required"]) == 1
+ assert "file" in items["required"]
+ assert items["additionalProperties"] is False
+ assert len(items["properties"]) == 1
+ assert "file" in items["properties"]
+ assert items["properties"]["file"]["type"] == "string"
+ assert items["properties"]["file"]["format"] == "data-url"
+
+ rt = db.RecordType()
+ rt.add_property(name="FileProp", datatype=db.LIST(db.FILE))
+
+ schema = rtjs(rt)
+ assert schema["properties"]["FileProp"]["type"] == "array"
+ assert schema["properties"]["FileProp"]["items"]["type"] == "string"
+ assert schema["properties"]["FileProp"]["items"]["format"] == "data-url"
+
+ # wrap in array (cf. https://github.com/rjsf-team/react-jsonschema-form/issues/3957)
+ print(schema)
+ schema = rtjs(rt, wrap_files_in_objects=True)
+ assert schema["properties"]["FileProp"]["type"] == "array"
+ assert "maxItems" not in schema["properties"]["FileProp"]
+ assert "items" in schema["properties"]["FileProp"]
+ items = schema["properties"]["FileProp"]["items"]
+ assert items["type"] == "object"
+ assert len(items["required"]) == 1
+ assert "file" in items["required"]
+ assert items["additionalProperties"] is False
+ assert len(items["properties"]) == 1
+ assert "file" in items["properties"]
+ assert items["properties"]["file"]["type"] == "string"
+ assert items["properties"]["file"]["format"] == "data-url"
+
+
+def test_broken():
+
+ rt = db.RecordType()
+ rt.add_property(name="something", datatype=None)
+
+ with raises(ValueError) as ve:
+
+ rtjs(rt)
+ assert str(ve).startswith("Unknown or no property datatype.")
+
+ rt = db.RecordType()
+ rt.add_property(name="MultiProp", datatype=db.INTEGER)
+ rt.add_property(name="MultiProp", datatype=db.INTEGER)
+
+ with raises(NotImplementedError) as nie:
+
+ rtjs(rt)
+ assert "MultiProp" in str(nie)
+ assert str(nie).startswith("Creating a schema for multi-properties is not specified.")
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_reference_options():
+ """Testing miscellaneous options.
+ """
+
+ model_str = """
+RT1:
+ description: some description
+ obligatory_properties:
+ some_date:
+ datatype: DATETIME
+ description: Just some date
+RT2:
+ obligatory_properties:
+ RT1:
+
+RT3:
+ obligatory_properties:
+ RT1_prop:
+ datatype: RT1
+ description: property description
+ """
+ model = parse_model_from_string(model_str)
+ # First test: without reference
+ rt1_dict = rtjs(model.get_deep("RT1"))
+ assert json.dumps(rt1_dict, indent=2) == """{
+ "type": "object",
+ "required": [
+ "some_date"
+ ],
+ "additionalProperties": true,
+ "description": "some description",
+ "title": "RT1",
+ "properties": {
+ "some_date": {
+ "description": "Just some date",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+ # Second test: with reference
+ rt2_deep = model.get_deep("RT2")
+ rt2_dict = rtjs(rt2_deep)
+ assert json.dumps(rt2_dict, indent=2) == """{
+ "type": "object",
+ "required": [
+ "RT1"
+ ],
+ "additionalProperties": true,
+ "title": "RT2",
+ "properties": {
+ "RT1": {
+ "description": "some description",
+ "oneOf": [
+ {
+ "title": "Existing entries",
+ "enum": [
+ "103",
+ "referencing"
+ ]
+ },
+ {
+ "type": "object",
+ "required": [
+ "some_date"
+ ],
+ "additionalProperties": true,
+ "description": "some description",
+ "title": "Create new",
+ "properties": {
+ "some_date": {
+ "description": "Just some date",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date"
+ },
+ {
+ "type": "string",
+ "format": "date-time"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+
+ # Third test: Reference prop shall be only existing references, no option to create new ones.
+ rt2_dict = rtjs(model.get_deep("RT2"), do_not_create=["RT1"])
+ assert json.dumps(rt2_dict, indent=2) == """{
+ "type": "object",
+ "required": [
+ "RT1"
+ ],
+ "additionalProperties": true,
+ "title": "RT2",
+ "properties": {
+ "RT1": {
+ "description": "some description",
+ "enum": [
+ "103",
+ "referencing"
+ ]
+ }
+ },
+ "$schema": "https://json-schema.org/draft/2020-12/schema"
+}"""
+ # No effect of do_not_create (real property name should be used)
+ rt3_dict = rtjs(model.get_deep("RT3"), do_not_create=["RT1"])
+ rt1_prop = rt3_dict["properties"]["RT1_prop"]
+ assert rt1_prop["description"] == "property description"
+ assert "oneOf" in rt1_prop.keys()
+ assert "enum" not in rt1_prop.keys()
+
+ # Now we use the real property name
+ rt3_dict = rtjs(model.get_deep("RT3"), do_not_create=["RT1_prop"])
+ rt1_prop = rt3_dict["properties"]["RT1_prop"]
+ assert rt1_prop["description"] == "property description"
+ assert "oneOf" not in rt1_prop.keys()
+ assert "enum" in rt1_prop.keys()
+ assert rt1_prop["enum"][0] == "103"
+
+
+def test_schema_modification():
+ """Testing functions which modify json schema dicts:
+
+- make_array()
+- merge_schemas().
+ """
+
+ model_str = """
+some_date:
+ datatype: DATETIME
+RT1:
+ obligatory_properties:
+ some_date:
+
+some_text:
+ datatype: TEXT
+RT2:
+ obligatory_properties:
+ some_text:
+ """
+ model = parse_model_from_string(model_str)
+ schema_RT1 = rtjs(model.get_deep("RT1"), additional_properties=False)
+ schema_RT2 = rtjs(model.get_deep("RT2"), additional_properties=False)
+
+ # Merge the schemata
+ merged_list = jsex.merge_schemas([schema_RT1, schema_RT2])
+ with raises(ValidationError):
+ validate({}, merged_list)
+ assert merged_list["type"] == "object"
+ assert merged_list["properties"]["RT1"]["title"] == "RT1"
+ assert merged_list["properties"]["RT2"]["properties"]["some_text"]["type"] == "string"
+
+ merged_dict = jsex.merge_schemas({"schema1": schema_RT1, "schema2": schema_RT2})
+ with raises(ValidationError):
+ validate({}, merged_dict)
+ assert merged_dict["type"] == "object"
+ assert merged_dict["properties"]["schema1"]["title"] == "RT1"
+ assert merged_dict["properties"]["schema2"]["properties"]["some_text"]["type"] == "string"
+
+ # Make an array
+ array = jsex.make_array(schema_RT1)
+ with raises(ValidationError):
+ validate({}, array)
+ assert array["type"] == "array"
+ assert array["items"] == schema_RT1
+
+
+def test_inheritance():
+ """Test data models with inherited properties."""
+ model_str = """
+some_date:
+ datatype: DATETIME
+RT1:
+ obligatory_properties:
+ some_date:
+RT2:
+ inherit_from_suggested:
+ - RT1
+ """
+ model = parse_model_from_string(model_str)
+ rt2_deep = model.get_deep("RT2")
+ assert "some_date" in [prop.name for prop in rt2_deep.properties]
+
+ model_str = """
+RT1:
+ obligatory_properties:
+ RT2:
+RT2:
+ inherit_from_suggested:
+ - RT1
+RT3:
+ inherit_from_suggested:
+ - RT4
+RT4:
+ inherit_from_suggested:
+ - RT3
+RT5:
+ inherit_from_suggested:
+ - RT5
+ """
+ model = parse_model_from_string(model_str)
+ # This must not lead to an infinite recursion
+ rt1_deep = model.get_deep("RT1")
+ rt2_deep = model.get_deep("RT2")
+ assert rt2_deep.get_property("RT2").name == rt1_deep.get_property("RT2").name
+ rt3_deep = model.get_deep("RT3")
+ assert rt3_deep.get_parents()[0].name == "RT4"
+ rt4_deep = model.get_deep("RT4")
+ assert rt4_deep.get_parents()[0].name == "RT3"
+ rt5_deep = model.get_deep("RT5")
+ assert rt5_deep.get_parents()[0].name == "RT5"
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_empty_retrieve():
+ """Special case: ``do_not_retrieve`` is set, or the retrieve result is empty."""
+ model_str = """
+RT1:
+ description: Some text.
+RT2:
+ obligatory_properties:
+ RT1:
+# some_text:
+# datatype: TEXT
+NoRecords:
+ description: A RecordType without Records.
+ recommended_properties:
+ some_text:
+ datatype: TEXT
+RT3:
+ obligatory_properties:
+ NoRecords:
+ """
+ model = parse_model_from_string(model_str)
+ schema_default = rtjs(model.get_deep("RT2"))
+ assert "oneOf" in schema_default["properties"]["RT1"]
+ assert any([el.get("title") == "Existing entries" for el in
+ schema_default["properties"]["RT1"]["oneOf"]])
+
+ schema_noexist = rtjs(model.get_deep("RT3"))
+ assert schema_noexist["properties"]["NoRecords"].get("type") == "object"
+
+ schema_noexist_noremote = rtjs(model.get_deep("RT3"), no_remote=True)
+ assert schema_noexist_noremote["properties"]["NoRecords"].get("type") == "object"
+ assert (schema_noexist_noremote["properties"]["NoRecords"].get("properties")
+ == OrderedDict([('some_text', {'type': 'string'})]))
+
+ uischema = {}
+ schema_noexist_noretrieve = rtjs(model.get_deep("RT2"), do_not_retrieve=["RT1"],
+ rjsf=uischema)
+ assert schema_noexist_noretrieve["properties"]["RT1"].get("type") == "object"
+ assert "some_date" in schema_noexist_noretrieve["properties"]["RT1"].get("properties")
+ assert not uischema
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_multiple_choice():
+ """Multiple choice is mostyly a matter of UI."""
+ model_str = """
+RT1:
+RT21:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+RT3:
+ obligatory_properties:
+ RT21:
+RT4:
+ obligatory_properties:
+ RT21:
+ datatype: LIST<RT21>
+ """
+ model = parse_model_from_string(model_str)
+ # generate a multiple choice, in first level
+ schema, uischema = rtjs(model.get_deep("RT21"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ assert schema["properties"]["RT1"]["uniqueItems"] is True
+ assert str(uischema) == "{'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}"
+
+ # second level
+ schema, uischema = rtjs(model.get_deep("RT3"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ assert schema["properties"]["RT21"]["properties"]["RT1"]["uniqueItems"] is True
+ assert (str(uischema)
+ == "{'RT21': {'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}}")
+
+ # second level with lists
+ schema, uischema = rtjs(model.get_deep("RT4"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ assert schema["properties"]["RT21"]["items"]["properties"]["RT1"]["uniqueItems"] is True
+ assert (str(uischema) ==
+ "{'RT21': {'items': {'RT1': {'ui:widget': 'checkboxes', "
+ "'ui:inline': True}}}}")
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_uischema():
+ model_str = """
+RT1:
+RT2:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+RT3:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+ """
+ model = parse_model_from_string(model_str)
+ schema_2, uischema_2 = rtjs(model.get_deep("RT2"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+ schema_3, uischema_3 = rtjs(model.get_deep("RT3"), additional_properties=False,
+ do_not_create=["RT1"], multiple_choice=["RT1"], rjsf=True)
+
+ # Merging #################################################################
+ # Using dictionaries
+ schemas_dict = {"schema_2": schema_2, "schema_3": schema_3}
+ uischemas_dict = {"schema_2": uischema_2, "schema_3": uischema_3}
+ merged_dict, merged_dict_ui = jsex.merge_schemas(schemas_dict, uischemas_dict)
+ assert merged_dict_ui["schema_2"] == merged_dict_ui["schema_3"]
+ assert (str(merged_dict_ui["schema_2"])
+ == "{'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}")
+
+ # Using lists
+ schemas_list = [schema_2, schema_3]
+ uischemas_list = [uischema_2, uischema_3]
+ merged_list, merged_list_ui = jsex.merge_schemas(schemas_list, uischemas_list)
+ assert merged_list["properties"]["RT2"] == merged_dict["properties"]["schema_2"]
+ assert merged_list_ui["RT2"] == merged_list_ui["RT3"]
+ assert merged_list_ui["RT2"] == merged_dict_ui["schema_2"]
+
+ # Asserting failures
+ with raises(ValueError):
+ jsex.merge_schemas(schemas_dict, uischemas_list)
+ with raises(ValueError):
+ jsex.merge_schemas(schemas_list, uischemas_dict)
+
+ # Arraying ################################################################
+ array2, array2_ui = jsex.make_array(schema_2, uischema_2)
+ assert array2["items"] == schema_2
+ assert array2_ui["items"] == uischema_2
+ assert (str(array2_ui["items"])
+ == "{'RT1': {'ui:widget': 'checkboxes', 'ui:inline': True}}")
+
+
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
+def test_schema_customization_with_dicts():
+ """Testing the ``additional_json_schema`` and ``additional_ui_schema`` parameters."""
+ model_str = """
+RT1:
+RT21:
+ obligatory_properties:
+ RT1:
+ datatype: LIST<RT1>
+ text:
+ datatype: TEXT
+ description: Some description
+RT3:
+ obligatory_properties:
+ number:
+ datatype: INTEGER
+ """
+ model = parse_model_from_string(model_str)
+
+ custom_schema = {
+ "RT21": {
+ "minProperties": 2,
+ },
+ "text": {
+ "format": "email",
+ "description": "Better description.",
+ },
+ "number": {
+ "minimum": 0,
+ "exclusiveMaximum": 100,
+ },
+ }
+
+ custom_ui_schema = {
+ "text": {
+ "ui:help": "Hint: keep it short.",
+ "ui:widget": "password",
+ },
+ "number": {
+ "ui:order": 2,
+ }
+ }
+
+ schema_21, uischema_21 = rtjs(model.get_deep("RT21"), additional_properties=False,
+ do_not_create=["RT1"], rjsf=True)
+ assert len(uischema_21) == 0
+ assert schema_21["properties"]["text"]["description"] == "Some description"
+ assert "format" not in schema_21["properties"]["text"]
+
+ schema_21, uischema_21 = rtjs(model.get_deep("RT21"), additional_properties=False,
+ additional_json_schema=custom_schema,
+ additional_ui_schema=custom_ui_schema, do_not_create=["RT1"],
+ rjsf=True)
+ assert (str(uischema_21)
+ == "{'text': {'ui:help': 'Hint: keep it short.', 'ui:widget': 'password'}}")
+ assert schema_21["properties"]["text"]["description"] == "Better description."
+ assert schema_21["properties"]["text"].get("format") == "email"
+ assert schema_21.get("minProperties") == 2
+
+ schema_3, uischema_3 = rtjs(model.get_deep("RT3"), additional_properties=False,
+ additional_json_schema=custom_schema,
+ additional_ui_schema=custom_ui_schema, rjsf=True)
+ assert (json.dumps(schema_3["properties"]["number"]) ==
+ '{"type": "integer", "minimum": 0, "exclusiveMaximum": 100}')
+ assert (str(uischema_3) == "{'number': {'ui:order': 2}}")
diff --git a/unittests/test_json_schema_model_parser.py b/unittests/test_json_schema_model_parser.py
index 7f47890f413dce5511cd498fe802e03a1af3be70..a991076e6a1e1a3e92cafc7f1bb88b42b4b2ab3d 100644
--- a/unittests/test_json_schema_model_parser.py
+++ b/unittests/test_json_schema_model_parser.py
@@ -103,7 +103,7 @@ def test_datamodel_with_atomic_properties():
assert isinstance(rt2, db.RecordType)
assert rt2.name == "Dataset2"
assert not rt2.description
- assert len(rt2.get_properties()) == 5
+ assert len(rt2.get_properties()) == 6
date_prop = rt2.get_property("date")
assert date_prop.datatype == db.DATETIME
@@ -121,6 +121,9 @@ def test_datamodel_with_atomic_properties():
float_prop2 = rt2.get_property("number_prop")
assert float_prop.datatype == float_prop2.datatype
+ null_prop = rt2.get_property("null_prop")
+ assert null_prop.datatype == db.TEXT
+
def test_required_no_list():
"""Exception must be raised when "required" is not a list."""
@@ -164,7 +167,7 @@ def test_enum():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["license"])
+ assert model[name].has_parent(model["license"], retrieve=False)
# Also allow enums with non-string types
number_enums = ["1.1", "2.2", "3.3"]
@@ -181,7 +184,7 @@ def test_enum():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["number_enum"])
+ assert model[name].has_parent(model["number_enum"], retrieve=False)
@pytest.mark.xfail(reason="Don't allow integer enums until https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/224 has been fixed")
@@ -207,7 +210,7 @@ def test_int_enum():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["int_enum"])
+ assert model[name].has_parent(model["int_enum"], retrieve=False)
def test_references():
@@ -339,7 +342,7 @@ def test_list():
assert isinstance(model[name], db.Record)
assert model[name].name == name
assert len(model[name].parents) == 1
- assert model[name].has_parent(model["license"])
+ assert model[name].has_parent(model["license"], retrieve=False)
def test_name_property():
@@ -356,3 +359,130 @@ def test_name_property():
assert str(err.value).startswith(
"The 'name' property must be string-typed, otherwise it cannot be identified with CaosDB's "
"name property.")
+
+
+def test_no_toplevel_entity():
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_no_toplevel_entity.schema.json"), top_level_recordtype=False)
+
+ assert "Dataset1" in model
+ rt1 = model["Dataset1"]
+
+ assert rt1.name == "Dataset1"
+ assert rt1.description == "Some description"
+ assert len(rt1.get_properties()) == 4
+
+ assert rt1.get_property("title") is not None
+ assert rt1.get_property("campaign") is not None
+ assert rt1.get_property("number_prop") is not None
+ assert rt1.get_property("user_id") is not None
+
+ title_prop = rt1.get_property("title")
+ assert title_prop.datatype == db.TEXT
+ assert rt1.get_importance(title_prop.name) == db.OBLIGATORY
+
+ campaign_prop = rt1.get_property("campaign")
+ assert campaign_prop.datatype == db.TEXT
+ assert rt1.get_importance(campaign_prop.name) == db.RECOMMENDED
+
+ float_prop = rt1.get_property("number_prop")
+ assert float_prop.datatype == db.DOUBLE
+ assert rt1.get_importance(float_prop.name) == db.OBLIGATORY
+
+ uid_prop = rt1.get_property("user_id")
+ assert uid_prop.datatype == db.TEXT
+ assert rt1.get_importance(uid_prop.name) == db.RECOMMENDED
+
+ # pattern properties without top-level entity:
+ assert "__PatternEntry_1" in model
+ assert "__PatternEntry_2" in model
+
+ pattern_boolean_rt = model["__PatternEntry_1"]
+ assert "pattern: " in pattern_boolean_rt.description
+ assert len(pattern_boolean_rt.properties) == 2
+ pp = pattern_boolean_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_boolean_rt.get_importance(pp.name) == db.OBLIGATORY
+ value_prop = pattern_boolean_rt.get_property("__PatternEntry_1_value")
+ assert value_prop.datatype == db.BOOLEAN
+
+ pattern_object_rt = model["__PatternEntry_2"]
+ assert "pattern: " in pattern_object_rt.description
+ assert len(pattern_object_rt.properties) == 2
+ pp = pattern_object_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_object_rt.get_importance(pp.name) == db.OBLIGATORY
+ date_id_prop = pattern_object_rt.get_property("date_id")
+ assert date_id_prop.datatype == db.TEXT
+
+
+def test_missing_array_items():
+
+ # strict behavior
+ with pytest.raises(JsonSchemaDefinitionError) as err:
+ parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_missing_array_items.schema.json"))
+
+ assert "{'type': 'array'}" in str(err)
+
+ # ignore all problems, so a RT is created that does not have the property
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_missing_array_items.schema.json"), ignore_unspecified_array_items=True)
+ assert "something_with_missing_array_items" in model
+ rt = model["something_with_missing_array_items"]
+ assert isinstance(rt, db.RecordType)
+ assert rt.get_property("missing") is None
+
+ # specify the type:
+ type_dict = {"missing": db.FILE}
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_missing_array_items.schema.json"), types_for_missing_array_items=type_dict)
+ assert "something_with_missing_array_items" in model
+ rt = model["something_with_missing_array_items"]
+ assert rt.get_property("missing") is not None
+ assert rt.get_property("missing").datatype == db.LIST(db.FILE)
+
+
+def test_pattern_properties():
+
+ model = parse_model_from_json_schema(os.path.join(
+ FILEPATH, "datamodel_pattern_properties.schema.json"))
+
+ assert "Dataset" in model
+ rt1 = model["Dataset"]
+ assert len(rt1.properties) == 2
+ for name in ["DatasetEntry_1", "DatasetEntry_2"]:
+ assert rt1.get_property(name) is not None
+ assert rt1.get_property(name).is_reference()
+
+ pattern_boolean_rt = model["DatasetEntry_1"]
+ assert "pattern: " in pattern_boolean_rt.description
+ assert len(pattern_boolean_rt.properties) == 2
+ pp = pattern_boolean_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_boolean_rt.get_importance(pp.name) == db.OBLIGATORY
+ value_prop = pattern_boolean_rt.get_property("DatasetEntry_1_value")
+ assert value_prop.datatype == db.BOOLEAN
+
+ pattern_object_rt = model["DatasetEntry_2"]
+ assert "pattern: " in pattern_object_rt.description
+ assert len(pattern_object_rt.properties) == 2
+ pp = pattern_object_rt.get_property("__matched_pattern")
+ assert pp.datatype == db.TEXT
+ assert pattern_object_rt.get_importance(pp.name) == db.OBLIGATORY
+ date_id_prop = pattern_object_rt.get_property("date_id")
+ assert date_id_prop.datatype == db.TEXT
+
+ assert "Dataset2" in model
+ rt2 = model["Dataset2"]
+ assert len(rt2.properties) == 2
+ # This has been tested elsewhere, just make sure that it is properly created
+ # in the presence of pattern properties, too.
+ assert rt2.get_property("datetime") is not None
+
+ assert rt2.get_property("Literally anything") is not None
+ assert rt2.get_property("Literally anything").is_reference()
+
+ pattern_named_rt = model["Literally anything"]
+ assert len(pattern_named_rt.properties) == 1
+ assert pattern_named_rt.get_property("__matched_pattern") is not None
diff --git a/unittests/test_table_importer.py b/unittests/test_table_importer.py
index 70f0f87f8706d72c386b18f54b7a9a10908eb477..599ea535d95d0b6c1216a935813d71c8e90c1d3b 100644
--- a/unittests/test_table_importer.py
+++ b/unittests/test_table_importer.py
@@ -41,6 +41,16 @@ from caosadvancedtools.table_importer import (CSVImporter, TableImporter,
from test_utils import BaseMockUpTest
+# For testing the table importer
+IMPORTER_KWARGS = dict(
+ converters={'c': float, 'd': yes_no_converter, 'x': float}, # x does not exist
+ datatypes={'a': str, 'b': int, 'float': float, 'x': int}, # x does not exist
+ obligatory_columns=['a', 'b'], unique_keys=[('a', 'b')],
+ existing_columns=['e'],
+)
+VALID_DF = pd.DataFrame(
+ [['a', 1, 2.0, 'yes', np.nan]], columns=['a', 'b', 'c', 'd', 'e'])
+
class ConverterTest(unittest.TestCase):
def test_yes_no(self):
@@ -143,22 +153,16 @@ class ConverterTest(unittest.TestCase):
class TableImporterTest(unittest.TestCase):
def setUp(self):
- self.importer_kwargs = dict(
- converters={'c': float, 'd': yes_no_converter},
- datatypes={'a': str, 'b': int},
- obligatory_columns=['a', 'b'], unique_keys=[('a', 'b')])
- self.valid_df = pd.DataFrame(
- [['a', 1, 2.0, 'yes']], columns=['a', 'b', 'c', 'd'])
+ self.importer_kwargs = IMPORTER_KWARGS
+ self.valid_df = VALID_DF
def test_missing_col(self):
- # check missing from converters
- df = pd.DataFrame(columns=['a', 'b', 'c'])
- importer = TableImporter(**self.importer_kwargs)
- self.assertRaises(ValueError, importer.check_columns, df)
- # check missing from datatypes
- df = pd.DataFrame(columns=['a', 'd', 'c'])
+ # check missing from existing
+ df = pd.DataFrame(columns=['a', 'b'])
importer = TableImporter(**self.importer_kwargs)
- self.assertRaises(ValueError, importer.check_columns, df)
+ with pytest.raises(DataInconsistencyError) as die:
+ importer.check_columns(df)
+ assert "Column 'e' missing" in str(die.value)
# check valid
importer.check_columns(self.valid_df)
@@ -177,12 +181,47 @@ class TableImporterTest(unittest.TestCase):
self.assertEqual(df_new.shape[1], 4)
self.assertEqual(df_new.iloc[0].b, 5)
+ # check that missing array-valued fields are detected correctly:
+ df = pd.DataFrame([[[None, None], 4, 2.0, 'yes'],
+ ['b', 5, 3.0, 'no']],
+ columns=['a', 'b', 'c', 'd'])
+ df_new = importer.check_missing(df)
+ self.assertEqual(df_new.shape[0], 1)
+ self.assertEqual(df_new.shape[1], 4)
+ self.assertEqual(df_new.iloc[0].b, 5)
+
def test_wrong_datatype(self):
importer = TableImporter(**self.importer_kwargs)
- df = pd.DataFrame([[None, np.nan, 2.0, 'yes'],
+ df = pd.DataFrame([[1234, 0, 2.0, 3, 'yes'],
+ [5678, 1, 2.0, 3, 'yes']],
+ columns=['a', 'b', 'c', 'float', 'd'])
+ # wrong datatypes before
+ assert df["a"].dtype == int
+ assert df["float"].dtype == int
+ # strict = False by default, so this shouldn't raise an error
+ importer.check_datatype(df)
+ # The types should be correct now.
+ assert df["a"].dtype == pd.StringDtype
+ assert df["float"].dtype == float
+
+ # Resetting `df` since check_datatype may change datatypes
+ df = pd.DataFrame([[None, 0, 2.0, 'yes'],
[5, 1, 2.0, 'yes']],
columns=['a', 'b', 'c', 'd'])
- self.assertRaises(DataInconsistencyError, importer.check_datatype, df)
+ # strict=True, so number in str column raises an error
+ self.assertRaises(DataInconsistencyError, importer.check_datatype, df, None, True)
+
+ df = pd.DataFrame([[0],
+ [1]],
+ columns=['float'])
+ # strict=True, so int in float column raises an error
+ self.assertRaises(DataInconsistencyError, importer.check_datatype, df, None, True)
+
+ # This is always wrong (float in int column)
+ df = pd.DataFrame([[None, np.nan, 2.0, 'yes'],
+ [5, 1.7, 2.0, 'yes']],
+ columns=['a', 'b', 'c', 'd'])
+ self.assertRaises(DataInconsistencyError, importer.check_datatype, df, None, False)
def test_unique(self):
importer = TableImporter(**self.importer_kwargs)
@@ -193,6 +232,35 @@ class TableImporterTest(unittest.TestCase):
self.assertEqual(df_new.shape[0], 1)
+def test_check_dataframe_existing_obligatory_columns(caplog):
+ """Needs caplog so remove from above class."""
+ # stricter test case; column 'a' must exist and have a value
+ strict_kwargs = IMPORTER_KWARGS.copy()
+ strict_kwargs["existing_columns"].append('a')
+
+ importer = TableImporter(**strict_kwargs)
+
+ # the valid df is still valid, since 'a' has a value
+ importer.check_dataframe(VALID_DF)
+
+ # Now 'a' doesn't
+ df_missing_a = pd.DataFrame(
+ [[np.nan, 1, 2.0, 'yes', 'e']], columns=['a', 'b', 'c', 'd', 'e'])
+
+ new_df = importer.check_dataframe(df_missing_a)
+ # Column is removed and a warning is in the logger:
+ assert new_df.shape[0] == 0
+ assert "Required information is missing (a) in 1. row" in caplog.text
+
+ df_missing_c = pd.DataFrame(
+ [['a', 1, 'yes', np.nan]], columns=['a', 'b', 'd', 'e'])
+ new_df = importer.check_dataframe(df_missing_c)
+ assert new_df.shape[0] == 1
+ assert new_df.shape[1] == 4
+
+ caplog.clear()
+
+
class XLSImporterTest(TableImporterTest):
def test_full(self):
""" test full run with example data """
@@ -233,6 +301,30 @@ class CSVImporterTest(TableImporterTest):
importer = CSVImporter(**self.importer_kwargs)
importer.read_file(tmp.name)
+ def test_with_generous_datatypes(self):
+ """Same as above but check that values are converted as expected."""
+ tmp = NamedTemporaryFile(delete=False, suffix=".csv")
+ tmp.close()
+ self.valid_df.to_csv(tmp.name)
+ # Copy and use float for columns with integer values, string for columns
+ # with numeric values
+ kwargs = self.importer_kwargs.copy()
+ kwargs["datatypes"] = {
+ 'a': str,
+ 'b': float,
+ 'c': str
+ }
+ importer = CSVImporter(**kwargs)
+ importer.read_file(tmp.name)
+
+ kwargs["datatypes"] = {
+ 'a': str,
+ 'b': str,
+ 'c': str
+ }
+ importer = CSVImporter(**kwargs)
+ importer.read_file(tmp.name)
+
class TSVImporterTest(TableImporterTest):
def test_full(self):
diff --git a/unittests/test_update_cache.py b/unittests/test_update_cache.py
index 4720f23de0b651b90e3b74ee13e06088462c5e31..8376da482b4828dd09de2ac6f3aca4fb9617c08d 100644
--- a/unittests/test_update_cache.py
+++ b/unittests/test_update_cache.py
@@ -42,8 +42,8 @@ class CacheTest(unittest.TestCase):
return c
def setUp(self):
- self.cache = UpdateCache(db_file=NamedTemporaryFile(delete=False).name)
- self.cache.create_cache()
+ self.cache = UpdateCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
self.run_id = "235234"
def test_insert(self):
diff --git a/unittests/test_utils.py b/unittests/test_utils.py
index 7369931799b00eba5a835458a6fad474de1d9039..468e9200de723c65c75e21912b5b3940d758821c 100644
--- a/unittests/test_utils.py
+++ b/unittests/test_utils.py
@@ -26,7 +26,7 @@ from tempfile import NamedTemporaryFile
import caosdb as db
from caosadvancedtools.utils import (check_win_path, get_referenced_files,
- string_to_person)
+ string_to_person, create_entity_link)
from caosdb import RecordType, configure_connection, get_config
from caosdb.connection.mockup import MockUpResponse, MockUpServerConnection
from caosdb.exceptions import TransactionError
@@ -140,3 +140,8 @@ class PathTest(unittest.TestCase):
assert check_win_path(r"C:\hallo")
assert check_win_path(r"\hallo")
assert not check_win_path("/hallo")
+
+
+class EntityLinkTest(unittest.TestCase):
+ def test_link(self):
+ assert "<a href='/Entity/1'>a</a>" == create_entity_link(db.Entity(id=1, name='a'))
diff --git a/unittests/test_yaml_model_parser.py b/unittests/test_yaml_model_parser.py
index 625e609e806c6eae9cb5fbacd347076dfd5053bc..de0ad8ae88927eba32059ada7af534289471a115 100644
--- a/unittests/test_yaml_model_parser.py
+++ b/unittests/test_yaml_model_parser.py
@@ -1,9 +1,27 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
import unittest
from datetime import date
from tempfile import NamedTemporaryFile
-from pytest import raises
+from pytest import deprecated_call, raises, mark
-import caosdb as db
+import linkahead as db
from caosadvancedtools.models.parser import (TwiceDefinedException,
YamlDefinitionError,
parse_model_from_string,
@@ -284,10 +302,12 @@ A:
def test_reference_property(self):
"""Test correct creation of reference property using an RT."""
- modeldef = """A:
+ modeldef = """
+A:
recommended_properties:
ref:
datatype: LIST<A>
+ description: new description
"""
model = parse_model_from_string(modeldef)
self.assertEqual(len(model), 2)
@@ -297,6 +317,7 @@ A:
elif key == "ref":
self.assertTrue(isinstance(value, db.Property))
self.assertEqual(value.datatype, "LIST<A>")
+ assert value.description == "new description"
class ExternTest(unittest.TestCase):
@@ -340,6 +361,35 @@ A:
assert "line {}".format(line) in yde.exception.args[0]
+def test_existing_model():
+ """Parsing more than one model may require to append to existing models."""
+ model_str_1 = """
+A:
+ obligatory_properties:
+ number:
+ datatype: INTEGER
+ """
+ model_str_2 = """
+B:
+ obligatory_properties:
+ A:
+ """
+ model_1 = parse_model_from_string(model_str_1)
+ model_2 = parse_model_from_string(model_str_2, existing_model=model_1)
+ for ent in ["A", "B", "number"]:
+ assert ent in model_2
+
+ model_str_redefine = """
+number:
+ datatype: DOUBLE
+ description: Hello number!
+ """
+ model_redefine = parse_model_from_string(model_str_redefine, existing_model=model_1)
+ print(model_redefine)
+ assert model_redefine["number"].description == "Hello number!"
+ assert model_redefine["number"].datatype == db.INTEGER # FIXME Shouldn't this be DOUBLE?
+
+
def test_define_role():
model = """
A:
@@ -476,7 +526,8 @@ F:
entities = parse_model_from_string(model)
-
+@mark.xfail(reason="""Issue is
+ https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/89""")
def test_double_property_value():
"""
Test whether it is possible to define a property, use it in a record
@@ -493,3 +544,75 @@ rec:
value: 2
"""
entities = parse_model_from_string(model)
+
+def test_issue_36():
+ """Test whether the `parent` keyword is deprecated.
+
+ See https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36.
+
+ """
+ model_string = """
+R1:
+ obligatory_properties:
+ prop1:
+ datatype: TEXT
+R2:
+ obligatory_properties:
+ prop2:
+ datatype: TEXT
+ recommended_properties:
+ prop3:
+ datatype: TEXT
+R3:
+ parent:
+ - R2
+ inherit_from_obligatory:
+ - R1
+"""
+ with deprecated_call():
+ # Check whether this is actually deprecated
+ model = parse_model_from_string(model_string)
+
+ assert "R3" in model
+ r3 = model["R3"]
+ assert isinstance(r3, db.RecordType)
+ for par in ["R1", "R2"]:
+ # Until removal, both do the same
+ assert has_parent(r3, par)
+ assert r3.get_parent(par)._flags["inheritance"] == db.OBLIGATORY
+
+
+def test_yaml_error():
+ """Testing error while parsing a yaml.
+ """
+
+ with raises(ValueError, match=r"line 2: .*"):
+ parse_model_from_yaml("unittests/models/model_invalid.yml")
+
+
+def test_inherit_error():
+ """Must fail with an understandable exception."""
+ model_string = """
+prop1:
+ inherit_from_obligatory: prop2
+ """
+ with raises(YamlDefinitionError,
+ match=r"Parents must be a list but is given as string: prop1 > prop2"):
+ parse_model_from_string(model_string)
+
+
+@mark.xfail(reason="""Issue is
+ https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/57""")
+def test_inherit_properties():
+ # TODO Is not even specified yet.
+ model_string = """
+prop1:
+ datatype: DOUBLE
+prop2:
+# role: Property
+ inherit_from_obligatory:
+ - prop1
+ """
+ model = parse_model_from_string(model_string)
+ prop2 = model["prop2"]
+ assert prop2.role == "Property"
diff --git a/utils/branch_exists.py b/utils/branch_exists.py
new file mode 100755
index 0000000000000000000000000000000000000000..9626e4aa81e4ee2bd9a239f6a0650dc4e383593f
--- /dev/null
+++ b/utils/branch_exists.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""
+Exit with error code 2 if the branch does not exist.
+"""
+import sys
+import argparse
+import requests
+from ref_to_commit import get_remote
+
+
+def branch_exists(repository, branch):
+ remote = get_remote(repository)
+ resp = requests.get(remote+"/repository/branches/"+branch).json()
+ return "message" not in resp
+
+
+def define_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("repository")
+ parser.add_argument("branchname")
+
+ return parser
+
+
+if __name__ == "__main__":
+ parser = define_parser()
+ args = parser.parse_args()
+ ret = branch_exists(repository=args.repository, branch=args.branchname)
+ if ret is False:
+ print("branch does not exist.")
+ sys.exit(2)
+ else:
+ print("branch exists.")
diff --git a/utils/ref_to_commit.py b/utils/ref_to_commit.py
new file mode 100755
index 0000000000000000000000000000000000000000..93f15f31b6158172cfca5a5095b13f6a4fcb22ab
--- /dev/null
+++ b/utils/ref_to_commit.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+replaces git branch names with the newest commit hash using gitlab api
+"""
+import argparse
+
+import requests
+
+
+_REPOS = {
+ "SERVER": "https://gitlab.indiscale.com/api/v4/projects/100",
+ "WEBUI": "https://gitlab.indiscale.com/api/v4/projects/98",
+ "PYLIB": "https://gitlab.indiscale.com/api/v4/projects/97",
+ "MYSQLBACKEND": "https://gitlab.indiscale.com/api/v4/projects/101",
+ "PYINT": "https://gitlab.indiscale.com/api/v4/projects/99",
+ "CPPLIB": "https://gitlab.indiscale.com/api/v4/projects/107",
+ "CPPINT": "https://gitlab.indiscale.com/api/v4/projects/111",
+ "ADVANCEDUSERTOOLS": "https://gitlab.indiscale.com/api/v4/projects/104"
+}
+
+
+def get_remote(repository):
+ return _REPOS[repository]
+
+
+def ref_to_commit(repository, reference):
+ remote = get_remote(repository)
+ r = requests.get(remote+"/repository/branches/"+reference).json()
+
+ if "name" in r:
+ return r["commit"]["short_id"]
+
+ return reference
+
+
+def define_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("repository")
+ parser.add_argument("reference")
+
+ return parser
+
+
+if __name__ == "__main__":
+ parser = define_parser()
+ args = parser.parse_args()
+ ret = ref_to_commit(repository=args.repository, reference=args.reference)
+ print(ret)