diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 066923e6765883752d36f2ba1c349e1cff21b809..14c3c1efc5b3974f6952b5ed439723c58b4627a5 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -1,3 +1,26 @@
+###############################
+###### Temporary Image ########
+###############################
+FROM debian:bookworm as git_base
+
+# Check for availability of DNS
+RUN if getent hosts indiscale.com > /dev/null; \
+ then echo "Connected to the internet and DNS available"; \
+ else echo "No internet connection or DNS not available"; \
+ fi
+
+COPY . /git
+
+# Delete .git because it is huge.
+RUN rm -r /git/.git
+
+# Install pycaosdb.ini for the tests
+RUN mv /git/.docker/tester_pycaosdb.ini /git/integrationtests/pycaosdb.ini
+
+###############################
+###### Main Image Build #######
+###############################
+
FROM debian:bookworm
RUN apt-get update && \
apt-get install \
@@ -26,16 +49,8 @@ ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANC
advanced_version.json
RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \
cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install --break-system-packages .[h5-crawler]
-COPY . /git
-
-# Delete .git because it is huge.
-RUN rm -r /git/.git
-
-# Install pycaosdb.ini for the tests
-RUN mv /git/.docker/tester_pycaosdb.ini /git/integrationtests/pycaosdb.ini
-# TODO Remove once https://github.com/ResearchObject/ro-crate-py/issues/203 has been resolved.
-RUN pip3 install --break-system-packages git+https://github.com/salexan2001/ro-crate-py.git@f-automatic-dummy-ids
+COPY --from=git_base /git /git
RUN cd /git/ && pip3 install --break-system-packages .[h5-crawler,spss,rocrate]
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index 02ccac5c48e039a3374a0d169f3b355f897e45fc..97f70320e37b1c1d8623e5fc1d98b6d72916e2b8 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -1,7 +1,7 @@
version: '3.7'
services:
sqldb:
- image: mariadb:10.4
+ image: mariadb:11.4
environment:
MYSQL_ROOT_PASSWORD: caosdb1234
networks:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 58476b391235eea3b8fff018dd361317fa5d3b83..f295e0b9480d56359ec1f1d30bc3be5bd54aea57 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,6 +104,27 @@ stages:
CAOSDB_TAG=${REFTAG};
fi
- echo $CAOSDB_TAG
+ - if [ -z "$PYLIB" ]; then
+ if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+ echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
+ if wget -O /dev/null https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME}>/dev/null ; then
+ PYLIB=$CI_COMMIT_REF_NAME ;
+ fi;
+ fi;
+ fi;
+ - PYLIB=${PYLIB:-dev}
+ - echo $PYLIB
+
+ - if [ -z "$ADVANCED" ]; then
+ if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+ echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
+ if wget -O /dev/null https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
+ ADVANCED=$CI_COMMIT_REF_NAME ;
+ fi;
+ fi;
+ fi;
+ - ADVANCED=${ADVANCED:-dev}
+ - echo $ADVANCED
info:
tags: [cached-dind]
@@ -113,51 +134,58 @@ info:
script:
- *env
-unittest_py3.11:
- tags: [cached-dind]
- stage: test
- image: $CI_REGISTRY_IMAGE
- script:
- - python3 -c "import sys; assert sys.version.startswith('3.11')"
- - tox
-
unittest_py3.9:
tags: [cached-dind]
stage: test
+ variables:
+ PYVER: "3.9"
image: python:3.9
script: &python_test_script
# install dependencies
+ - *env
- pip install pytest pytest-cov
- # TODO: Use f-branch logic here
- - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
- - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
- # TODO: Remove once
- # https://github.com/ResearchObject/ro-crate-py/issues/203 has
- # been resolved.
- - pip install git+https://github.com/salexan2001/ro-crate-py.git@f-automatic-dummy-ids
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@${PYLIB}
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@${ADVANCED}
- pip install .[h5-crawler,spss,rocrate]
+ - echo "import sys; assert sys.version.startswith('$PYVER')"
+ - python3 -c "import sys; assert sys.version.startswith('$PYVER')"
# actual test
- caosdb-crawler --help
- - pytest --cov=caosdb -vv ./unittests
+ - make unittest
unittest_py3.10:
- tags: [cached-dind]
+ variables:
+ PYVER: "3.10"
stage: test
+ tags: [cached-dind]
image: python:3.10
script: *python_test_script
-unittest_py3.12:
+unittest_py3.11:
+ variables:
+ PYVER: "3.11"
tags: [cached-dind]
stage: test
+ image: python:3.11
+ script: *python_test_script
+
+unittest_py3.12:
+ variables:
+ PYVER: "3.12"
+ stage: test
+ tags: [cached-dind]
image: python:3.12
script: *python_test_script
unittest_py3.13:
+ variables:
+ PYVER: "3.13"
tags: [cached-dind]
stage: test
image: python:3.13
script: *python_test_script
+
inttest:
tags: [docker]
services:
@@ -174,6 +202,8 @@ inttest:
- *env
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- echo $CAOSDB_TAG
+ - echo $PYLIB
+ - echo $ADVANCED
- cd .docker
# Store mariadb version
@@ -235,34 +265,14 @@ build-testenv:
- pushes
needs: []
script:
+ - *env
- df -h
- command -v wget
- - if [ -z "$PYLIB" ]; then
- if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
- echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
- PYLIB=$CI_COMMIT_REF_NAME ;
- fi;
- fi;
- fi;
- - PYLIB=${PYLIB:-dev}
- - echo $PYLIB
-
- - if [ -z "$ADVANCED" ]; then
- if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
- echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
- ADVANCED=$CI_COMMIT_REF_NAME ;
- fi;
- fi;
- fi;
- - ADVANCED=${ADVANCED:-dev}
- - echo $ADVANCED
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker build
- --build-arg PYLIB=${PYLIB}
+ --build-arg PYLIB=${PYLIB:dev}
--build-arg ADVANCED=${ADVANCED:dev}
--file .docker/Dockerfile
-t $CI_REGISTRY_IMAGE .
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a09037614b8ba004ef0374ec76b42c0f7e512e1d..f54c2a254cae78a242630207434ec73b77ad2abc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,61 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.11.0] - 2025-03-05 ##
+
+### Added ###
+
+- Validation module for checking a list of generated records against a list of json schemas
+ that can be generated from a yaml data model file.
+- DictElementConverters can now make use of `match_properties` which
+ works analogous to `match_properties` in ROCrateEntityConverter and
+ `match_attrib` in XMLConverter.
+- `match_properties` is a method of class Converter and can for
+ example be used by CustomConverters.
+- ZipFileConverter that opens zip files and exposes their contents as
+ File and Directory structure elements.
+- `linkahead-crawler` script as alias for `caosdb-crawler`.
+- New transformers of the form `cast_to_*` which allow casting
+ variables to `int`, `float`, `str` and `bool`.
+- Transformer function definition in the cfood support variable
+ substitutions now.
+- `crawler_main` and `scanner.scan_directory` now support list of
+ directories to be crawled, too. Note that giving a list of
+ directories is currently incompatible with
+ `securityMode=SecurityMode.RETRIEVE` or
+ `securityMode=SecurityMode.INSERT` since the functionality to
+ authoriye pending inserts or updates doesn't support path lists yet
+ and will raise a NotImplementedError for now.
+- `match_newer_than_file` option for `DirectoryConverter`: A reference
+ file containing (only) an ISO-formatted datetime string can be
+ specified here. Directories with this option won't match if all
+ their contents were last modified before that datetime.
+
+### Changed ###
+
+- Registered identifiables can also be used by children of the given RecordType
+ if no registered identifiable is defined for them.
+- ROCrate converter supports dereferencing property values with a single "@id"-property during
+ subtree generation.
+- ROCrate converter supports the special property "variablesMeasured" in addition to "hasPart".
+- `None` and other NA values (i.e., values where `pandas.isna` is
+ `True`) are now interpreted as empty strings in
+ `converters.match_name_and_value` instead of being cast to string naïvely
+
+### Fixed ###
+
+- `spss_to_datamodel` script works again.
+- The cfood now supports bi-directional references when defining records on the same level.
+ (See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/175)
+- [#112](https://gitlab.com/linkahead/linkahead-crawler/-/issues/112)
+ Children of CSVTableConverter match despite match_value: ".+" and
+ empty cell. This has been fixed by treating None and NA values in
+ `converters.match_name_and_value` (see above).
+
+### Documentation ###
+
+- Added documentation for ROCrateConverter, ELNFileConverter, and ROCrateEntityConverter
+
## [0.10.1] - 2024-11-13 ##
### Fixed ###
@@ -30,9 +85,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Units for properties. They can be specified by giving the property as a dict in the form
```yaml
MyRecord:
- my_prop:
- value: 5
- unit: m
+ my_prop:
+ value: 5
+ unit: m
```
- Support for Python 3.13
- ROCrateConverter, ELNFileConverter and ROCrateEntityConverter for crawling ROCrate and .eln files
diff --git a/CITATION.cff b/CITATION.cff
index ed859432b26cde913f7283fb8e969a97b7b74f41..8f4e22a4f8b56c8640e7d0a9a5ccae93010b4847 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -17,6 +17,6 @@ authors:
given-names: Alexander
orcid: https://orcid.org/0000-0003-4124-9649
title: CaosDB - Crawler
-version: 0.10.1
+version: 0.11.0
doi: 10.3390/data9020024
-date-released: 2024-11-13
\ No newline at end of file
+date-released: 2025-03-05
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 95fc2bf61473b94decfb43d0c5ba0d3fda535a07..7167ebfdf106f5129ce7941706b9e871d51e551f 100644
--- a/Makefile
+++ b/Makefile
@@ -44,5 +44,5 @@ lint:
.PHONY: lint
unittest:
- tox -r
+ pytest --cov=caoscrawler -vv ./unittests
.PHONY: unittest
diff --git a/integrationtests/test_crawler_main.py b/integrationtests/test_crawler_main.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2eebf4f04e195754eaf71dc5e829b6a77a4cc4b
--- /dev/null
+++ b/integrationtests/test_crawler_main.py
@@ -0,0 +1,95 @@
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# 2024 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import logging
+import tempfile
+
+from pathlib import Path
+
+import linkahead as db
+
+from caoscrawler import crawl
+from caoscrawler.crawl import (crawler_main, SecurityMode)
+from linkahead.utils.register_tests import clear_database, set_test_key
+
+set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
+
+INTTESTDIR = Path(__file__).parent
+
+
+def test_list_of_paths(clear_database, monkeypatch):
+
+ # Mock the status record
+ dummy_status = {
+ "n_calls": 0
+ }
+
+ def _mock_update_status_record(run_id, n_inserts, n_updates, status):
+ print("Update mocked status")
+ dummy_status["run_id"] = run_id
+ dummy_status["n_inserts"] = n_inserts
+ dummy_status["n_updates"] = n_updates
+ dummy_status["status"] = status
+ dummy_status["n_calls"] += 1
+ monkeypatch.setattr(crawl, "_update_status_record", _mock_update_status_record)
+
+ # mock SSS environment
+ monkeypatch.setenv("SHARED_DIR", tempfile.gettempdir())
+
+ # We need only one dummy RT
+ rt = db.RecordType(name="TestType").insert()
+ basepath = INTTESTDIR / "test_data" / "crawler_main_with_list_of_dirs"
+ dirlist = [basepath / "dir1", basepath / "dir2"]
+ crawler_main(
+ dirlist,
+ cfood_file_name=basepath / "cfood.yml",
+ identifiables_definition_file=basepath / "identifiable.yml"
+ )
+ recs = db.execute_query("FIND TestType")
+ assert len(recs) == 2
+ assert "Test1" in [r.name for r in recs]
+ assert "Test2" in [r.name for r in recs]
+
+ assert dummy_status["n_inserts"] == 2
+ assert dummy_status["n_updates"] == 0
+ assert dummy_status["status"] == "OK"
+ assert dummy_status["n_calls"] == 1
+
+
+def test_not_implemented_list_with_authorization(caplog, clear_database):
+
+ rt = db.RecordType(name="TestType").insert()
+ basepath = INTTESTDIR / "test_data" / "crawler_main_with_list_of_dirs"
+ dirlist = [basepath / "dir1", basepath / "dir2"]
+
+ # This is not implemented yet, so check log for correct error.
+ ret = crawler_main(
+ dirlist,
+ cfood_file_name=basepath / "cfood.yml",
+ identifiables_definition_file=basepath / "identifiable.yml",
+ securityMode=SecurityMode.RETRIEVE
+ )
+ # crawler_main hides the error, but has a non-zero return code and
+ # errors in the log:
+ assert ret != 0
+ err_tuples = [t for t in caplog.record_tuples if t[1] == logging.ERROR]
+ assert len(err_tuples) == 1
+ assert "currently implemented only for single paths, not for lists of paths" in err_tuples[0][2]
+ # No inserts after the errors
+ assert len(db.execute_query("FIND TestType")) == 0
diff --git a/integrationtests/test_data/crawler_main_with_list_of_dirs/cfood.yml b/integrationtests/test_data/crawler_main_with_list_of_dirs/cfood.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c7f22ce07e9b401915aefde3bf7e3a78d92e2bd6
--- /dev/null
+++ b/integrationtests/test_data/crawler_main_with_list_of_dirs/cfood.yml
@@ -0,0 +1,10 @@
+---
+metadata:
+ crawler-version: 0.10.2
+---
+BaseDirElement:
+ type: Directory
+ match: ^dir(?P<dir_number>[0-9]+)$$
+ records:
+ TestType:
+ name: Test$dir_number
diff --git a/integrationtests/test_data/crawler_main_with_list_of_dirs/dir1/.gitkeep b/integrationtests/test_data/crawler_main_with_list_of_dirs/dir1/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/integrationtests/test_data/crawler_main_with_list_of_dirs/dir2/.gitkeep b/integrationtests/test_data/crawler_main_with_list_of_dirs/dir2/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/integrationtests/test_data/crawler_main_with_list_of_dirs/identifiable.yml b/integrationtests/test_data/crawler_main_with_list_of_dirs/identifiable.yml
new file mode 100644
index 0000000000000000000000000000000000000000..6d608cece0ae7c2aa6461fb56025a8ac8e4faf6f
--- /dev/null
+++ b/integrationtests/test_data/crawler_main_with_list_of_dirs/identifiable.yml
@@ -0,0 +1,2 @@
+TestType:
+ - name
diff --git a/integrationtests/test_issues.py b/integrationtests/test_issues.py
index cb1e2e0925dd85b9f6cadf2b56b22aface4bb468..0506fa4db03e9b3638051e6ec4fa132bd348a988 100644
--- a/integrationtests/test_issues.py
+++ b/integrationtests/test_issues.py
@@ -1,4 +1,4 @@
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
# 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
@@ -16,20 +16,22 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
+import tempfile
+
import linkahead as db
+import yaml
from caosadvancedtools.models.parser import parse_model_from_string
from caoscrawler.crawl import Crawler
from caoscrawler.identifiable import Identifiable
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
-from caoscrawler.scanner import (create_converter_registry,
+from caoscrawler.scanner import (_load_definition_from_yaml_dict,
+ create_converter_registry,
scan_structure_elements)
from caoscrawler.structure_elements import DictElement
from linkahead.cached import cache_clear
from linkahead.utils.register_tests import clear_database, set_test_key
from pytest import fixture, mark, raises
-import tempfile
-
set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
@@ -332,6 +334,64 @@ def test_indiscale_87(clear_database):
print("---")
+def test_issue_16(clear_database):
+ """
+ This is another a test for:
+ https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/16
+
+ In addition to the two unit tests for recursive definition in `test_scanner.py` this system test
+ tests whether recursively defined records can be synchronized correctly using the crawler.
+ """
+ recursive_yaml = """
+FirstConverter:
+ type: DictElement
+ records:
+ Experiment:
+ subtree:
+ Converter:
+ type: DictElement
+ records:
+ Block:
+ name: block 1
+ Experiment: $Experiment
+ Experiment:
+ name: experiment 1
+ Block: $Block
+ """
+
+ crawler_definition = _load_definition_from_yaml_dict(
+ [yaml.load(recursive_yaml, Loader=yaml.SafeLoader)])
+ converter_registry = create_converter_registry(crawler_definition)
+
+ # Nested DictElements that match the yaml structure in recursive_yaml:
+ data = {"data": {
+ }}
+ records = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
+ converter_registry)
+
+ rt_exp = db.RecordType(name="Experiment").insert()
+ rt_block = db.RecordType(name="Block").insert()
+
+ ident = CaosDBIdentifiableAdapter()
+ ident.load_from_yaml_object(yaml.safe_load("""
+Experiment:
+- name
+Block:
+- name
+"""))
+
+ crawler = Crawler(identifiableAdapter=ident)
+ crawler.synchronize(crawled_data=records)
+
+ exp_res = db.execute_query("FIND Experiment")
+ assert len(exp_res) == 1
+ exp_block = db.execute_query("FIND Block")
+ assert len(exp_block) == 1
+
+ assert exp_res[0].get_property("Block").value == exp_block[0].id
+ assert exp_block[0].get_property("Experiment").value == exp_res[0].id
+
+
def test_issue_14(clear_database):
"""
Issue title: Some parent updates are required before inserts
diff --git a/setup.cfg b/setup.cfg
index 5eadc00b53a8e2515351909aa7a6f7c8a7ce0071..da645c0d7615a1a3caab8dabd8af1893b72bdf61 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = caoscrawler
-version = 0.10.1
+version = 0.11.0
author = Alexander Schlemmer
author_email = alexander.schlemmer@ds.mpg.de
description = A new crawler for LinkAhead
@@ -39,8 +39,9 @@ per-file-ignores = __init__.py:F401
[options.entry_points]
console_scripts =
+ linkahead-crawler = caoscrawler.crawl:main
caosdb-crawler = caoscrawler.crawl:main
- spss_to_datamodel = caoscrawler.conv_impl.spss:spss_to_datamodel_main
+ spss_to_datamodel = caoscrawler.converters.spss:spss_to_datamodel_main
csv_to_datamodel = caoscrawler.scripts.generators:csv_to_datamodel_main
[options.extras_require]
@@ -49,3 +50,5 @@ h5-crawler =
numpy
spss =
pandas[spss]
+rocrate =
+ rocrate
diff --git a/src/caoscrawler/cfood-schema.yml b/src/caoscrawler/cfood-schema.yml
index c5e0eaad092c12efbceb5f55b62b3d7cf8afdccf..d2e4cea24f0f2803499116420091b36e95b2c781 100644
--- a/src/caoscrawler/cfood-schema.yml
+++ b/src/caoscrawler/cfood-schema.yml
@@ -88,6 +88,12 @@ cfood:
match_value:
description: a regexp that is matched to the value of a key-value pair
type: string
+ match_newer_than_file:
+ description: |
+ Only relevant for Directory. A path to a file containing
+ an ISO-formatted datetime. Only match if the contents of the
+ Directory have been modified after that datetime.
+ type: string
record_from_dict:
description: Only relevant for PropertiesFromDictElement. Specify the root record which is generated from the contained dictionary.
type: object
diff --git a/src/caoscrawler/converters/__init__.py b/src/caoscrawler/converters/__init__.py
index 670d4e966c72c6bcf45d0d46c1db715fb79d8ab5..edb7b3633cea2657dc3b9638379a3e57c37c87e4 100644
--- a/src/caoscrawler/converters/__init__.py
+++ b/src/caoscrawler/converters/__init__.py
@@ -18,11 +18,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-"""Submdule containing all default and optional converters."""
+"""Submodule containing all default and optional converters."""
from .. import utils
from .converters import *
from .xml_converter import *
+from .zipfile_converter import ZipFileConverter
try:
from .spss import SPSSConverter
diff --git a/src/caoscrawler/converters/converters.py b/src/caoscrawler/converters/converters.py
index 64a557ce4e26fd8bfd345000d3abf18bf0360117..e16b2c0fbaeeee419b0e3f235339dc18cd4da885 100644
--- a/src/caoscrawler/converters/converters.py
+++ b/src/caoscrawler/converters/converters.py
@@ -249,11 +249,35 @@ out: tuple
return (propvalue, propunit, collection_mode)
-def create_records(values: GeneralStore, records: RecordStore, def_records: dict):
- # list of keys to identify, which variables have been set by which paths:
- # the items are tuples:
- # 0: record name
- # 1: property name
+def create_records(values: GeneralStore,
+ records: RecordStore,
+ def_records: dict) -> list[tuple[str, str]]:
+ """
+ Create records in GeneralStore `values` and RecordStore `records` as given
+ by the definition in `def_records`.
+
+ This function will be called during scanning using the cfood definition.
+ It also should be used by CustomConverters to set records as automatic substitution
+ and other crawler features are applied automatically.
+
+ Parameters
+ ----------
+ values: GeneralStore
+ This GeneralStore will be used to access variables that are needed during variable substitution
+ in setting the properties of records and files.
+ Furthermore, the records that are generated in this function will be stored in this GeneralStore
+ **additionally to** storing them in the RecordStore given as the second argument to this function.
+
+ records: RecordStore
+ The RecordStore where the generated records will be stored.
+
+ Returns
+ -------
+ : list[tuple[str, str]]
+ A list of tuples containing the record names (1st element of tuple) and respective property names
+ as 2nd element of the tuples. This list will be used by the scanner for creating the debug tree.
+
+ """
keys_modified = []
for name, record in def_records.items():
@@ -286,11 +310,22 @@ def create_records(values: GeneralStore, records: RecordStore, def_records: dict
if (role == "Record" and "parents" not in record):
c_record.add_parent(name)
- c_record = records[name]
-
if isinstance(record, str):
raise RuntimeError(
"dict expected, but found str: {}".format(record))
+
+ # We do a second run over the def_records, here. Having finished the first run
+ # for creating the records (in the variable and records stores) makes sure that
+ # records, that are defined on this level can already be accessed during variable substitution
+ # in the properties that will be set in the next block.
+ for name, record in def_records.items():
+ # See above:
+ if record is None:
+ record = {}
+
+ c_record = records[name]
+
+ # Set the properties:
for key, value in record.items():
if key == "parents" or key == "role":
continue
@@ -320,7 +355,8 @@ def create_records(values: GeneralStore, records: RecordStore, def_records: dict
c_record.add_property(name=key, value=propvalue, unit=propunit)
else:
if collection_mode == "list":
- if propunit and c_record.get_property(key).unit and propunit != c_record.get_property(key).unit:
+ if (propunit and c_record.get_property(key).unit
+ and propunit != c_record.get_property(key).unit):
raise RuntimeError(
f"Property '{key}' has contradictory units: "
f"{propunit} and {c_record.get_property(key).unit}"
@@ -456,6 +492,90 @@ class Converter(object, metaclass=ABCMeta):
raise RuntimeError("Condition does not match.")
values.update(m)
+ def match_properties(self, properties: dict, vardict: dict, label: str = "match_properties"):
+ """This method can be used to generically match 'match_properties' from the cfood definition
+ with the behavior described as follows:
+
+ 'match_properties' is a dictionary of key-regexps and value-regexp pairs. Each key matches
+ a property name and the corresponding value matches its property value.
+
+ What a property means in the context of the respective converter can be different, examples:
+
+ * XMLTag: attributes of the node
+ * ROCrate: properties of the ROCrateEntity
+ * DictElement: properties of the dict
+
+ label can be used to customize the name of the dictionary in the definition.
+
+ This method is not called by default, but can be called from child classes.
+
+ Typically it would be used like this from methods overwriting `match`::
+
+ if not self.match_properties(<properties>, vardict):
+ return None
+
+ vardict will be updated in place when there are
+ matches. <properties> is a dictionary taken from the structure
+ element that contains the properties in the context of this
+ converter.
+
+
+ Parameters
+ ----------
+
+ properties: dict
+ The dictionary containing the properties to be matched.
+
+ vardict: dict
+ This dictionary will be used to store the variables created during the matching.
+
+ label: str
+ Default "match_properties". Can be used to change the name
+ of the property in the definition. E.g. the xml converter
+ uses "match_attrib" which makes more sense in the context
+ of xml trees.
+
+ Returns
+ -------
+
+ : bool
+ Returns True when properties match and False
+ otherwise. The vardict dictionary is updated in place.
+
+ """
+ if label in self.definition:
+ # This matcher works analogously to the attributes matcher in the XMLConverter
+ for prop_def_key, prop_def_value in self.definition[label].items():
+ match_counter = 0
+ matched_m_prop = None
+ matched_m_prop_value = None
+ for prop_key, prop_value in properties.items():
+ # print("{} = {}".format(prop_key, prop_value))
+ # TODO: automatic conversion to str ok?
+ m_prop = re.match(prop_def_key, str(prop_key))
+ if m_prop is not None:
+ match_counter += 1
+ matched_m_prop = m_prop
+ # TODO: automatic conversion to str ok?
+ m_prop_value = re.match(prop_def_value, str(prop_value))
+ if m_prop_value is None:
+ return False
+ matched_m_prop_value = m_prop_value
+ # TODO: How to deal with multiple matches?
+ # There are multiple options:
+ # - Allow multiple attribute-key matches: Leads to possible overwrites of variables
+ # - Require unique attribute-key and attribute-value matches: Very complex
+ # - Only allow one single attribute-key to match and run attribute-value match separately.
+ # Currently the latter option is implemented.
+ # TODO: The ROCrateEntityConverter implements a very similar behavior.
+ if match_counter == 0:
+ return False
+ elif match_counter > 1:
+ raise RuntimeError("Multiple properties match the same {} entry.".format(label))
+ vardict.update(matched_m_prop.groupdict())
+ vardict.update(matched_m_prop_value.groupdict())
+ return True
+
def apply_transformers(self, values: GeneralStore, transformer_functions: dict):
"""
Check if transformers are defined using the "transform" keyword.
@@ -491,10 +611,19 @@ class Converter(object, metaclass=ABCMeta):
" one element with they key being the name"
" of the function!")
tr_func_key = list(tr_func_el.keys())[0]
- tr_func_params = tr_func_el[tr_func_key]
+
if tr_func_key not in transformer_functions:
raise RuntimeError("Unknown transformer function: {}".format(tr_func_key))
+ # Do variable replacment on function parameters:
+ if tr_func_el[tr_func_key] is not None:
+ # Create a copy of the function parameters:
+ tr_func_params = dict(tr_func_el[tr_func_key])
+ for key in tr_func_params:
+ tr_func_params[key] = replace_variables(tr_func_params[key], values)
+ else:
+ tr_func_params = None
+
# Retrieve the function from the dictionary:
tr_func = transformer_functions[tr_func_key]
# Call the function:
@@ -676,6 +805,11 @@ class DirectoryConverter(Converter):
m = re.match(self.definition["match"], element.name)
if m is None:
return None
+ if "match_newer_than_file" in self.definition:
+ last_modified = self._get_most_recent_change_in_dir(element)
+ reference = self._get_reference_file_timestamp()
+ if last_modified < reference:
+ return None
return m.groupdict()
@staticmethod
@@ -698,6 +832,49 @@ class DirectoryConverter(Converter):
return children
+ @staticmethod
+ def _get_most_recent_change_in_dir(element: Directory) -> datetime.datetime:
+ """Return the datetime of the most recent change of any file
+ or directory in the given Directory element.
+
+ """
+ most_recent = os.path.getmtime(element.path)
+
+ for root, _, files in os.walk(element.path):
+ mtimes = [os.path.getmtime(root)] + \
+ [os.path.getmtime(os.path.join(root, fname)) for fname in files]
+ if max(mtimes) > most_recent:
+ most_recent = max(mtimes)
+
+ return datetime.datetime.fromtimestamp(most_recent)
+
+ def _get_reference_file_timestamp(self) -> datetime.datetime:
+ """Return a time stamp read from a reference file if it
+ exists. Otherwise return datetime.datetime.min, i.e., the
+ earliest datetime known to datetime.
+
+ """
+
+ if "match_newer_than_file" not in self.definition:
+ logger.debug("No reference file specified.")
+ return datetime.datetime.min
+
+ elif not os.path.isfile(self.definition["match_newer_than_file"]):
+ logger.debug("Reference file doesn't exist.")
+ return datetime.datetime.min
+
+ with open(self.definition["match_newer_than_file"]) as ref_file:
+ stamp_str = ref_file.readline().strip()
+ try:
+ return datetime.datetime.fromisoformat(stamp_str)
+ except ValueError as e:
+ logger.error(
+ f"Reference file in {self.definition['match_newer_than_file']} "
+ "doesn't contain a ISO formatted datetime in its first line. "
+ "Match regardless of modification times."
+ )
+ raise e
+
class SimpleFileConverter(Converter):
"""Just a file, ignore the contents."""
@@ -876,7 +1053,12 @@ class DictElementConverter(Converter):
# TODO: See comment on types and inheritance
if not isinstance(element, DictElement):
raise RuntimeError("Element must be a DictElement.")
- return match_name_and_value(self.definition, element.name, element.value)
+ vardict = match_name_and_value(self.definition, element.name, element.value)
+
+ if not self.match_properties(element.value, vardict):
+ return None
+
+ return vardict
class PropertiesFromDictConverter(DictElementConverter):
@@ -1113,17 +1295,17 @@ class YAMLFileConverter(SimpleFileConverter):
def match_name_and_value(definition, name, value):
"""Take match definitions from the definition argument and apply regular expression to name and
- possibly value
+ possibly value.
- one of the keys 'match_name' and "match' needs to be available in definition
- 'match_value' is optional
+ Exactly one of the keys ``match_name`` and ``match`` must exist in ``definition``,
+ ``match_value`` is optional
Returns
-------
out:
None, if match_name or match lead to no match. Otherwise, returns a dictionary with the
- matched groups, possibly including matches from using match_value
+ matched groups, possibly including matches from using `definition["match_value"]`
"""
if "match_name" in definition:
@@ -1145,7 +1327,10 @@ out:
m1 = {}
if "match_value" in definition:
- m2 = re.match(definition["match_value"], str(value), re.DOTALL)
+ # None values will be interpreted as empty strings for the
+ # matcher.
+ m_value = str(value) if (value is not None and not pd.isna(value)) else ""
+ m2 = re.match(definition["match_value"], m_value, re.DOTALL)
if m2 is None:
return None
else:
diff --git a/src/caoscrawler/converters/rocrate.py b/src/caoscrawler/converters/rocrate.py
index b84462acba2fdd7e60094e38edc38605c80deb11..7dcad86589961f03f1e755ddbc0b60742cf4ed4a 100644
--- a/src/caoscrawler/converters/rocrate.py
+++ b/src/caoscrawler/converters/rocrate.py
@@ -32,15 +32,13 @@ import tempfile
from typing import Optional
from zipfile import ZipFile
-import linkahead as db
import rocrate
from rocrate.rocrate import ROCrate
-from ..stores import GeneralStore, RecordStore
+from ..stores import GeneralStore
from ..structure_elements import (Directory, File, ROCrateEntity,
StructureElement)
-from .converters import (Converter, ConverterValidationError,
- SimpleFileConverter, convert_basic_element)
+from .converters import Converter, SimpleFileConverter, convert_basic_element
class ROCrateConverter(SimpleFileConverter):
@@ -169,33 +167,24 @@ class ROCrateEntityConverter(Converter):
# Store the result of all individual regexp variable results:
vardict = {}
+ # TODO: I accidentally used "match_type" instead
+ # of "match_entity_type". This was completely
+ # unnoticed. So add it to schema and adapt tests.
+
if "match_entity_type" in self.definition:
- m_type = re.match(self.definition["match_entity_type"], element.type)
+ entity_type = element.entity.type
+ if isinstance(entity_type, list):
+ # TODO: this seems to be a bug in kadi4mat RO-Crates
+ # ./ has type ['Dataset']
+ # instead of type 'Dataset'
+ entity_type = entity_type[0]
+ m_type = re.match(self.definition["match_entity_type"], entity_type)
if m_type is None:
return None
vardict.update(m_type.groupdict())
- if "match_properties" in self.definition:
- # This matcher works analogously to the attributes matcher in the XMLConverter
- for prop_def_key, prop_def_value in self.definition["match_properties"].items():
- match_counter = 0
- matched_m_prop = None
- matched_m_prop_value = None
- for prop_key, prop_value in element.entity.properties().items():
- m_prop = re.match(prop_def_key, prop_key)
- if m_prop is not None:
- match_counter += 1
- matched_m_prop = m_prop
- m_prop_value = re.match(prop_def_value, prop_value)
- if m_prop_value is None:
- return None
- matched_m_prop_value = m_prop_value
- if match_counter == 0:
- return None
- elif match_counter > 1:
- raise RuntimeError("Multiple properties match the same match_prop entry.")
- vardict.update(matched_m_prop.groupdict())
- vardict.update(matched_m_prop_value.groupdict())
+ if not self.match_properties(element.entity.properties(), vardict):
+ return None
return vardict
@@ -207,7 +196,21 @@ class ROCrateEntityConverter(Converter):
# Add the properties:
for name, value in eprops.items():
- children.append(convert_basic_element(value, name))
+ if isinstance(value, dict):
+ # This is - according to the standard - only allowed, if it's flat, i.e.
+ # it contains a single element with key == "@id" and the id as value which
+ # is supposed to be dereferenced:
+ if not (len(value) == 1 and "@id" in value):
+ raise RuntimeError("The JSON-LD is not flat.")
+ dereferenced = element.entity.crate.dereference(value["@id"])
+ if dereferenced is not None:
+ children.append(
+ ROCrateEntity(element.folder, dereferenced))
+ else:
+ # This is just an external ID and will be added as simple DictElement
+ children.append(convert_basic_element(value, name))
+ else:
+ children.append(convert_basic_element(value, name))
# Add the files:
if isinstance(element.entity, rocrate.model.file.File):
@@ -215,10 +218,12 @@ class ROCrateEntityConverter(Converter):
children.append(File(name, os.path.join(element.folder, path, name)))
# Parts of this entity are added as child entities:
- if "hasPart" in eprops:
- for p in eprops["hasPart"]:
- children.append(
- ROCrateEntity(element.folder, element.entity.crate.dereference(
- p["@id"])))
+ for sublist in ("hasPart", "variableMeasured"):
+ if sublist in eprops:
+ for p in eprops[sublist]:
+ children.append(
+ ROCrateEntity(element.folder, element.entity.crate.dereference(
+ p["@id"])))
+ # TODO: See https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/195 for discussion.
return children
diff --git a/src/caoscrawler/converters/xml_converter.py b/src/caoscrawler/converters/xml_converter.py
index b9f7487ee633d0ba25a3b81b78b9a3561274edc9..60d7b49431fb011a06b7105a16471b0b3c7b2268 100644
--- a/src/caoscrawler/converters/xml_converter.py
+++ b/src/caoscrawler/converters/xml_converter.py
@@ -25,10 +25,9 @@ from __future__ import annotations
import re
from typing import Optional
-import linkahead as db
import lxml.etree
-from ..stores import GeneralStore, RecordStore
+from ..stores import GeneralStore
from ..structure_elements import (File, StructureElement, XMLAttributeNode,
XMLTagElement, XMLTextNode)
from .converters import (Converter, ConverterValidationError,
@@ -163,33 +162,8 @@ class XMLTagConverter(Converter):
return None
vardict.update(m_text.groupdict())
- if "match_attrib" in self.definition:
- for attrib_def_key, attrib_def_value in self.definition["match_attrib"].items():
- match_counter = 0
- matched_m_attrib = None
- matched_m_attrib_value = None
- for attr_key, attr_value in element.tag.attrib.items():
- m_attrib = re.match(attrib_def_key, attr_key)
- if m_attrib is not None:
- match_counter += 1
- matched_m_attrib = m_attrib
- m_attrib_value = re.match(attrib_def_value, attr_value)
- if m_attrib_value is None:
- return None
- matched_m_attrib_value = m_attrib_value
- # TODO: How to deal with multiple matches?
- # There are multiple options:
- # - Allow multiple attribute-key matches: Leads to possible overwrites of variables
- # - Require unique attribute-key and attribute-value matches: Very complex
- # - Only allow one single attribute-key to match and run attribute-value match separately.
- # Currently the latter option is implemented.
- # TODO: The ROCrateEntityConverter implements a very similar behavior.
- if match_counter == 0:
- return None
- elif match_counter > 1:
- raise RuntimeError("Multiple attributes match the same match_attrib entry.")
- vardict.update(matched_m_attrib.groupdict())
- vardict.update(matched_m_attrib_value.groupdict())
+ if not self.match_properties(element.tag.attrib, vardict, "match_attrib"):
+ return None
return vardict
diff --git a/src/caoscrawler/converters/zipfile_converter.py b/src/caoscrawler/converters/zipfile_converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..7073e66a266168e17eb9b6143e7dc6292b5149dc
--- /dev/null
+++ b/src/caoscrawler/converters/zipfile_converter.py
@@ -0,0 +1,82 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Converters take structure elements and create Records and new structure elements from them.
+
+This converter opens zip files, unzips them into a temporary directory and
+exposes its contents as File structure elements.
+
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from os.path import isdir, join
+from zipfile import ZipFile
+
+from ..stores import GeneralStore
+from ..structure_elements import Directory, File, StructureElement
+from .converters import SimpleFileConverter
+
+
+class ZipFileConverter(SimpleFileConverter):
+
+ """Convert zipfiles.
+ """
+
+ def setup(self):
+ self._tempdir = None
+
+ def cleanup(self):
+ self._tempdir.cleanup()
+
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
+ """
+ Loads an ROCrate from an rocrate file or directory.
+
+ Arguments:
+ ----------
+ element must be a File or Directory (structure element).
+
+ Returns:
+ --------
+ A list with an ROCrateElement representing the contents of the .eln-file or None
+ in case of errors.
+ """
+
+ if isinstance(element, File):
+ self._tempdir = tempfile.TemporaryDirectory()
+ unzd_path = self._tempdir.name
+ with ZipFile(element.path) as zipf:
+ zipf.extractall(unzd_path)
+
+ entity_ls = []
+ for el in os.listdir(unzd_path):
+ path = join(unzd_path, el)
+ if isdir(path):
+ entity_ls.append(Directory(el, path))
+ else:
+ entity_ls.append(File(el, path))
+
+ return entity_ls
+ else:
+ raise ValueError("create_children was called with wrong type of StructureElement")
+ return None
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index a79e4434ee8f58fd1cc2646ced85c0d02d3fb66b..e0d243979faee8f44cdcee3b0e49c15af640c378 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -531,8 +531,8 @@ one with the entities that need to be updated and the other with entities to be
prop.value = Crawler._get_property_id_for_datatype(
rtname=prop.datatype, name=prop.value)
except (db.EmptyUniqueQueryError, db.QueryNotUniqueError):
- logger.error("The Property {prop.name} with datatype={prop.datatype} has the "
- "value {prop.value} and there is no appropriate Entity with such "
+ logger.error(f"The Property {prop.name} with datatype={prop.datatype} has the "
+ f"value {prop.value} and there is no appropriate Entity with such "
"a name.")
raise
else:
@@ -548,8 +548,8 @@ one with the entities that need to be updated and the other with entities to be
name=el))
except (db.EmptyUniqueQueryError, db.QueryNotUniqueError):
logger.error(
- "The Property {prop.name} with datatype={prop.datatype} has the "
- "value {prop.value} and there is no appropriate Entity with such "
+ f"The Property {prop.name} with datatype={prop.datatype} has the "
+ f"value {prop.value} and there is no appropriate Entity with such "
"a name.")
raise
else:
@@ -621,7 +621,7 @@ one with the entities that need to be updated and the other with entities to be
crawled_data: Optional[list[db.Record]] = None,
no_insert_RTs: Optional[list[str]] = None,
no_update_RTs: Optional[list[str]] = None,
- path_for_authorized_run: Optional[str] = "",
+ path_for_authorized_run: Optional[Union[str, list[str]]] = "",
):
"""
This function applies several stages:
@@ -643,7 +643,7 @@ one with the entities that need to be updated and the other with entities to be
no_update_RTs : list[str], optional
list of RecordType names. Records that have one of those RecordTypes
as parent will not be updated
- path_for_authorized_run : str, optional
+ path_for_authorized_run : str or list[str], optional
only used if there are changes that need authorization before being
applied. The form for rerunning the crawler with the authorization
of these changes will be generated with this path. See
@@ -661,6 +661,12 @@ one with the entities that need to be updated and the other with entities to be
"use for example the Scanner to create this data."))
crawled_data = self.crawled_data
+ if isinstance(path_for_authorized_run, list) and self.securityMode != SecurityMode.UPDATE:
+ raise NotImplementedError(
+ "Authorization of inserts and updates is currently implemented only "
+ "for single paths, not for lists of paths."
+ )
+
to_be_inserted, to_be_updated = self._split_into_inserts_and_updates(
SyncGraph(crawled_data, self.identifiableAdapter))
@@ -1004,7 +1010,7 @@ def _store_dry_run_data(ins, upd):
"update": updates}))
-def crawler_main(crawled_directory_path: str,
+def crawler_main(crawled_directory_path: Union[str, list[str]],
cfood_file_name: str,
identifiables_definition_file: Optional[str] = None,
debug: bool = False,
@@ -1022,8 +1028,8 @@ def crawler_main(crawled_directory_path: str,
Parameters
----------
- crawled_directory_path : str
- path to be crawled
+ crawled_directory_path : str or list[str]
+ path(s) to be crawled
cfood_file_name : str
filename of the cfood to be used
identifiables_definition_file : str
@@ -1115,42 +1121,28 @@ def crawler_main(crawled_directory_path: str,
crawler.run_id)
_update_status_record(crawler.run_id, len(inserts), len(updates), status="OK")
return 0
- except ForbiddenTransaction as err:
- logger.debug(traceback.format_exc())
- logger.error(err)
- _update_status_record(crawler.run_id, 0, 0, status="FAILED")
- return 1
- except ConverterValidationError as err:
- logger.debug(traceback.format_exc())
- logger.error(err)
- _update_status_record(crawler.run_id, 0, 0, status="FAILED")
- return 1
- except ImpossibleMergeError as err:
- logger.debug(traceback.format_exc())
- logger.error(
- "Encountered conflicting information when creating Records from the crawled "
- f"data:\n\n{err}"
- )
- _update_status_record(crawler.run_id, 0, 0, status="FAILED")
- return 1
- except TransactionError as err:
- logger.debug(traceback.format_exc())
- logger.error(err)
- logger.error("Transaction error details:")
- for suberr in err.errors:
- logger.error("---")
- logger.error(suberr.msg)
- logger.error(suberr.entity)
- return 1
except Exception as err:
logger.debug(traceback.format_exc())
logger.error(err)
-
- if "SHARED_DIR" in os.environ:
- # pylint: disable=E0601
- domain = get_config_setting("public_host_url")
- logger.error("Unexpected Error: Please tell your administrator about this and provide "
- f"the following path.\n{get_shared_resource_link(domain, debuglog_public)}")
+ # Special treatment for known error types
+ if isinstance(err, ImpossibleMergeError):
+ logger.error(
+ "Encountered conflicting information when creating Records from the crawled "
+ f"data:\n\n{err}"
+ )
+ elif isinstance(err, TransactionError):
+ logger.error("Transaction error details:")
+ for suberr in err.errors:
+ logger.error("---")
+ logger.error(suberr.msg)
+ logger.error(suberr.entity)
+ # Unkown errors get a special message
+ elif not isinstance(err, (ConverterValidationError, ForbiddenTransaction)):
+ if "SHARED_DIR" in os.environ:
+ # pylint: disable=E0601
+ domain = get_config_setting("public_host_url")
+ logger.error("Unexpected Error: Please tell your administrator about this and provide "
+ f"the following path.\n{get_shared_resource_link(domain, debuglog_public)}")
_update_status_record(crawler.run_id, 0, 0, status="FAILED")
return 1
@@ -1174,6 +1166,7 @@ def parse_args():
"This file will only be generated if this option is set.")
parser.add_argument("--debug", required=False, action="store_true",
help="Path name of the cfood yaml file to be used.")
+ # TODO allow to provide multiple directories to be crawled on the commandline
parser.add_argument("crawled_directory_path",
help="The subtree of files below the given path will "
"be considered. Use '/' for everything.")
diff --git a/src/caoscrawler/default_transformers.yml b/src/caoscrawler/default_transformers.yml
index ffcb1b15bd2bad71083cc8f0ba84172ee3daf2b0..0de9a6e0585c5246fa5a21ffcbdfc37cfdc2b88d 100644
--- a/src/caoscrawler/default_transformers.yml
+++ b/src/caoscrawler/default_transformers.yml
@@ -15,3 +15,15 @@ date_parse:
datetime_parse:
package: caoscrawler.transformer_functions
function: datetime_parse
+cast_to_int:
+ package: caoscrawler.transformer_functions
+ function: cast_to_int
+cast_to_float:
+ package: caoscrawler.transformer_functions
+ function: cast_to_float
+cast_to_bool:
+ package: caoscrawler.transformer_functions
+ function: cast_to_bool
+cast_to_str:
+ package: caoscrawler.transformer_functions
+ function: cast_to_str
diff --git a/src/caoscrawler/identifiable_adapters.py b/src/caoscrawler/identifiable_adapters.py
index 592f603bef508771d734ff633f8cdb2c100742d5..6169a99e7bf47daffb53332b7e0b6513730f2561 100644
--- a/src/caoscrawler/identifiable_adapters.py
+++ b/src/caoscrawler/identifiable_adapters.py
@@ -45,6 +45,13 @@ from .utils import has_parent
logger = logging.getLogger(__name__)
+def _retrieve_RecordType(id=None, name=None):
+ """
+ Retrieve the RecordType from LinkAhead. For mocking purposes.
+ """
+ return db.RecordType(name=name, id=id).retrieve()
+
+
def get_children_of_rt(rtname):
"""Supply the name of a recordtype. This name and the name of all children RTs are returned in
a list"""
@@ -586,8 +593,7 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
self.load_from_yaml_object(identifiable_data)
def load_from_yaml_object(self, identifiable_data):
- """Load identifiables defined in a yaml object.
- """
+ """Load identifiables defined in a yaml object. """
for rt_name, id_list in identifiable_data.items():
rt = db.RecordType().add_parent(rt_name)
@@ -611,7 +617,7 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
self.register_identifiable(rt_name, rt)
def register_identifiable(self, name: str, definition: db.RecordType):
- self._registered_identifiables[name] = definition
+ self._registered_identifiables[name.lower()] = definition
def get_file(self, identifiable: Identifiable):
warnings.warn(
@@ -639,11 +645,42 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
"""
if len(record.parents) == 0:
return None
- # TODO We need to treat the case where multiple parents exist properly.
- rt_name = record.parents[0].name
- for name, definition in self._registered_identifiables.items():
- if definition.parents[0].name.lower() == rt_name.lower():
- return definition
+ registered = []
+ for parent in record.parents:
+ prt = _retrieve_RecordType(id=parent.id, name=parent.name)
+ reg = self._get_registered_for_rt(prt)
+ if reg is not None:
+ registered.append(reg)
+ # TODO we might in future want to check whether the registered identifiables are the same
+ if len(registered) > 1:
+ raise RuntimeError("Multiple registered identifiables found for a Record "
+ f"with the following parents: {record.parents}")
+ elif len(registered) == 1:
+ return registered[0]
+ else:
+ return None
+
+ def _get_registered_for_rt(self, rt: db.RecordType):
+ """
+ returns the registered identifiable for the given RecordType or the
+ registered identifiable of the first parent
+ """
+ if rt.name.lower() in self._registered_identifiables:
+ return self._registered_identifiables[rt.name.lower()]
+ if len(rt.parents) == 0:
+ return None
+ registered = []
+ for parent in rt.parents:
+ prt = _retrieve_RecordType(id=parent.id, name=parent.name)
+ registered.append(self._get_registered_for_rt(prt))
+ # TODO we might in future want to check whether the registered identifiables are the same
+ if len(registered) > 1:
+ raise RuntimeError("Multiple registered identifiables found for the RecordType "
+ f" {rt.name} with the following parents: {rt.parents}")
+ elif len(registered) == 1:
+ return registered[0]
+ else:
+ return None
def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
query_string = self.create_query_for_identifiable(identifiable)
diff --git a/src/caoscrawler/scanner.py b/src/caoscrawler/scanner.py
index 89bd1c04411665bf4832d6bccce69bbe1b11cad1..af1f4173e95827606a02979ddd6d7fcd9f133271 100644
--- a/src/caoscrawler/scanner.py
+++ b/src/caoscrawler/scanner.py
@@ -421,7 +421,7 @@ def scanner(items: list[StructureElement],
# --------------------------------------------------------------------------------
-def scan_directory(dirname: str, crawler_definition_path: str,
+def scan_directory(dirname: Union[str, list[str]], crawler_definition_path: str,
restricted_path: Optional[list[str]] = None,
debug_tree: Optional[DebugTree] = None):
""" Crawl a single directory.
@@ -434,10 +434,12 @@ def scan_directory(dirname: str, crawler_definition_path: str,
Parameters
----------
+ dirname: str or list[str]
+ directory or list of directories to be scanned
restricted_path: optional, list of strings
- Traverse the data tree only along the given path. When the end of the given path
- is reached, traverse the full tree as normal. See docstring of 'scanner' for
- more details.
+ Traverse the data tree only along the given path. When the end
+ of the given path is reached, traverse the full tree as
+ normal. See docstring of 'scanner' for more details.
Returns
-------
@@ -455,26 +457,31 @@ def scan_directory(dirname: str, crawler_definition_path: str,
if not dirname:
raise ValueError(
"You have to provide a non-empty path for crawling.")
- dir_structure_name = os.path.basename(dirname)
-
- # TODO: needs to be covered somewhere else
- crawled_directory = dirname
- if not dir_structure_name and dirname.endswith('/'):
- if dirname == '/':
- # Crawling the entire file system
- dir_structure_name = "root"
- else:
- # dirname had a trailing '/'
- dir_structure_name = os.path.basename(dirname[:-1])
-
- return scan_structure_elements(Directory(dir_structure_name,
- dirname),
- crawler_definition,
- converter_registry,
- restricted_path=restricted_path,
- debug_tree=debug_tree,
- registered_transformer_functions=registered_transformer_functions
- )
+ if not isinstance(dirname, list):
+ dirname = [dirname]
+ dir_element_list = []
+ for dname in dirname:
+ dir_structure_name = os.path.basename(dname)
+
+ # TODO: needs to be covered somewhere else
+ crawled_directory = dname
+ if not dir_structure_name and dname.endswith(os.path.sep):
+ if dname == os.path.sep:
+ # Crawling the entire file system
+ dir_structure_name = "root"
+ else:
+ # dirname had a trailing '/'
+ dir_structure_name = os.path.basename(dname[:-1])
+ dir_element_list.append(Directory(dir_structure_name, dname))
+
+ return scan_structure_elements(
+ dir_element_list,
+ crawler_definition,
+ converter_registry,
+ restricted_path=restricted_path,
+ debug_tree=debug_tree,
+ registered_transformer_functions=registered_transformer_functions
+ )
def scan_structure_elements(items: Union[list[StructureElement], StructureElement],
diff --git a/src/caoscrawler/sync_node.py b/src/caoscrawler/sync_node.py
index d912d6465a68270411c121f65b4c5a828c9c667e..46187c0d63afd6c18de6dd1df3304f13badb1899 100644
--- a/src/caoscrawler/sync_node.py
+++ b/src/caoscrawler/sync_node.py
@@ -256,9 +256,9 @@ class SyncNode(db.Entity):
def parent_in_list(parent: Parent, plist: ParentList) -> bool:
"""helper function that checks whether a parent with the same name or ID is in the plist"""
- return plist.filter(parent)
+ return plist.filter_by_identity(parent)
def property_in_list(prop: db.Property, plist: PropertyList) -> bool:
"""helper function that checks whether a property with the same name or ID is in the plist"""
- return plist.filter(prop)
+ return plist.filter_by_identity(prop)
diff --git a/src/caoscrawler/transformer_functions.py b/src/caoscrawler/transformer_functions.py
index ce08bc6bc05caa84f342cdc25f3243c5bab0b79c..117d0b021d4ec0b0efc79c5db0d7ed397207933f 100644
--- a/src/caoscrawler/transformer_functions.py
+++ b/src/caoscrawler/transformer_functions.py
@@ -99,3 +99,56 @@ Parameters
fmt = params.get("datetime_format", fmt_default)
dt_str = datetime.datetime.strptime(in_value, fmt).strftime(fmt_default)
return dt_str
+
+
+def cast_to_int(in_value: Any, params: dict) -> int:
+ """
+ Cast the `in_value` to int.
+
+ Parameters
+ ==========
+ No parameters.
+ """
+ return int(in_value)
+
+
+def cast_to_float(in_value: Any, params: dict) -> float:
+ """
+ Cast the `in_value` to float.
+
+ Parameters
+ ==========
+ No parameters.
+ """
+ return float(in_value)
+
+
+def cast_to_bool(in_value: Any, params: dict) -> bool:
+ """
+ Cast the `in_value` to bool.
+
+ This is done by comparing `in_value` to "True".
+ Only "true", "True", "False" and "false" are accepted as possible values.
+ All other input values raise an error.
+
+ Parameters
+ ==========
+ No parameters.
+ """
+ val = str(in_value).lower()
+ if val == "true":
+ return True
+ if val == "false":
+ return False
+ raise ValueError("Invalid value for type cast to bool: {}".format(in_value))
+
+
+def cast_to_str(in_value: Any, params: dict) -> str:
+ """
+ Cast the `in_value` to str.
+
+ Parameters
+ ==========
+ No parameters.
+ """
+ return str(in_value)
diff --git a/src/caoscrawler/validator.py b/src/caoscrawler/validator.py
new file mode 100644
index 0000000000000000000000000000000000000000..33e29b02db429e3382248bbd80d2d00cd7b07c6b
--- /dev/null
+++ b/src/caoscrawler/validator.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Alexander Schlemmer
+#
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+This module contains functions to validate the output of a scanner run with a
+json schema.
+"""
+
+import jsonschema
+import linkahead as db
+# from caosadvancedtools.models.parser import parse_model_from_string
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema
+from caosadvancedtools.models.parser import parse_model_from_yaml
+from jsonschema import ValidationError
+from linkahead.high_level_api import convert_to_python_object
+
+
+def load_json_schema_from_datamodel_yaml(filename: str) -> dict[str, dict]:
+ """
+ Load a data model yaml file (using caosadvancedtools) and convert
+ all record types into a json schema using the json_schema_exporter module.
+
+ Arguments
+ ---------
+ filename: str
+ The filename of the yaml file to load.
+
+ Returns
+ -------
+ A dict of json schema objects. The keys are the record types for which the schemas
+ are generated.
+ """
+
+ model = parse_model_from_yaml(filename)
+
+ rt_schemas = {}
+ for el_key, el in model.items():
+ if isinstance(el, db.RecordType):
+ rt_schemas[el_key] = recordtype_to_json_schema(el)
+
+ return rt_schemas
+
+
+def representer_ordereddict(dumper, data):
+ """
+ Helper function to be able to represent the converted json schema objects correctly as yaml.
+ This representer essentially replaced OrderedDict objects with simple dict objects.
+
+ Since Python 3.7 dicts are ordered by default, see e.g.:
+ https://softwaremaniacs.org/blog/2020/02/05/dicts-ordered/en/
+
+ Example how to use the representer:
+ ```python
+ yaml.add_representer(OrderedDict, caoscrawler.validator.representer_ordereddict)
+ ```
+ """
+ return dumper.represent_data(dict(data))
+
+
+def _apply_schema_patches(pobj: dict):
+ """
+ Changes applied:
+ - properties are moved vom subitem "proeprties" to top-level.
+ - The following keys are deleted: parents, role, name, description, metadata, properties
+ """
+ if "properties" not in pobj:
+ # this is probably a file
+ return pobj
+ for prop in pobj["properties"]:
+ if isinstance(pobj["properties"][prop], dict):
+ pobj[prop] = _apply_schema_patches(pobj["properties"][prop])
+ else:
+ pobj[prop] = pobj["properties"][prop]
+
+ for keyd in ("parents", "role", "name",
+ "description", "metadata", "properties"):
+ if keyd in pobj:
+ del pobj[keyd]
+
+ return pobj
+
+
+def convert_record(record: db.Record):
+ """
+ Convert a record into a form suitable for validation with jsonschema.
+
+ Uses `high_level_api.convert_to_python_object`
+ Afterwards `_apply_schema_patches` is called recursively to refactor the dictionary
+ to match the current form of the jsonschema.
+
+ Arguments:
+ ----------
+ record: db.Record
+ The record that is supposed to be converted.
+ """
+ pobj = convert_to_python_object(record).serialize()
+ return _apply_schema_patches(pobj)
+
+
+def validate(records: list[db.Record], schemas: dict[str, dict]) -> list[tuple]:
+ """
+ Validate a list of records against a dictionary of schemas.
+ The keys of the dictionary are record types and the corresponding values are json schemata
+ associated with that record type. The current implementation assumes that each record that is
+ checked has exactly one parent and raises an error if that is not the case.
+ The schema belonging to a record is identified using the name of the first (and only) parent
+ of the record.
+
+ Arguments:
+ ----------
+
+ records: list[db.Record]
+ List of records that will be validated.
+
+ schemas: dict[str, dict]
+ A dictionary of JSON schemas generated using `load_json_schema_from_datamodel_yaml`.
+
+ Returns:
+ --------
+ A list of tuples, one element for each record:
+
+ - Index 0: A boolean that determines whether the schema belonging to the record type of the
+ record matched.
+ - Index 1: A validation error if the schema did not match or None otherwise.
+ """
+
+ retval = []
+ for r in records:
+ if len(r.parents) != 1:
+ raise NotImplementedError(
+ "Schema validation is only supported if records have exactly one parent.")
+ parname = r.parents[0].name
+ if parname not in schemas:
+ raise RuntimeError(
+ "No schema for record type {} in schema dictionary.".format(parname))
+ try:
+ jsonschema.validate(convert_record(r), schemas[parname])
+ retval.append((True, None))
+ except ValidationError as ex:
+ retval.append((False, ex))
+ return retval
diff --git a/src/doc/cfood.rst b/src/doc/cfood.rst
index a42d593035bd37d0712986c958fb8ad7ad287968..0c7726d2017b955ecd7472d57dc259ff9a7bab53 100644
--- a/src/doc/cfood.rst
+++ b/src/doc/cfood.rst
@@ -207,9 +207,9 @@ following.
ValueWithUnitElt:
type: TextElement
match_name: ^my_prop$
- match_value: "^(?P<number>\\d+\\.?\\d*)\s+(?P<unit>.+)" # Extract value and unit from a string which
- # has a number followed by at least one whitespace
- # character followed by a unit.
+ match_value: "^(?P<number>\\d+\\.?\\d*)\\s+(?P<unit>.+)" # Extract value and unit from a string which
+ # has a number followed by at least one whitespace
+ # character followed by a unit.
records:
MyRecord:
MyProp:
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
index b3aa02a151a4d03c1531094ea01a5246cb02ba73..e1cbb10eff2c86034fc24a3fb0c73949e202df30 100644
--- a/src/doc/concepts.rst
+++ b/src/doc/concepts.rst
@@ -79,7 +79,7 @@ A Registered Identifiable is the blue print for Identifiables.
You can think of registered identifiables as identifiables without concrete values for properties.
RegisteredIdentifiables are
associated with RecordTypes and define of what information an identifiable for that RecordType
-exists. There can be multiple Registered Identifiables for one RecordType.
+exists. There cannot be multiple Registered Identifiables for one RecordType.
If identifiables shall contain references to the object to be identified, the Registered
Identifiable must list the RecordTypes of the Entities that have those references.
@@ -95,6 +95,37 @@ RecordType name in the configuration which will only require, that ANY Record
references the Record at hand.
+Instead of defining registered identifiables for a RecordType
+directly, they can be defined for their parents. I.e., if there is no
+registered identifiable for a RecordType, then it will be checked
+whether there is a parent that has one. If multiple recordtypes exist
+in the inheritance chain with a registered identifiable, then the one
+that is closest to the direct parent is used. In case of multiple
+inheritance, only one branch must have registered identifiables.
+
+The reason for this behavior is the following. If there were
+mutliple registered identifiables that could be used to identify a
+given record and only a single one of them would used, it might be
+that the existence check returns a different result than if the other
+one would be used. This would allow for unpredictable and inconsistent
+behavior (Example: one registered identifiable contains the name
+another one property date. Using the name might imply that the record
+does not exist and using the date might imply that it does. Thus, for
+any Record the registered identifiable must be unique). Analogous
+Example: If you think in the context of relational databases, there
+can always only be a foreign key associated with one table.
+
+.. note::
+
+ In case of using the registered identifiable of a parent, the
+ identifiable will be created by using the parent
+ RecordType. Example: The registered identifiable is defined for the
+ parent "Experiment" and the RecordType at hand "LaseExperiment" is
+ a child of "Experiment". Then the identifiable will construct a
+ query that searches for "Experiment" Records (and not
+ "LaseExperiment" Records).
+
+
Identified Records
++++++++++++++++++
TODO
diff --git a/src/doc/conf.py b/src/doc/conf.py
index f9cab5b71b5ef2cc1c751020d651f4c6b6d0a3e7..a1e9dbded97fe82fdee4d0df1e30a6fe46be6bae 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -33,10 +33,10 @@ copyright = '2024, IndiScale'
author = 'Alexander Schlemmer'
# The short X.Y version
-version = '0.10.1'
+version = '0.11.0'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.10.1'
+release = '0.11.0'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/converters/further_converters.rst b/src/doc/converters/further_converters.rst
index a334c8778f440e108fd141b0fc53ec06765deb8c..0fffc2e7de1bd23327194c6379cca94bd7c72a29 100644
--- a/src/doc/converters/further_converters.rst
+++ b/src/doc/converters/further_converters.rst
@@ -98,3 +98,90 @@ given ``recordname``, this record can be used within the cfood. Most
importantly, this record stores the internal path of this array within the HDF5
file in a text property, the name of which can be configured with the
``internal_path_property_name`` option which defaults to ``internal_hdf5_path``.
+
+
+
+ROCrateConverter
+================
+
+The ROCrateConverter unpacks ro-crate files, and creates one instance of the
+``ROCrateEntity`` structure element for each contained object. Currently only
+zipped ro-crate files are supported. The created ROCrateEntities wrap a
+``rocrate.model.entity.Entity`` with a path to the folder the ROCrate data
+is saved in. They are appended as children and can then be accessed via the
+subtree and treated using the :ref:`ROCrateEntityConverter`.
+
+To use the ROCrateConverter, you need to install the LinkAhead crawler with its
+optional ``rocrate`` dependency.
+
+ELNFileConverter
+----------------
+
+As .eln files are zipped ro-crate files, the ELNFileConverter works analogously
+to the ROCrateConverter and also creates ROCrateEntities for contained objects.
+
+ROCrateEntityConverter
+----------------------
+
+The ROCrateEntityConverter unpacks the ``rocrate.model.entity.Entity`` wrapped
+within a ROCrateEntity, and appends all properties, contained files, and parts
+as children. Properties are converted to a basic element matching their value
+(``BooleanElement``, ``IntegerElement``, etc.) and can be matched using
+match_properties. Each ``rocrate.model.file.File`` is converted to a crawler
+File object, which can be matched with SimpleFile. And each subpart of the
+ROCrateEntity is also converted to a ROCrateEntity, which can then again be
+treated using this converter.
+
+The ``match_entity_type`` keyword can be used to match a ROCrateEntity using its
+entity_type. With the ``match_properties`` keyword, properties of a ROCrateEntity
+can be either matched or extracted, as seen in the cfood example below:
+* with ``match_properties: "@id": ro-crate-metadata.json`` the ROCrateEntities
+can be filtered to only match the metadata json files.
+* with ``match_properties: dateCreated: (?P<dateCreated>.*)`` the ``dateCreated``
+entry of that metadata json file is extracted and accessible through the
+``dateCreated`` variable.
+* the example could then be extended to use any other entry present in the metadata
+json to filter the results, or insert the extracted information into generated records.
+
+Example cfood
+-------------
+
+One short cfood to generate records for each .eln file in a directory and
+their metadata files could be:
+
+.. code-block:: yaml
+
+ ---
+ metadata:
+ crawler-version: 0.9.0
+ ---
+ Converters:
+ ELNFile:
+ converter: ELNFileConverter
+ package: caoscrawler.converters.rocrate
+ ROCrateEntity:
+ converter: ROCrateEntityConverter
+ package: caoscrawler.converters.rocrate
+
+ ParentDirectory:
+ type: Directory
+ match: (.*)
+ subtree:
+ ELNFile:
+ type: ELNFile
+ match: (?P<filename>.*)\.eln
+ records:
+ ELNExampleRecord:
+ filename: $filename
+ subtree:
+ ROCrateEntity:
+ type: ROCrateEntity
+ match_properties:
+ "@id": ro-crate-metadata.json
+ dateCreated: (?P<dateCreated>.*)
+ records:
+ MDExampleRecord:
+ parent: $ELNFile
+ filename: ro-crate-metadata.json
+ time: $dateCreated
+
diff --git a/src/doc/converters/standard_converters.rst b/src/doc/converters/standard_converters.rst
index 586b84b48be78f1307298a11ad61a2448c3c3cd7..96b089f2252e44e764ae35ceea560bcdf06858c6 100644
--- a/src/doc/converters/standard_converters.rst
+++ b/src/doc/converters/standard_converters.rst
@@ -6,9 +6,17 @@ These are the standard converters that exist in a default installation. For wri
Directory Converter
===================
-The Directory Converter creates StructureElements for each File and Directory
-inside the current Directory. You can match a regular expression against the
-directory name using the 'match' key.
+
+The Directory Converter creates StructureElements for each File and
+Directory inside the current Directory. You can match a regular
+expression against the directory name using the 'match' key.
+
+With the optional ``match_newer_than_file`` key, a path to file
+containing only an ISO-formatted datetime string can be specified. If
+this is done, a directory will only match if it contains at least one
+file or directory that has been modified since that datetime. If the
+file doesn't exist or contains an invalid string, the directory will
+be matched regardless of the modification times.
Simple File Converter
=====================
@@ -41,6 +49,41 @@ The following StructureElement types are typically created by the DictElement co
Note that you may use ``TextElement`` for anything that exists in a text format that can be
interpreted by the server, such as date and datetime strings in ISO-8601 format.
+match_properties
+----------------
+
+`match_properties` is a dictionary of key-regexps and value-regexp pairs and can be used to
+match direct properties of a `DictElement`. Each key matches
+a property name and the corresponding value matches its property value.
+
+Example:
+........
+
+.. code-block:: json
+
+ {
+ "@type": "PropertyValue",
+ "additionalType": "str",
+ "propertyID": "testextra",
+ "value": "hi"
+ }
+
+When applied to a dict loaded from the above json, a `DictElementConverter` with the following definition:
+
+.. code-block:: yaml
+
+ Example:
+ type: DictElement
+ match_properties:
+ additionalType: (?P<addt>.*)$
+ property(.*): (?P<propid>.*)$
+
+will match and create two variables:
+
+- `addt = "str"`
+- `propid = "testextra"`
+
+
Scalar Value Converters
=======================
`BooleanElementConverter`, `FloatElementConverter`, `TextElementConverter`, and
@@ -318,7 +361,7 @@ The XMLTagConverter is a generic converter for XMLElements with the following ma
no converter implemented that can match XMLAttributeNodes.
Namespaces
-**********
+..........
The default is to take the namespace map from the current node and use
it in xpath queries. Because default namespaces cannot be handled by
@@ -331,3 +374,31 @@ XMLTextNodeConverter
In the future, this converter can be used to match XMLTextNodes that
are generated by the XMLTagConverter.
+
+
+ZipFileConverter
+================
+
+This converter opens zip files, unzips them into a temporary directory and
+exposes its contents as File structure elements.
+
+Usage Example:
+--------------
+
+.. code-block:: yaml
+
+ ExampleZipFile:
+ type: ZipFile
+ match: example\.zip$
+ subtree:
+ DirInsideZip:
+ type: Directory
+ match: experiments$
+ FileInsideZip:
+ type: File
+ match: description.odt$
+
+This converter will match and open files called ``example.zip``. If
+the file contains a directory called ``experiments`` it will be
+processed further by the respective converter in the subtree. The same
+is true for a file called ``description.odt``.
diff --git a/src/doc/converters/transform_functions.rst b/src/doc/converters/transform_functions.rst
index 22df35c8521ea0d70b2ebf7b7c8bc7c52e176bd3..35c11093714f59cf3139a2544b5eae2f5a9c17f2 100644
--- a/src/doc/converters/transform_functions.rst
+++ b/src/doc/converters/transform_functions.rst
@@ -38,10 +38,120 @@ An example that splits the variable ``a`` and puts the generated list in ``b`` i
Report:
tags: $b
-This splits the string in '$a' and stores the resulting list in '$b'. This is here used to add a
-list valued property to the Report Record.
+This splits the string in '$a' and stores the resulting list in
+'$b'. This is here used to add a list valued property to the Report
+Record. Note that from LinkAhead Crawler 0.11.0 onwards, the value of
+``marker`` in the above example can also be read in from a variable in
+the usual ``$`` notation:
+
+.. code-block:: yaml
+
+ # ... variable ``separator`` is defined somewhere above this part, e.g.,
+ # by reading a config file.
+ Experiment:
+ type: Dict
+ match: ".*"
+ transform:
+ param_split:
+ in: $a
+ out: $b
+ functions:
+ - split:
+ marker: $separator # Now the separator is read in from a
+ # variable, so we can, e.g., change from
+ # '|' to ';' without changing the cfood
+ # definition.
+ records:
+ Report:
+ tags: $b
+
There are a number of transform functions that are defined by default (see
``src/caoscrawler/default_transformers.yml``). You can define custom transform functions by adding
them to the cfood definition (see :doc:`CFood Documentation<../cfood>`).
+
+
+Custom Transformers
+===================
+
+Custom transformers are basically python functions having a special form/signature. They need to
+be registered in the cfood definition in order to be available during the scanning process.
+
+Let's assume we want to implement a transformer that replaces all occurrences of single letters
+in the value of a variable with a different letter each. So passing "abc" as `in_letters` and
+"xyz" as `out_letters` would result in a replacement of a value of "scan started" to
+"szxn stxrted". We could implement this in python using the
+following code:
+
+.. code-block:: python
+
+ def replace_letters(in_value: Any, in_parameters: dict) -> Any:
+ """
+ Replace letters in variables
+ """
+
+ # The arguments to the transformer (as given by the definition in the cfood)
+ # are contained in `in_parameters`. We need to make sure they are set or
+ # set their defaults otherwise:
+
+ if "in_letters" not in in_parameters:
+ raise RuntimeError("Parameter `in_letters` missing.")
+
+ if "out_letters" not in in_parameters:
+ raise RuntimeError("Parameter `out_letters` missing.")
+
+ l_in = in_parameters["in_letters"]
+ l_out = in_parameters["out_letters"]
+
+
+ if len(l_in) != len(l_out):
+ raise RuntimeError("`in_letters` and `out_letters` must have the same length.")
+
+ for l1, l2 in zip(l_in, l_out):
+ in_value = in_value.replace(l1, l2)
+
+ return in_value
+
+
+This code needs to be put into a module that can be found during runtime of the crawler.
+One possibility is to install the package into the same virtual environment that is used
+to run the crawler.
+
+In the cfood the transfomer needs to be registered:
+
+.. code-block:: yaml
+
+ ---
+ metadata:
+ crawler-version: 0.10.2
+ macros:
+ ---
+ #Converters: # put custom converters here
+ Transformers:
+ replace_letters: # This name will be made available in the cfood
+ function: replace_letters
+ package: utilities.replace_letters
+
+This would assume that the code for the function `replace_letters` is residing in a file
+called `replace_letters.py` that is stored in a package called `utilities`.
+
+The transformer can then be used in a converter, e.g.:
+
+
+.. code-block:: yaml
+
+ Experiment:
+ type: Dict
+ match: ".*"
+ transform:
+ replace_letters:
+ in: $a
+ out: $b
+ functions:
+ - replace_letters: # This is the name of our custom transformer
+ in_letters: "abc"
+ out_letters: "xyz"
+ records:
+ Report:
+ tags: $b
diff --git a/tox.ini b/tox.ini
index 1b695d26c56e1e251404700cfd04fb50a7dc0d8d..d44fbb6d50c58f44fe7944a2a49711b8def18cd6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,11 +6,8 @@ skip_missing_interpreters = true
deps = .[h5-crawler,spss,rocrate]
pytest
pytest-cov
- # TODO: Make this f-branch sensitive
- git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
- git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
- # TODO Remove once https://github.com/ResearchObject/ro-crate-py/issues/203 has been resolved.
- git+https://github.com/salexan2001/ro-crate-py.git@f-automatic-dummy-ids
+ linkahead
+ caosadvancedtools
commands = caosdb-crawler --help
py.test --cov=caoscrawler -vv {posargs}
diff --git a/unittests/datamodels/datamodel.yaml b/unittests/datamodels/datamodel.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2759ecba7f2967062937d9b2f4805a9b501ab6c4
--- /dev/null
+++ b/unittests/datamodels/datamodel.yaml
@@ -0,0 +1,6 @@
+Dataset:
+ obligatory_properties:
+ keywords:
+ datatype: TEXT
+ dateModified:
+ datatype: DATETIME
diff --git a/unittests/eln_cfood.yaml b/unittests/eln_cfood.yaml
index ab8e7108f511b0450d37c3e60162e412d4a1bf3b..bb29b7da7c1e6c3fc555038412f42ff2ab4d28fa 100644
--- a/unittests/eln_cfood.yaml
+++ b/unittests/eln_cfood.yaml
@@ -26,11 +26,18 @@ DataDir:
"@id": records-example/$
name: (?P<name>.*)
keywords: (?P<keywords>.*)
- description: (?P<description>.*)
dateModified: (?P<dateModified>.*)
records:
Dataset:
name: $name
keywords: $keywords
- description: $description
dateModified: $dateModified
+ subtree:
+ Description:
+ type: ROCrateEntity
+ match_type: TextObject
+ match_properties:
+ text: (?P<description>.*)
+ records:
+ Dataset:
+ description: $description
diff --git a/unittests/eln_files/PASTA.eln b/unittests/eln_files/PASTA.eln
deleted file mode 100644
index 61866e7d5f57cb32191af6663be230153092e712..0000000000000000000000000000000000000000
Binary files a/unittests/eln_files/PASTA.eln and /dev/null differ
diff --git a/unittests/eln_files/records-example.eln b/unittests/eln_files/records-example.eln
index 09ed53fc179e80a240ab773247d6f9adee71b429..4907bcc4e88e2152fdf2675a50ca661b666c947d 100644
Binary files a/unittests/eln_files/records-example.eln and b/unittests/eln_files/records-example.eln differ
diff --git a/unittests/test_cfood_metadata.py b/unittests/test_cfood_metadata.py
index c606a0a1afcc15d48164694768bae02adfb0fc0b..b123f98584ba99ed4fec412732cb2bf536034a91 100644
--- a/unittests/test_cfood_metadata.py
+++ b/unittests/test_cfood_metadata.py
@@ -18,7 +18,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
from tempfile import NamedTemporaryFile
-from unittest.mock import MagicMock, Mock, patch
+from unittest.mock import patch
import pytest
import yaml
@@ -33,7 +33,7 @@ def _temp_file_load(txt: str):
definition using load_definition from Crawler.
"""
definition = None
- with NamedTemporaryFile() as f:
+ with NamedTemporaryFile(delete=False) as f:
f.write(txt.encode())
f.flush()
definition = load_definition(f.name)
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index 6c7db6ed346fc5e6d0d286024e96ef8828c5c872..e4b442d91060c7ba98cb1a910156b1800f050be3 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -29,12 +29,14 @@ import importlib
import json
import logging
import os
+import pytest
+import yaml
+
from itertools import product
from pathlib import Path
+from tempfile import NamedTemporaryFile
import linkahead as db
-import pytest
-import yaml
from caoscrawler.converters import (Converter, ConverterValidationError,
DateElementConverter, DictElementConverter,
@@ -1021,3 +1023,109 @@ def test_properties_from_dict_nested(converter_registry):
# The "old" DictConverter should have added the additional property:
assert myrec.get_property("additional_from_other") is not None
assert myrec.get_property("additional_from_other").value == "other"
+
+
+def test_dict_match_properties(converter_registry):
+
+ root_dict_element = DictElement("RootDict", {
+ "prop_a": "value",
+ "prop_b": "25",
+ "prop_c": 24
+ })
+
+ def_dict = {
+ "RootElt": {
+ # Root dictionary
+ "type": "DictElement",
+ "match_properties": {
+ "prop_a": "(?P<a>.*)$",
+ "prop_[^ac]": "(?P<b>.*)$",
+ "prop_c": "(?P<c>.*)$",
+ },
+ "records": {
+ # Define top-level, use below in subtrees
+ "MyRec": {
+ "prop_a": "$a",
+ "prop_b": "$b",
+ "$a": "$c"
+ }
+ }}}
+ records = scan_structure_elements(root_dict_element, def_dict, converter_registry)
+ assert len(records) == 1
+ record = records[0]
+ assert record.get_property("prop_a").value == "value"
+ assert record.get_property("prop_b").value == "25"
+ assert record.get_property("value").value == "24" # Note the type change here
+
+ root_dict_element = DictElement("RootDict", {
+ "prop_a": "value",
+ "prop_b": "25",
+ # Property missing
+ })
+
+ records = scan_structure_elements(root_dict_element, def_dict, converter_registry)
+ assert len(records) == 0
+
+ with pytest.raises(RuntimeError, match="Multiple properties match the same match_properties entry."):
+ root_dict_element = DictElement("RootDict", {
+ "prop_a": "value",
+ "prop_b": "25",
+ "prop_d": 24 # duplicate matches
+ })
+ records = scan_structure_elements(root_dict_element, def_dict, converter_registry)
+
+
+def test_directory_converter_change_date(caplog, converter_registry):
+ """Test that only directories that were modified after a certain
+ date are crawled.
+
+ """
+ test_dir_element = Directory("test_directories", UNITTESTDIR / "test_directories")
+ date_of_dir_change = DirectoryConverter._get_most_recent_change_in_dir(test_dir_element)
+ past_date = date_of_dir_change - datetime.timedelta(days=1)
+ future_date = date_of_dir_change + datetime.timedelta(days=1)
+
+ tmpfi = NamedTemporaryFile(delete=False)
+
+ # Write down past
+ with open(tmpfi.name, "w") as fi:
+ fi.write(f"{past_date.isoformat()}\n")
+
+ converter_def = {
+ "type": "Directory",
+ "match": "^test_directories$",
+ "match_newer_than_file": tmpfi.name
+ }
+ dc = DirectoryConverter(name="DC1", definition=converter_def,
+ converter_registry=converter_registry)
+ assert dc.match(test_dir_element) is not None
+
+ # Write down future, so nothing should match
+ with open(tmpfi.name, "w") as fi:
+ fi.write(f"{future_date.isoformat()}\n")
+ assert dc.match(test_dir_element) is None
+
+ # Also match in the corner case of equality:
+ with open(tmpfi.name, "w") as fi:
+ fi.write(f"{date_of_dir_change.isoformat()}\n")
+ assert dc.match(test_dir_element) is not None
+
+ # Match but warn
+ with open(tmpfi.name, "w") as fi:
+ fi.write(f"This is garbage.\n")
+ with pytest.raises(ValueError):
+ dc.match(test_dir_element)
+ assert len(caplog.record_tuples) == 1
+ assert caplog.record_tuples[0][1] == logging.ERROR
+ assert tmpfi.name in caplog.record_tuples[0][2]
+ assert "doesn't contain a ISO formatted datetime in its first line" in caplog.record_tuples[0][2]
+
+ # Match anything since file doesn't exist, inform in debug log.
+ os.remove(tmpfi.name)
+ # Clear log and enforce debug level.
+ caplog.clear()
+ caplog.set_level(logging.DEBUG)
+ assert dc.match(test_dir_element) is not None
+ assert len(caplog.record_tuples) == 1
+ assert caplog.record_tuples[0][1] == logging.DEBUG
+ assert "Reference file doesn't exist." == caplog.record_tuples[0][2]
diff --git a/unittests/test_crawler.py b/unittests/test_crawler.py
index e88ce454061fb268fa49e986f8392f71296beb07..bdb22ba2171c6d52633c4429d98735e560cf6375 100644
--- a/unittests/test_crawler.py
+++ b/unittests/test_crawler.py
@@ -372,6 +372,8 @@ def test_split_into_inserts_and_updates_with_circ(crawler_mocked_identifiable_re
crawler._split_into_inserts_and_updates(st)
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable_retrieve):
crawler = crawler_mocked_identifiable_retrieve
# A
@@ -400,6 +402,8 @@ def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable
# TODO write test where the unresoled entity is not part of the identifiable
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.crawl.cached_get_entity_by",
new=Mock(side_effect=mock_get_entity_by))
@patch("caoscrawler.identifiable_adapters.cached_query",
@@ -583,6 +587,8 @@ def test_split_into_inserts_and_updates_diff_backref(crawler_mocked_for_backref_
assert len(insert) == 1
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
def test_replace_entities_with_ids():
crawler = Crawler()
a = (db.Record().add_parent("B").add_property("A", 12345)
@@ -595,6 +601,8 @@ def test_replace_entities_with_ids():
assert a.get_property("C").value == [12345, 233324]
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.crawl.cached_get_entity_by",
new=Mock(side_effect=mock_get_entity_by))
@patch("caoscrawler.identifiable_adapters.cached_get_entity_by",
@@ -620,6 +628,8 @@ def test_synchronization_no_commit(upmock, insmock):
assert len(ups) == 1
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.crawl.cached_get_entity_by",
new=Mock(side_effect=mock_get_entity_by))
@patch("caoscrawler.identifiable_adapters.cached_get_entity_by",
@@ -824,9 +834,9 @@ def test_restricted_path(create_mock):
def test_split_restricted_path():
- assert ["el"] == split_restricted_path("/el")
- assert ["el"] == split_restricted_path("/el/")
- assert ["el", "el"] == split_restricted_path("/el/el")
+ assert ["el"] == split_restricted_path(os.path.sep + "el")
+ assert ["el"] == split_restricted_path(os.path.sep + "el" + os.path.sep)
+ assert ["el", "el"] == split_restricted_path(os.path.sep + "el" + os.path.sep + "el")
# Filter the warning because we want to have it here and this way it does not hinder running
diff --git a/unittests/test_directories/examples_tables/ExperimentalData/test_with_empty.csv b/unittests/test_directories/examples_tables/ExperimentalData/test_with_empty.csv
new file mode 100644
index 0000000000000000000000000000000000000000..be25239a6d96ecde3876a7bbabdae8769994b455
--- /dev/null
+++ b/unittests/test_directories/examples_tables/ExperimentalData/test_with_empty.csv
@@ -0,0 +1,4 @@
+event,date
+event_a,2025-02-06
+event_b,
+event_c,2025-02-06T09:00:00
diff --git a/unittests/test_directories/examples_tables/crawler_for_issue_112.yml b/unittests/test_directories/examples_tables/crawler_for_issue_112.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4bab5adabcf889d7784583a80dcbb94b714fd3fc
--- /dev/null
+++ b/unittests/test_directories/examples_tables/crawler_for_issue_112.yml
@@ -0,0 +1,27 @@
+ExperimentalData:
+ type: Directory
+ match: ExperimentalData
+ subtree:
+ CSVTable:
+ type: CSVTableConverter
+ match: "test_with_empty\\.csv"
+ subtree:
+ Row:
+ type: DictElement
+ records:
+ Event:
+ subtree:
+ EventName:
+ type: TextElement
+ match_name: "event"
+ match_value: "(?P<name>.*)"
+ records:
+ Event:
+ name: $name
+ Date:
+ type: Datetime
+ match_name: "date"
+ match_value: "(?P<date>.+)"
+ records:
+ Event:
+ event_time: $date
diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py
index bdc0ab850d1a8253e876e8b1a6bc621327802f79..5108e83c83db16f1b44d836bf22d21d8e871ee8f 100644
--- a/unittests/test_identifiable_adapters.py
+++ b/unittests/test_identifiable_adapters.py
@@ -44,6 +44,20 @@ from caoscrawler.sync_graph import SyncNode
UNITTESTDIR = Path(__file__).parent
+def mock_retrieve_RecordType(id, name):
+ return {
+ "Person": db.RecordType(name="Person"),
+ "Keyword": db.RecordType(name="Keyword"),
+ "Project": db.RecordType(name="Project"),
+ "A": db.RecordType(name="A"),
+ "Experiment": db.RecordType(name="Experiment"),
+ "Lab": db.RecordType(name="Lab"),
+ "Analysis": db.RecordType(name="Analysis"),
+ "MetaAnalysis": db.RecordType(name="MetaAnalysis").add_parent("Analysis"),
+ "Measurement": db.RecordType(name="Measurement").add_parent("Experiment")
+ }[name]
+
+
def test_create_query_for_identifiable():
query = IdentifiableAdapter.create_query_for_identifiable(
Identifiable(record_type="Person", properties={"first_name": "A", "last_name": "B"}))
@@ -100,6 +114,8 @@ def test_create_query_for_identifiable():
assert query == ("FIND RECORD 'record type' WITH name='it\\'s weird'")
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=mock_retrieve_RecordType))
def test_load_from_yaml_file():
ident = CaosDBIdentifiableAdapter()
ident.load_from_yaml_definition(
@@ -175,6 +191,8 @@ def test_convert_value():
assert convert_value(A()) == " a "
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=mock_retrieve_RecordType))
def test_get_identifiable():
ident = CaosDBIdentifiableAdapter()
ident.load_from_yaml_definition(UNITTESTDIR / "example_identifiables.yml")
@@ -283,3 +301,40 @@ def test_referencing_entity_has_appropriate_type():
assert rft(dummy.parents, registered_identifiable)
registered_identifiable.properties[0].value = ["B", "*"]
assert rft(dummy.parents, registered_identifiable)
+
+
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=mock_retrieve_RecordType))
+def test_get_registered_identifiable():
+ # Test the case that the record has a parent for which an identifiable is registered
+ ident = CaosDBIdentifiableAdapter()
+ ident.load_from_yaml_definition(UNITTESTDIR / "example_identifiables.yml")
+ rec = db.Record().add_parent(name="Experiment")
+ registered = ident.get_registered_identifiable(rec)
+ assert registered is not None
+ assert registered.parents[0].name == "Experiment"
+
+ # Test the same but with an additional parent
+ rec = db.Record().add_parent(name="Lab").add_parent(name="Experiment")
+ registered = ident.get_registered_identifiable(rec)
+ assert registered is not None
+ assert registered.parents[0].name == "Experiment"
+
+ # Test the same but with an additional parent that also has a registered identifiable
+ rec = db.Record().add_parent(name="Analysis").add_parent(name="Experiment")
+ with pytest.raises(RuntimeError):
+ registered = ident.get_registered_identifiable(rec)
+
+ # Test the same but with an additional parent that has a parent with a registered identifiable
+ rec = db.Record().add_parent(name="MetaAnalysis").add_parent(name="Experiment")
+ with pytest.raises(RuntimeError):
+ registered = ident.get_registered_identifiable(rec)
+
+ # Test the case that the record has a parent for which no identifiable is registered
+ # and there is a registered identifiable for a grand parent
+ ident = CaosDBIdentifiableAdapter()
+ ident.load_from_yaml_definition(UNITTESTDIR / "example_identifiables.yml")
+ rec = db.Record().add_parent(name="Measurement")
+ registered = ident.get_registered_identifiable(rec)
+ assert registered is not None
+ assert registered.parents[0].name == "Experiment"
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
index a6de65400f42018c3fdcde7b2f29d4fd200bf62b..779f77711fe18df2433f03580e7e3e4f2035f0f4 100644
--- a/unittests/test_issues.py
+++ b/unittests/test_issues.py
@@ -20,14 +20,44 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-from pytest import mark
+import importlib
-from caoscrawler.converters import CrawlerTemplate, replace_variables
+from pathlib import Path
+from pytest import fixture, mark
+
+from caoscrawler.converters import (CrawlerTemplate, replace_variables, TextElementConverter)
from caoscrawler.crawl import Crawler
-from caoscrawler.scanner import (create_converter_registry,
+from caoscrawler.scanner import (create_converter_registry, scan_directory,
scan_structure_elements)
from caoscrawler.stores import GeneralStore
-from caoscrawler.structure_elements import DictElement
+from caoscrawler.structure_elements import DictElement, TextElement
+
+
+UNITTESTDIR = Path(__file__).parent
+
+
+@fixture
+def converter_registry():
+ converter_registry: dict[str, dict[str, str]] = {
+ "TextElement": {
+ "converter": "TextElementConverter",
+ "package": "caoscrawler.converters"},
+ "Directory": {
+ "converter": "DirectoryConverter",
+ "package": "caoscrawler.converters"},
+ "CSVTableConverter": {
+ "converter": "CSVTableConverter",
+ "package": "caoscrawler.converters"},
+ "Datetime": {
+ "converter": "DatetimeElementConverter",
+ "package": "caoscrawler.converters"
+ }
+ }
+
+ for key, value in converter_registry.items():
+ module = importlib.import_module(value["package"])
+ value["class"] = getattr(module, value["converter"])
+ return converter_registry
def test_issue_10():
@@ -148,3 +178,49 @@ def test_issue_93():
propvalue_template = CrawlerTemplate(propvalue)
assert (propvalue_template.safe_substitute(**values.get_storage())
== f"some text before >> This is {exp} << some text after")
+
+
+def test_issue_112(converter_registry):
+ """Test that empty table cells are not matched in case of
+ ``match_value: ".+"``.
+
+ See https://gitlab.com/linkahead/linkahead-crawler/-/issues/112.
+
+ """
+ tec = TextElementConverter(
+ name="TestTextConverter",
+ definition={
+ "match_name": ".*",
+ "match_value": "(?P<content>.+)"
+ },
+ converter_registry=converter_registry
+ )
+
+ empty = TextElement(name="empty", value='')
+ assert tec.match(empty) is None
+
+ empty_none = TextElement(name="empty", value=None)
+ assert tec.match(empty_none) is None
+
+ non_empty = TextElement(name="empty", value=' ')
+ matches = tec.match(non_empty)
+ assert "content" in matches
+ assert matches["content"] == ' '
+
+ # Cfood definition for CSV example file
+ records = scan_directory(UNITTESTDIR / "test_directories" / "examples_tables" / "ExperimentalData",
+ UNITTESTDIR / "test_directories" / "examples_tables" / "crawler_for_issue_112.yml")
+ assert records
+ for rec in records:
+ print(rec.name)
+ assert len(rec.parents.filter_by_identity(name="Event")) > 0
+ assert rec.name in ["event_a", "event_b", "event_c"]
+ if rec.name == "event_a":
+ assert rec.get_property("event_time") is not None
+ assert rec.get_property("event_time").value == "2025-02-06"
+ if rec.name == "event_b":
+ # `date` field is empty, so there must be no match
+ assert rec.get_property("event_time") is None
+ if rec.name == "event_c":
+ assert rec.get_property("event_time") is not None
+ assert rec.get_property("event_time").value == "2025-02-06T09:00:00"
diff --git a/unittests/test_macros.py b/unittests/test_macros.py
index a87b633e8585a03431575426733cae6ba31b7acf..03fe0e665652bb12e204d76857771c1d064ec28a 100644
--- a/unittests/test_macros.py
+++ b/unittests/test_macros.py
@@ -50,10 +50,10 @@ def _temp_file_load(txt: str):
definition using load_definition from Crawler.
"""
definition = None
- with NamedTemporaryFile() as f:
+ with NamedTemporaryFile(delete=False) as f:
f.write(txt.encode())
f.flush()
- definition = load_definition(f.name)
+ definition = load_definition(f.name)
return definition
diff --git a/unittests/test_rocrate_converter.py b/unittests/test_rocrate_converter.py
index ef59a37c7a9ca91f85d3a62b4f5b6f5c12559575..4b6bde171c789017e95a38729ae93f49ecf3f97b 100644
--- a/unittests/test_rocrate_converter.py
+++ b/unittests/test_rocrate_converter.py
@@ -27,22 +27,16 @@ import importlib
import os
from pathlib import Path
-import jsonschema
import linkahead as db
import pytest
import rocrate
import yaml
-from linkahead.high_level_api import convert_to_python_object
-from lxml.etree import fromstring
-from rocrate.model.entity import Entity
-from rocrate.rocrate import ROCrate
-
from caoscrawler import scanner
from caoscrawler.converters import ELNFileConverter, ROCrateEntityConverter
-from caoscrawler.scanner import load_definition
from caoscrawler.stores import GeneralStore
from caoscrawler.structure_elements import (DictElement, File, ROCrateEntity,
TextElement)
+from rocrate.model.entity import Entity
UNITTESTDIR = Path(__file__).parent
@@ -82,6 +76,12 @@ def eln_entities(basic_eln_converter):
return entities
+@pytest.mark.xfail(
+ reason="The example files for PASTA have not yet been updated in:"
+ "https://github.com/TheELNConsortium/TheELNFileFormat/tree/master/examples/PASTA"
+ "However, there was the announcement that these files are going to follow the"
+ "flattened structure soon: https://github.com/TheELNConsortium/TheELNFileFormat/issues/98"
+)
def test_load_pasta(basic_eln_converter):
"""
Test for loading the .eln example export from PASTA.
@@ -105,7 +105,7 @@ def test_load_kadi4mat(basic_eln_converter):
match = basic_eln_converter.match(f_k4mat)
assert match is not None
entities = basic_eln_converter.create_children(GeneralStore(), f_k4mat)
- assert len(entities) == 10
+ assert len(entities) == 17
assert isinstance(entities[0], ROCrateEntity)
assert isinstance(entities[0].folder, str)
assert isinstance(entities[0].entity, Entity)
@@ -137,15 +137,15 @@ match_properties:
match = ds2.match(eln_entities[1])
assert match is not None
- assert match["dateCreated"] == "2024-08-21T12:07:45.115990+00:00"
+ assert match["dateCreated"] == "2024-11-19T13:44:35.476888+00:00"
children = ds2.create_children(GeneralStore(), eln_entities[1])
assert len(children) == 8
assert isinstance(children[0], TextElement)
assert children[0].name == "@id"
assert children[0].value == "ro-crate-metadata.json"
- assert isinstance(children[5], DictElement)
- assert children[5].value == {'@id': 'https://kadi.iam.kit.edu'}
+ assert isinstance(children[5], ROCrateEntity)
+ assert children[5].name == "https://kadi.iam.kit.edu"
def test_file(eln_entities):
@@ -184,13 +184,20 @@ match_properties:
assert match is not None
children = ds_parts.create_children(GeneralStore(), ent_parts)
+ # Number of children = number of properties + number of parts +
+ # number of variables measured + number of files
+ assert len(children) == (len(ent_parts.entity.properties()) +
+ len(ent_parts.entity.properties()["hasPart"]) +
+ len(ent_parts.entity.properties()["variableMeasured"]))
- # Number of children = number of properties + number of parts:
- assert len(children) == len(ent_parts.entity.properties()) + 4
entity_children = [f for f in children if isinstance(f, ROCrateEntity)]
- assert len(entity_children) == 4
+ assert len(entity_children) == 13
+ file_counter = 0
+
for f in entity_children:
- assert isinstance(f.entity, rocrate.model.file.File)
+ if isinstance(f.entity, rocrate.model.file.File):
+ file_counter += 1
+ assert file_counter == 4
def test_scanner():
@@ -199,7 +206,14 @@ def test_scanner():
assert len(rlist) == 1
assert isinstance(rlist[0], db.Record)
assert rlist[0].name == "records-example"
- assert rlist[0].description == "This is a sample record."
+ # This assertion was moved to a different test, see below:
+ # assert rlist[0].description == "This is a sample record."
assert rlist[0].parents[0].name == "Dataset"
assert rlist[0].get_property("keywords").value == "sample"
assert rlist[0].get_property("dateModified").value == "2024-08-21T11:43:17.626965+00:00"
+
+
+def test_description_reference():
+ rlist = scanner.scan_directory(os.path.join(UNITTESTDIR, "eln_files/"),
+ os.path.join(UNITTESTDIR, "eln_cfood.yaml"))
+ assert rlist[0].description == "This is a sample record."
diff --git a/unittests/test_scanner.py b/unittests/test_scanner.py
index 5cbbc63406ffb3f5ec1f9019ed7877d7880d7b69..c531f66fd38a714ba4f6f538d41c9fbaeb364d44 100644
--- a/unittests/test_scanner.py
+++ b/unittests/test_scanner.py
@@ -30,7 +30,7 @@ from functools import partial
from pathlib import Path
from tempfile import NamedTemporaryFile
from unittest.mock import MagicMock, Mock, patch
-
+import os
import linkahead as db
import pytest
import yaml
@@ -110,7 +110,7 @@ def test_record_structure_generation():
assert len(subc[1]) == 0
# The data analysis node creates one variable for the node itself:
- assert subd[0]["DataAnalysis"] == "examples_article/DataAnalysis"
+ assert subd[0]["DataAnalysis"] == os.path.join("examples_article", "DataAnalysis")
assert subc[0]["DataAnalysis"] is False
subd = dbt.debug_tree[dircheckstr("DataAnalysis", "2020_climate-model-predict")]
@@ -128,9 +128,10 @@ def test_record_structure_generation():
assert subd[0]["identifier"] == "climate-model-predict"
assert subd[0]["Project"].__class__ == db.Record
- assert subd[0]["DataAnalysis"] == "examples_article/DataAnalysis"
+ assert subd[0]["DataAnalysis"] == os.path.join("examples_article", "DataAnalysis")
assert subc[0]["DataAnalysis"] is True
- assert subd[0]["project_dir"] == "examples_article/DataAnalysis/2020_climate-model-predict"
+ assert subd[0]["project_dir"] == os.path.join(
+ "examples_article", "DataAnalysis", "2020_climate-model-predict")
assert subc[0]["project_dir"] is False
# Check the copy flags for the first level in the hierarchy:
@@ -405,3 +406,92 @@ def test_units():
assert rec.get_property("may_be_overwritten") is not None
assert rec.get_property("may_be_overwritten").value == "400"
assert rec.get_property("may_be_overwritten").unit == "°C"
+
+
+def test_recursive_definition():
+ """
+ This is basically a test for:
+ https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/16
+ """
+
+ recursive_yaml = """
+Converter:
+ type: DictElement
+ records:
+ Block:
+ Experiment: $Experiment
+ Experiment:
+ Block: $Block
+ """
+
+ crawler_definition = _load_definition_from_yaml_dict(
+ [yaml.load(recursive_yaml, Loader=yaml.SafeLoader)])
+ converter_registry = create_converter_registry(crawler_definition)
+
+ data = {
+ "value_with_unit": "1.1 m",
+ "array_with_units": [
+ "1.1 cm",
+ "2.2 cm"
+ ]
+ }
+ records = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
+ converter_registry)
+
+ assert len(records) == 2
+ assert len(records[0].parents) == 1
+ assert records[0].parents[0].name == "Block"
+ assert len(records[1].parents) == 1
+ assert records[1].parents[0].name == "Experiment"
+
+ assert records[0].get_property("Experiment").value == records[1]
+ assert records[1].get_property("Block").value == records[0]
+
+
+def test_recursive_definition_2():
+ """
+ This is another a test for:
+ https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/16
+
+ It defines Experiment on a different level, therefore allowing the recursive definition.
+ This is, however, no workaround for test_recursive_definition as a bidirectional link on the
+ same level is still not achieved.
+ """
+
+ recursive_yaml = """
+FirstConverter:
+ type: DictElement
+ records:
+ Experiment:
+ subtree:
+ Converter:
+ type: DictElement
+ records:
+ Block:
+ Experiment: $Experiment
+ Experiment:
+ Block: $Block
+ """
+
+ crawler_definition = _load_definition_from_yaml_dict(
+ [yaml.load(recursive_yaml, Loader=yaml.SafeLoader)])
+ converter_registry = create_converter_registry(crawler_definition)
+
+ data = {"data": {
+ "value_with_unit": "1.1 m",
+ "array_with_units": [
+ "1.1 cm",
+ "2.2 cm"
+ ]
+ }}
+ records = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
+ converter_registry)
+
+ assert len(records) == 2
+ assert len(records[0].parents) == 1
+ assert records[0].parents[0].name == "Block"
+ assert len(records[1].parents) == 1
+ assert records[1].parents[0].name == "Experiment"
+
+ assert records[0].get_property("Experiment").value == records[1]
+ assert records[1].get_property("Block").value == records[0]
diff --git a/unittests/test_scripts.py b/unittests/test_scripts.py
new file mode 100644
index 0000000000000000000000000000000000000000..da03c1f24fbd3d7ca13cfa55d6f69c0cb5a6a6f1
--- /dev/null
+++ b/unittests/test_scripts.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# This file is a part of the LinkAhead project.
+#
+# Copyright (C) 2024 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Test if the scripts work as expected.
+"""
+
+from subprocess import run
+
+SCRIPTS = [
+ "linkahead-crawler",
+ "caosdb-crawler",
+ "spss_to_datamodel",
+ "csv_to_datamodel",
+]
+
+
+def test_script_loading():
+ """Run the scripts with "-h"."""
+ for script in SCRIPTS:
+ run([script, "-h"], check=True)
diff --git a/unittests/test_sync_graph.py b/unittests/test_sync_graph.py
index 06f0dfb9eb3d3536d26dcfd354ca27f08ef99a02..030306b95578865cdbfe19bdef2998a573848bd5 100644
--- a/unittests/test_sync_graph.py
+++ b/unittests/test_sync_graph.py
@@ -140,6 +140,8 @@ def test_create_reference_mapping():
assert backward_references_backref[id(b)] == set()
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.sync_graph.cached_get_entity_by",
new=Mock(side_effect=mock_get_entity_by))
def test_SyncGraph_init():
@@ -204,6 +206,8 @@ def test_SyncGraph_init():
assert el.identifiable is not None or el.id is not None
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
new=Mock(side_effect=lambda x: [x]))
def test_merge_into_trivial(simple_adapter):
@@ -282,6 +286,8 @@ def test_merge_into_trivial(simple_adapter):
assert se_b in st.backward_references_backref[id(se_c)]
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
new=Mock(side_effect=lambda x: [x]))
def test_merge_into_simple(simple_adapter):
@@ -363,6 +369,8 @@ def test_merge_into_simple(simple_adapter):
se_b in st.backward_references_backref[id(se_c)]
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
new=Mock(side_effect=lambda x: [x]))
def test_backward_references_backref():
@@ -381,6 +389,8 @@ def test_backward_references_backref():
assert st.nodes[1] in st.backward_references_backref[id(st.nodes[0])]
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
new=Mock(side_effect=lambda x: [x]))
def test_set_id_of_node(simple_adapter):
@@ -484,6 +494,8 @@ def test_set_id_of_node(simple_adapter):
assert len(st.nodes) == 2
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.sync_graph.cached_get_entity_by",
new=Mock(side_effect=mock_get_entity_by))
def test_merging(simple_adapter):
@@ -569,6 +581,8 @@ def test_merging(simple_adapter):
assert len(st.unchecked) == 0
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
def test_update_of_reference_values(simple_adapter):
# multiple nodes are merged including one that is referenced
# assure that this still leads to the value of the property of the referencing node to be
@@ -592,6 +606,8 @@ def test_update_of_reference_values(simple_adapter):
assert b_prop.id == 101
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
def test_ignoring_irrelevant_references(simple_adapter):
# make sure that a circle of references is no problem if one references is not identifying
b = db.Record(name='b').add_parent("RT5")
@@ -653,6 +669,8 @@ def test_set_each_scalar_value():
assert a.properties[0].value is None
+@patch("caoscrawler.identifiable_adapters._retrieve_RecordType",
+ new=Mock(side_effect=lambda id, name: db.RecordType(id=id, name=name)))
@patch("caoscrawler.identifiable_adapters.cached_query",
new=Mock(side_effect=mock_cached_only_rt_allow_empty))
def test_merge_referenced_by():
diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py
index 0571dbd31de9b37230f0ee1d93c22c6df47c87e7..a2d227adc5b0c6a8f2f96cb054e1c7670e980e10 100644
--- a/unittests/test_transformers.py
+++ b/unittests/test_transformers.py
@@ -30,17 +30,16 @@ See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107
import importlib
from pathlib import Path
-from unittest.mock import MagicMock, Mock, patch
+from unittest.mock import Mock
-import linkahead as db
import pytest
-import yaml
-from pytest import raises
-
from caoscrawler.converters import Converter, ListElementConverter
from caoscrawler.scanner import create_transformer_registry, scan_directory
from caoscrawler.stores import GeneralStore
-from caoscrawler.transformer_functions import replace, split
+from caoscrawler.transformer_functions import (cast_to_bool, cast_to_float,
+ cast_to_int, cast_to_str,
+ replace, split)
+from pytest import raises
UNITTESTDIR = Path(__file__).parent
@@ -163,3 +162,55 @@ def test_empty_functions_list(converter_registry):
conv.apply_transformers(values, transformer_functions)
assert values['b'] == "16_45"
+
+
+def test_cast_transformer_functions():
+ for val in ("True", "true", "False", "false"):
+ assert type(cast_to_bool(val, {})) == bool
+ if val[1] == "r":
+ assert cast_to_bool(val, {}) is True
+ else:
+ assert cast_to_bool(val, {}) is False
+ for val_err in ("jaksdlfj", "0", 1):
+ with pytest.raises(ValueError):
+ cast_to_bool(val_err, {})
+ assert cast_to_bool(False, {}) is False
+ assert cast_to_bool(True, {}) is True
+
+ assert cast_to_int("24", {}) == 24
+ assert cast_to_int(24.0, {}) == 24
+ assert cast_to_int(24, {}) == 24
+ assert cast_to_int("-24", {}) == -24
+ with pytest.raises(ValueError):
+ cast_to_int("24dsf", {})
+ with pytest.raises(ValueError):
+ cast_to_int("24.0", {}) == 24
+
+ assert cast_to_float("24", {}) == 24.0
+ assert cast_to_float("24.0", {}) == 24.0
+ assert cast_to_float(24.0, {}) == 24.0
+ assert cast_to_float(24, {}) == 24.0
+ with pytest.raises(ValueError):
+ cast_to_float("24dsf", {})
+
+ assert cast_to_str(24, {}) == "24"
+
+
+def test_replace_variables():
+ vals = GeneralStore()
+ vals["test"] = "with"
+ vals["a"] = "str_without_replacement"
+ conv = Mock()
+ conv.definition = {}
+ conv.definition["transform"] = {
+ "test": {
+ "in": "$a",
+ "out": "$a",
+ "functions": [
+ {"replace": {
+ "remove": "without",
+ "insert": "$test"
+ }}
+ ]}}
+ Converter.apply_transformers(conv, vals, {"replace": replace})
+ assert vals["a"] == "str_with_replacement"
diff --git a/unittests/test_utilities.py b/unittests/test_utilities.py
index 463e304a99161f2294e5d202611dcf0b829e2045..a9b052524957b6f8c1e0378e3153fc06f4f36806 100644
--- a/unittests/test_utilities.py
+++ b/unittests/test_utilities.py
@@ -20,22 +20,23 @@
#
import pytest
-
+from os.path import sep
from caoscrawler.crawl import split_restricted_path
from caoscrawler.utils import MissingImport, get_shared_resource_link
def test_split_restricted_path():
assert split_restricted_path("") == []
- assert split_restricted_path("/") == []
- assert split_restricted_path("test/") == ["test"]
- assert split_restricted_path("/test/") == ["test"]
- assert split_restricted_path("test/bla") == ["test", "bla"]
- assert split_restricted_path("/test/bla") == ["test", "bla"]
- assert split_restricted_path("/test1/test2/bla") == ["test1", "test2", "bla"]
- assert split_restricted_path("/test//bla") == ["test", "bla"]
- assert split_restricted_path("//test/bla") == ["test", "bla"]
- assert split_restricted_path("///test//bla////") == ["test", "bla"]
+ assert split_restricted_path(f"{sep}") == []
+ assert split_restricted_path(f"test{sep}") == ["test"]
+ assert split_restricted_path(f"{sep}test{sep}") == ["test"]
+ assert split_restricted_path(f"test{sep}bla") == ["test", "bla"]
+ assert split_restricted_path(f"{sep}test{sep}bla") == ["test", "bla"]
+ assert split_restricted_path(f"{sep}test1{sep}test2{sep}bla") == ["test1", "test2", "bla"]
+ assert split_restricted_path(f"{sep}test{sep}{sep}bla") == ["test", "bla"]
+ assert split_restricted_path(f"{sep}{sep}test{sep}bla") == ["test", "bla"]
+ assert split_restricted_path(
+ f"{sep}{sep}{sep}test{sep}{sep}bla{sep}{sep}{sep}{sep}") == ["test", "bla"]
def test_dummy_class():
diff --git a/unittests/test_validation.py b/unittests/test_validation.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3215963f67b61241b321a0eb7345f9fe6fde1f2
--- /dev/null
+++ b/unittests/test_validation.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Alexander Schlemmer <a.schlemmer@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+test validation
+"""
+from os.path import join
+from pathlib import Path
+
+import jsonschema
+import linkahead as db
+import pytest
+from caoscrawler.validator import (convert_record,
+ load_json_schema_from_datamodel_yaml,
+ validate)
+from jsonschema import ValidationError
+
+UNITTESTDIR = Path(__file__).parent
+
+
+def test_create_json_schema():
+ json = load_json_schema_from_datamodel_yaml(join(UNITTESTDIR, "datamodels", "datamodel.yaml"))
+ r = db.Record()
+ r.add_parent(name="Dataset")
+ r.add_property(name="keywords", value="jakdlfjakdf")
+ r.add_property(name="dateModified", value="2024-11-16")
+
+ pobj = convert_record(r)
+ # print(yaml.dump(pobj))
+ # print(yaml.dump(json[0]))
+ assert "Dataset" in json
+ jsonschema.validate(pobj, json["Dataset"])
+
+ # Failing test:
+ r = db.Record()
+ r.add_parent(name="Dataset")
+ r.add_property(name="keywordss", value="jakdlfjakdf")
+ r.add_property(name="dateModified", value="2024-11-16")
+
+ pobj = convert_record(r)
+
+ with pytest.raises(ValidationError, match=".*'keywords' is a required property.*"):
+ jsonschema.validate(pobj, json["Dataset"])
+
+
+def test_validation():
+ """
+ Test for the main validation API function `validate`
+ """
+ json = load_json_schema_from_datamodel_yaml(
+ join(UNITTESTDIR, "datamodels", "datamodel.yaml"))
+ r1 = db.Record()
+ r1.add_parent(name="Dataset")
+ r1.add_property(name="keywords", value="jakdlfjakdf")
+ r1.add_property(name="dateModified", value="2024-11-16")
+
+ r2 = db.Record()
+ r2.add_parent(name="Dataset")
+ r2.add_property(name="keywordss", value="jakdlfjakdf")
+ r2.add_property(name="dateModified", value="2024-11-16")
+
+ valres = validate([r1, r2], json)
+ assert valres[0][0] is True
+ assert valres[0][1] is None
+ assert not valres[1][0]
+ assert isinstance(valres[1][1], ValidationError)
+ assert valres[1][1].message == "'keywords' is a required property"
diff --git a/unittests/test_zipfile_converter.py b/unittests/test_zipfile_converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..451d23c93bfc15889d5b7a9f97ef1f157aece6ee
--- /dev/null
+++ b/unittests/test_zipfile_converter.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Alexander Schlemmer <a.schlemmer@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+test the zip-file converter
+"""
+import importlib
+import os
+from pathlib import Path
+
+import pytest
+import yaml
+from caoscrawler.converters import DirectoryConverter, ZipFileConverter
+from caoscrawler.stores import GeneralStore
+from caoscrawler.structure_elements import Directory, File
+
+UNITTESTDIR = Path(__file__).parent
+
+
+@pytest.fixture
+def converter_registry():
+ converter_registry: dict[str, dict[str, str]] = {
+ "ZipFile": {
+ "converter": "ZipFileConverter",
+ "package": "caoscrawler.converters"},
+ }
+
+ for key, value in converter_registry.items():
+ module = importlib.import_module(value["package"])
+ value["class"] = getattr(module, value["converter"])
+ return converter_registry
+
+
+@pytest.mark.xfail(
+ reason="The example files for PASTA have not yet been updated in:"
+ "https://github.com/TheELNConsortium/TheELNFileFormat/tree/master/examples/PASTA"
+ "However, there was the announcement that these files are going to follow the"
+ "flattened structure soon: https://github.com/TheELNConsortium/TheELNFileFormat/issues/98"
+)
+def test_zipfile_converter(converter_registry):
+ zipfile = File("PASTA.eln", os.path.join(UNITTESTDIR, "eln_files", "PASTA.eln"))
+ zip_conv = ZipFileConverter(yaml.safe_load("""
+type: ZipFile
+match: .*$
+"""), "TestZipFileConverter", converter_registry)
+
+ match = zip_conv.match(zipfile)
+ assert match is not None
+
+ children = zip_conv.create_children(GeneralStore(), zipfile)
+ assert len(children) == 1
+ assert children[0].name == "PASTA"
+
+ dir_conv = DirectoryConverter(yaml.safe_load("""
+type: Directory
+match: ^PASTA$
+"""), "TestDirectory", converter_registry)
+ match = dir_conv.match(children[0])
+ assert match is not None
+ children = dir_conv.create_children(GeneralStore(), children[0])
+ assert len(children) == 5
+ print(children)
+ for i in range(2):
+ assert isinstance(children[i], Directory)
+ for i in range(2, 5):
+ assert isinstance(children[i], File)
+
+
+def test_zipfile_minimal(converter_registry):
+ zipfile = File("empty.zip", os.path.join(UNITTESTDIR, "zip_minimal", "empty.zip"))
+ zip_conv = ZipFileConverter(yaml.safe_load("""
+type: ZipFile
+match: .*$
+"""), "TestZipFileConverter", converter_registry)
+
+ match = zip_conv.match(zipfile)
+ assert match is not None
+
+ children = zip_conv.create_children(GeneralStore(), zipfile)
+ assert len(children) == 2
+
+ file_obj = None
+ dir_obj = None
+ for ch in children:
+ if isinstance(ch, File):
+ file_obj = ch
+ elif isinstance(ch, Directory):
+ dir_obj = ch
+ else:
+ assert False
+ assert file_obj is not None and dir_obj is not None
+ assert file_obj.name == "empty.txt"
+
+ dir_conv = DirectoryConverter(yaml.safe_load("""
+type: Directory
+match: ^folder$
+"""), "TestDirectory", converter_registry)
+ match = dir_conv.match(dir_obj)
+ assert match is not None
+ children = dir_conv.create_children(GeneralStore(), dir_obj)
+ assert len(children) == 3
+ for i in range(3):
+ assert isinstance(children[i], File)
diff --git a/unittests/zip_minimal/empty.zip b/unittests/zip_minimal/empty.zip
new file mode 100644
index 0000000000000000000000000000000000000000..3eb2cee755e1b0265b13b1ee8f31c2aa1abe62de
Binary files /dev/null and b/unittests/zip_minimal/empty.zip differ