diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 62c41db72e668cd555a1bcac9151c6b522fe5791..e43223568252b2e7a1504610692fe20dc9d78348 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,27 +121,21 @@ unittest_py3.11:
- python3 -c "import sys; assert sys.version.startswith('3.11')"
- tox
-unittest_py3.8:
+unittest_py3.9:
tags: [cached-dind]
stage: test
- image: python:3.8
+ image: python:3.9
script: &python_test_script
# install dependencies
- pip install pytest pytest-cov
# TODO: Use f-branch logic here
- pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
- pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
- - pip install .[h5-crawler,spss]
+ - pip install .[h5-crawler,spss,rocrate]
# actual test
- caosdb-crawler --help
- pytest --cov=caosdb -vv ./unittests
-unittest_py3.9:
- tags: [cached-dind]
- stage: test
- image: python:3.9
- script: *python_test_script
-
unittest_py3.10:
tags: [cached-dind]
stage: test
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 923e941960d5725674240a8196e63e01a0241c1d..3b55a8fedf6b8ffc8907728fae4fa96da07e810d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
unit: m
```
- Support for Python 3.13
+- ROCrateConverter, ELNFileConverter and ROCrateEntityConverter for crawling ROCrate and .eln files
- `max_log_level` parameter to `logging.configure_server_side_logging`
to control the server-side debuglog's verboosity, and an optional
`sss_max_log_level` parameter to `crawler_main` to control the SSS
@@ -42,6 +43,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
+* Support for Python 3.8 (end of life)
+
### Fixed ###
- Added better error message for some cases of broken converter and
diff --git a/setup.cfg b/setup.cfg
index 558599013f3556a41481305ba587e3947a403d63..4e2056bdb23da2ba734a1aeda60cfc5e6a6f3e64 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,7 +17,7 @@ classifiers =
package_dir =
= src
packages = find:
-python_requires = >=3.8
+python_requires = >=3.9
install_requires =
caosadvancedtools >= 0.7.0
importlib-resources
@@ -49,3 +49,5 @@ h5-crawler =
numpy
spss =
pandas[spss]
+rocrate =
+ rocrate @ git+https://github.com/salexan2001/ro-crate-py.git@f-automatic-dummy-ids
diff --git a/src/caoscrawler/converters/__init__.py b/src/caoscrawler/converters/__init__.py
index 540a4cfca9ff19248baab2bc0fe8d10987d4bd1f..70fca6c44c90a3bbb44bd05c34e51cafae91a229 100644
--- a/src/caoscrawler/converters/__init__.py
+++ b/src/caoscrawler/converters/__init__.py
@@ -30,3 +30,18 @@ except ImportError as err:
SPSSConverter: type = utils.MissingImport(
name="SPSSConverter", hint="Try installing with the `spss` extra option.",
err=err)
+
+try:
+ from .rocrate import ROCrateEntityConverter
+ from .rocrate import ROCrateConverter
+ from .rocrate import ELNFileConverter
+except ImportError as err:
+ ROCrateEntityConverter: type = utils.MissingImport(
+ name="ROCrateEntityConverter", hint="Try installing with the `rocrate` extra option.",
+ err=err)
+ ROCrateConverter: type = utils.MissingImport(
+ name="ROCrateConverter", hint="Try installing with the `rocrate` extra option.",
+ err=err)
+ ELNFileConverter: type = utils.MissingImport(
+ name="ELNFileConverter", hint="Try installing with the `rocrate` extra option.",
+ err=err)
diff --git a/src/caoscrawler/converters/converters.py b/src/caoscrawler/converters/converters.py
index 22686e0dbae26e3322059928cf3ba0b4522f672c..64a557ce4e26fd8bfd345000d3abf18bf0360117 100644
--- a/src/caoscrawler/converters/converters.py
+++ b/src/caoscrawler/converters/converters.py
@@ -400,6 +400,15 @@ class Converter(object, metaclass=ABCMeta):
self.converters.append(Converter.converter_factory(
converter_definition, converter_name, converter_registry))
+ self.setup()
+
+ def setup(self):
+ """
+ Analogous to `cleanup`. Can be used to set up variables that are permanently
+ stored in this converter.
+ """
+ pass
+
@staticmethod
def converter_factory(definition: dict, name: str, converter_registry: dict):
"""Create a Converter instance of the appropriate class.
@@ -619,6 +628,13 @@ class Converter(object, metaclass=ABCMeta):
"""
pass
+ def cleanup(self):
+ """
+ This function is called when the converter runs out of scope and can be used to
+ clean up objects that were needed in the converter or its children.
+ """
+ pass
+
class DirectoryConverter(Converter):
"""
diff --git a/src/caoscrawler/converters/rocrate.py b/src/caoscrawler/converters/rocrate.py
new file mode 100644
index 0000000000000000000000000000000000000000..286061ef6dbe9c7caf851fe32932dee848ac55d4
--- /dev/null
+++ b/src/caoscrawler/converters/rocrate.py
@@ -0,0 +1,225 @@
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Converters take structure elements and create Records and new structure elements from them.
+
+This converter converts ro-crate files which may also be .eln-files.
+
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import rocrate
+from rocrate.rocrate import ROCrate
+
+import linkahead as db
+
+from .converters import SimpleFileConverter, ConverterValidationError, Converter, convert_basic_element
+from ..stores import GeneralStore, RecordStore
+from ..structure_elements import (File, Directory, StructureElement, ROCrateEntity)
+
+from zipfile import ZipFile
+
+import tempfile
+import os
+import re
+
+
+class ROCrateConverter(SimpleFileConverter):
+
+ """Convert ro-crate files / directories.
+ """
+
+ def setup(self):
+ self._tempdir = None
+
+ def cleanup(self):
+ self._tempdir.cleanup()
+
+ def typecheck(self, element: StructureElement):
+ """
+ Check whether the current structure element can be converted using
+ this converter.
+ """
+ return isinstance(element, File) or isinstance(element, Directory)
+
+ def match(self, element: StructureElement) -> Optional[dict]:
+ m = re.match(self.definition["match"], element.name)
+ if m is None:
+ return None
+ return m.groupdict()
+
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
+ """
+ Loads an ROCrate from an rocrate file or directory.
+
+ Arguments:
+ ----------
+ element must be a File or Directory (structure element).
+
+ Returns:
+ --------
+ A list with an ROCrateElement representing the contents of the .eln-file or None
+ in case of errors.
+ """
+
+ if isinstance(element, File):
+ self._tempdir = tempfile.TemporaryDirectory()
+ with ZipFile(element.path) as zipf:
+ zipf.extractall(self._tempdir.name)
+ crate_path = self._tempdir.name
+ crate = ROCrate(crate_path)
+ entity_ls = []
+ for ent in crate.get_entities():
+ entity_ls.append(ROCrateEntity(crate_path, ent))
+ return entity_ls
+ elif isinstance(element, Directory):
+ # This would be an unzipped .eln file
+ # As this is possible for rocrate files, I think it is reasonable
+ # to support it as well.
+ raise NotImplementedError()
+ else:
+ raise ValueError("create_children was called with wrong type of StructureElement")
+ return None
+
+
+class ELNFileConverter(ROCrateConverter):
+
+ """Convert .eln-Files
+ See: https://github.com/TheELNConsortium/TheELNFileFormat
+
+ These files are basically RO-Crates with some minor differences:
+ - The ro-crate metadata file is not on top-level within the .eln-zip-container,
+ but in a top-level subdirectory.
+ """
+
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
+ """
+ Loads an ROCrate from an .eln-file or directory.
+
+ This involves unzipping the .eln-file to a temporary folder and creating an ROCrate object
+ from its contents.
+
+ Arguments:
+ ----------
+ element must be a File or Directory (structure element).
+
+ Returns:
+ --------
+ A list with an ROCrateElement representing the contents of the .eln-file or None
+ in case of errors.
+ """
+
+ if isinstance(element, File):
+ self._tempdir = tempfile.TemporaryDirectory()
+ with ZipFile(element.path) as zipf:
+ zipf.extractall(self._tempdir.name)
+ cratep = os.listdir(self._tempdir.name)
+ if len(cratep) != 1:
+ raise RuntimeError(".eln file must contain exactly one folder")
+ crate_path = os.path.join(self._tempdir.name, cratep[0])
+ crate = ROCrate(crate_path)
+ entity_ls = []
+ for ent in crate.get_entities():
+ entity_ls.append(ROCrateEntity(crate_path, ent))
+ return entity_ls
+ elif isinstance(element, Directory):
+ # This would be an unzipped .eln file
+ # As this is possible for rocrate files, I think it is reasonable
+ # to support it as well.
+ raise NotImplementedError()
+ else:
+ raise ValueError("create_children was called with wrong type of StructureElement")
+ return None
+
+
+class ROCrateEntityConverter(Converter):
+
+ def typecheck(self, element: StructureElement):
+ """
+ Check whether the current structure element can be converted using
+ this converter.
+ """
+ return isinstance(element, ROCrateEntity)
+
+ def match(self, element: StructureElement) -> Optional[dict]:
+ # See https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/145
+ # for a suggestion for the design of the matching algorithm.
+ if not isinstance(element, ROCrateEntity):
+ raise TypeError("Element must be an instance of ROCrateEntity.")
+
+ # Store the result of all individual regexp variable results:
+ vardict = {}
+
+ if "match_entity_type" in self.definition:
+ m_type = re.match(self.definition["match_entity_type"], element.type)
+ if m_type is None:
+ return None
+ vardict.update(m_type.groupdict())
+
+ if "match_properties" in self.definition:
+ # This matcher works analogously to the attributes matcher in the XMLConverter
+ for prop_def_key, prop_def_value in self.definition["match_properties"].items():
+ match_counter = 0
+ matched_m_prop = None
+ matched_m_prop_value = None
+ for prop_key, prop_value in element.entity.properties().items():
+ m_prop = re.match(prop_def_key, prop_key)
+ if m_prop is not None:
+ match_counter += 1
+ matched_m_prop = m_prop
+ m_prop_value = re.match(prop_def_value, prop_value)
+ if m_prop_value is None:
+ return None
+ matched_m_prop_value = m_prop_value
+ if match_counter == 0:
+ return None
+ elif match_counter > 1:
+ raise RuntimeError("Multiple properties match the same match_prop entry.")
+ vardict.update(matched_m_prop.groupdict())
+ vardict.update(matched_m_prop_value.groupdict())
+
+ return vardict
+
+ def create_children(self, generalStore: GeneralStore, element: StructureElement):
+
+ children = []
+
+ eprops = element.entity.properties()
+
+ # Add the properties:
+ for name, value in eprops.items():
+ children.append(convert_basic_element(value, name))
+
+ # Add the files:
+ if isinstance(element.entity, rocrate.model.file.File):
+ path, name = os.path.split(eprops["@id"])
+ children.append(File(name, os.path.join(element.folder, path, name)))
+
+ # Parts of this entity are added as child entities:
+ if "hasPart" in eprops:
+ for p in eprops["hasPart"]:
+ children.append(
+ ROCrateEntity(element.folder, element.entity.crate.dereference(
+ p["@id"])))
+
+ return children
diff --git a/src/caoscrawler/converters/xml_converter.py b/src/caoscrawler/converters/xml_converter.py
index 0f25c0c0947421f0561c42318ac0abddabb447fc..bd3f6cf0fdcc5fed5b5452da8a17a8a877009b06 100644
--- a/src/caoscrawler/converters/xml_converter.py
+++ b/src/caoscrawler/converters/xml_converter.py
@@ -183,6 +183,7 @@ class XMLTagConverter(Converter):
# - Require unique attribute-key and attribute-value matches: Very complex
# - Only allow one single attribute-key to match and run attribute-value match separately.
# Currently the latter option is implemented.
+ # TODO: The ROCrateEntityConverter implements a very similar behavior.
if match_counter == 0:
return None
elif match_counter > 1:
diff --git a/src/caoscrawler/scanner.py b/src/caoscrawler/scanner.py
index eeb2bdbf8f0f0d96579598cd8842739a3d154b93..27711e6a7c4e69df3c2d99aca7a427670b153765 100644
--- a/src/caoscrawler/scanner.py
+++ b/src/caoscrawler/scanner.py
@@ -400,6 +400,9 @@ def scanner(items: list[StructureElement],
crawled_data, debug_tree,
registered_transformer_functions)
+ # Clean up converter:
+ converter.cleanup()
+
if restricted_path and not path_found:
raise RuntimeError("A 'restricted_path' argument was given that is not contained in "
"the data tree")
diff --git a/src/caoscrawler/structure_elements/__init__.py b/src/caoscrawler/structure_elements/__init__.py
index 4b925a567f87febdac7b5547111a468eb0a3253c..351f1069708ec94c0dd27313b6329d89858d4330 100644
--- a/src/caoscrawler/structure_elements/__init__.py
+++ b/src/caoscrawler/structure_elements/__init__.py
@@ -20,4 +20,12 @@
"""Submdule containing all default and optional converters."""
+from .. import utils
from .structure_elements import *
+
+try:
+ from .rocrate_structure_elements import ROCrateEntity
+except ImportError as err:
+ ROCrateEntity: type = utils.MissingImport(
+ name="ROCrateEntity", hint="Try installing with the `rocrate` extra option.",
+ err=err)
diff --git a/src/caoscrawler/structure_elements/rocrate_structure_elements.py b/src/caoscrawler/structure_elements/rocrate_structure_elements.py
new file mode 100644
index 0000000000000000000000000000000000000000..d39617432fcb63220d3acbb63a618b0445165388
--- /dev/null
+++ b/src/caoscrawler/structure_elements/rocrate_structure_elements.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2024 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+from rocrate.model.entity import Entity
+from .structure_elements import StructureElement
+
+
+class ROCrateEntity(StructureElement):
+ """
+ Store entities contained in ROCrates.
+ """
+
+ def __init__(self, folder: str, entity: Entity):
+ """
+ Initializes this ROCrateEntity.
+
+ Arguments:
+ ----------
+ folder: str
+ The folder that contains the ROCrate data. In case of a zipped ROCrate, this
+ is a temporary folder that the ROCrate was unzipped to.
+ The folder is the folder containing the ro-crate-metadata.json.
+
+ entity: Entity
+ The ROCrate entity that is stored in this structure element.
+ The entity automatically contains an attribute ".crate"
+ that stores the ROCrate that this entity belongs to. It can be used
+ e.g. to look up links to other entities (ROCrate.dereference).
+ """
+ super().__init__(entity.properties()["@id"])
+ self.folder = folder
+ self.entity = entity
diff --git a/tox.ini b/tox.ini
index 41249e4277391c5ffa4ec13fc4da1a6ee1f48491..e003e26ecd16861c3b8a8d991fc789c78d203e5b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,7 +3,7 @@ envlist = py38, py39, py310, py311, py312, py313
skip_missing_interpreters = true
[testenv]
-deps = .[h5-crawler,spss]
+deps = .[h5-crawler,spss,rocrate]
pytest
pytest-cov
# TODO: Make this f-branch sensitive
diff --git a/unittests/eln_cfood.yaml b/unittests/eln_cfood.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ab8e7108f511b0450d37c3e60162e412d4a1bf3b
--- /dev/null
+++ b/unittests/eln_cfood.yaml
@@ -0,0 +1,36 @@
+---
+metadata:
+ crawler-version: 0.9.2
+ macros:
+---
+Converters:
+ ELNFile:
+ converter: ELNFileConverter
+ package: caoscrawler.converters
+ ROCrateEntity:
+ converter: ROCrateEntityConverter
+ package: caoscrawler.converters
+
+DataDir:
+ type: Directory
+ match: .*
+ subtree:
+ ELNFile:
+ type: ELNFile
+ match: ^.*\.eln$
+ subtree:
+ RecordsExample:
+ type: ROCrateEntity
+ match_type: Dataset
+ match_properties:
+ "@id": records-example/$
+ name: (?P<name>.*)
+ keywords: (?P<keywords>.*)
+ description: (?P<description>.*)
+ dateModified: (?P<dateModified>.*)
+ records:
+ Dataset:
+ name: $name
+ keywords: $keywords
+ description: $description
+ dateModified: $dateModified
diff --git a/unittests/eln_files/PASTA.eln b/unittests/eln_files/PASTA.eln
new file mode 100644
index 0000000000000000000000000000000000000000..61866e7d5f57cb32191af6663be230153092e712
Binary files /dev/null and b/unittests/eln_files/PASTA.eln differ
diff --git a/unittests/eln_files/records-example.eln b/unittests/eln_files/records-example.eln
new file mode 100644
index 0000000000000000000000000000000000000000..09ed53fc179e80a240ab773247d6f9adee71b429
Binary files /dev/null and b/unittests/eln_files/records-example.eln differ
diff --git a/unittests/test_rocrate_converter.py b/unittests/test_rocrate_converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..16cfc3a3c3aec811219da7006d4722d9abf6dcf7
--- /dev/null
+++ b/unittests/test_rocrate_converter.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Alexander Schlemmer <a.schlemmer@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+test the XML converters
+"""
+import importlib
+import json
+import pytest
+import sys
+import yaml
+import os
+
+from lxml.etree import fromstring
+from pathlib import Path
+
+from rocrate.rocrate import ROCrate
+from rocrate.model.entity import Entity
+import rocrate
+
+from caoscrawler.converters import (ELNFileConverter, ROCrateEntityConverter)
+from caoscrawler.scanner import load_definition
+from caoscrawler.stores import GeneralStore
+from caoscrawler.structure_elements import ROCrateEntity, File, TextElement, DictElement
+
+from caoscrawler import scanner
+from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema
+from caosadvancedtools.models.parser import parse_model_from_yaml
+from linkahead.high_level_api import convert_to_python_object
+import json
+import jsonschema
+
+import linkahead as db
+
+
+UNITTESTDIR = Path(__file__).parent
+
+
+@pytest.fixture
+def converter_registry():
+ converter_registry: dict[str, dict[str, str]] = {
+ "ELNFile": {
+ "converter": "ELNFileConverter",
+ "package": "caoscrawler.converters"},
+ "ROCrateEntity": {
+ "converter": "ROCrateEntityConverter",
+ "package": "caoscrawler.converters",
+ }
+ }
+
+ for key, value in converter_registry.items():
+ module = importlib.import_module(value["package"])
+ value["class"] = getattr(module, value["converter"])
+ return converter_registry
+
+
+@pytest.fixture
+def basic_eln_converter(converter_registry):
+ return ELNFileConverter(yaml.safe_load("""
+type: ELNFile
+match: .*\\.eln
+"""), "TestELNConverter", converter_registry)
+
+
+@pytest.fixture
+def eln_entities(basic_eln_converter):
+ f_k4mat = File("records-example.eln",
+ os.path.join(UNITTESTDIR, "eln_files", "records-example.eln"))
+ store = GeneralStore()
+ entities = basic_eln_converter.create_children(store, f_k4mat)
+ return entities
+
+
+def test_load_pasta(basic_eln_converter):
+ """
+ Test for loading the .eln example export from PASTA.
+ """
+ f_pasta = File("PASTA.eln", os.path.join(UNITTESTDIR, "eln_files", "PASTA.eln"))
+ match = basic_eln_converter.match(f_pasta)
+ assert match is not None
+ entities = basic_eln_converter.create_children(GeneralStore(), f_pasta)
+ assert len(entities) == 20
+ assert isinstance(entities[0], ROCrateEntity)
+ assert isinstance(entities[0].folder, str)
+ assert isinstance(entities[0].entity, Entity)
+
+
+def test_load_kadi4mat(basic_eln_converter):
+ """
+ Test for loading the .eln example export from PASTA.
+ """
+ f_k4mat = File("records-example.eln",
+ os.path.join(UNITTESTDIR, "eln_files", "records-example.eln"))
+ match = basic_eln_converter.match(f_k4mat)
+ assert match is not None
+ entities = basic_eln_converter.create_children(GeneralStore(), f_k4mat)
+ assert len(entities) == 10
+ assert isinstance(entities[0], ROCrateEntity)
+ assert isinstance(entities[0].folder, str)
+ assert isinstance(entities[0].entity, Entity)
+
+
+def test_match_rocrate_entities(eln_entities):
+ ds1 = ROCrateEntityConverter(yaml.safe_load("""
+type: ROCrateEntity
+match_properties:
+ "@id": \\./
+ datePublished: (?P<datePublished>.*)
+"""), "TestELNConverter", converter_registry)
+
+ match = ds1.match(eln_entities[0])
+ assert match is not None
+
+ ds2 = ROCrateEntityConverter(yaml.safe_load("""
+type: ROCrateEntity
+match_type: CreativeWork
+match_properties:
+ "@id": ro-crate-metadata.json
+ dateCreated: (?P<dateCreated>.*)
+"""), "TestELNConverter", converter_registry)
+
+ match = ds2.match(eln_entities[0])
+ assert match is None
+ match = ds1.match(eln_entities[1])
+ assert match is None
+
+ match = ds2.match(eln_entities[1])
+ assert match is not None
+ assert match["dateCreated"] == "2024-08-21T12:07:45.115990+00:00"
+
+ children = ds2.create_children(GeneralStore(), eln_entities[1])
+ assert len(children) == 8
+ assert isinstance(children[0], TextElement)
+ assert children[0].name == "@id"
+ assert children[0].value == "ro-crate-metadata.json"
+ assert isinstance(children[5], DictElement)
+ assert children[5].value == {'@id': 'https://kadi.iam.kit.edu'}
+
+
+def test_file(eln_entities):
+ ds_csv = ROCrateEntityConverter(yaml.safe_load("""
+type: ROCrateEntity
+match_type: File
+match_properties:
+ "@id": .*\.csv$
+"""), "TestELNConverter", converter_registry)
+
+ ent_csv = eln_entities[5]
+ match = ds_csv.match(ent_csv)
+ assert match is not None
+
+ children = ds_csv.create_children(GeneralStore(), ent_csv)
+
+ # Number of children = number of properties + number of files:
+ assert len(children) == len(ent_csv.entity.properties()) + 1
+ # Get the file:
+ f_csv = [f for f in children if isinstance(f, File)][0]
+ with open(f_csv.path) as f:
+ text = f.read()
+ assert "Ultrasound Transducer" in text
+
+
+def test_has_part(eln_entities):
+ ds_parts = ROCrateEntityConverter(yaml.safe_load("""
+type: ROCrateEntity
+match_type: Dataset
+match_properties:
+ "@id": records-example/
+"""), "TestELNConverter", converter_registry)
+
+ ent_parts = eln_entities[2]
+ match = ds_parts.match(ent_parts)
+ assert match is not None
+
+ children = ds_parts.create_children(GeneralStore(), ent_parts)
+
+ # Number of children = number of properties + number of parts:
+ assert len(children) == len(ent_parts.entity.properties()) + 4
+ entity_children = [f for f in children if isinstance(f, ROCrateEntity)]
+ assert len(entity_children) == 4
+ for f in entity_children:
+ assert isinstance(f.entity, rocrate.model.file.File)
+
+
+def test_scanner():
+ rlist = scanner.scan_directory(os.path.join(UNITTESTDIR, "eln_files/"),
+ os.path.join(UNITTESTDIR, "eln_cfood.yaml"))
+ assert len(rlist) == 1
+ assert isinstance(rlist[0], db.Record)
+ assert rlist[0].name == "records-example"
+ assert rlist[0].description == "This is a sample record."
+ assert rlist[0].parents[0].name == "Dataset"
+ assert rlist[0].get_property("keywords").value == "sample"
+ assert rlist[0].get_property("dateModified").value == "2024-08-21T11:43:17.626965+00:00"