diff --git a/CHANGELOG.md b/CHANGELOG.md index 001bf62a262ee970f7dc0de93b09cf8dea14507e..2db8e113fefdefeeaa4817ec61292feff86dc832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -* Fixed #12 +* FIX: #12 * FIX: Variables are now also replaced when the value is given as a list. ++ FIX: #35 Parent cannot be set from value ### Security diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py index e3f72b10ce2694853d6bc0644c736f0d621ed881..48c3c947b0f5480a8d64f3d895c80b455c66bb3a 100644 --- a/src/caoscrawler/converters.py +++ b/src/caoscrawler/converters.py @@ -67,6 +67,18 @@ class ConverterValidationError(Exception): def replace_variables(propvalue, values: GeneralStore): + """ + This function replaces variables in property values (and possibly other locations, + where the crawler can replace cfood-internal variables). + + This function checks whether the value that is to be replaced is of type db.Entity. + In this case the entity is returned (note that this is of course only possible, if the + occurrence of the variable is directly at the beginning of the value and e.g. no string + concatenation is attempted. + + In any other case the variable substitution is carried out and a new string with the + replaced variables is returned. + """ # Check if the replacement is a single variable containing a record: match = re.match(r"^\$(\{)?(?P<varname>[0-9a-zA-Z_]+)(\})?$", propvalue) if match is not None: @@ -213,8 +225,10 @@ def create_records(values: GeneralStore, # parents will be added when they aren't present in the record yet: if "parents" in record: for parent in record["parents"]: - if not has_parent(c_record, parent): - c_record.add_parent(parent) + # Do the variables replacement: + var_replaced_parent = replace_variables(parent, values) + if not has_parent(c_record, var_replaced_parent): + c_record.add_parent(var_replaced_parent) else: # add the "fallback" parent only for Records, not for Files: if role == "Record": diff --git a/unittests/test_directories/example_substitutions/substitutions_parents.yml b/unittests/test_directories/example_substitutions/substitutions_parents.yml new file mode 100644 index 0000000000000000000000000000000000000000..107e766ccd833fab618cecfc04f13bc29abc80a6 --- /dev/null +++ b/unittests/test_directories/example_substitutions/substitutions_parents.yml @@ -0,0 +1,25 @@ + +ExperimentalData: # name of the converter + type: Directory + match: ExperimentalData + records: + Project: + name: project + subtree: + File: # name of the converter + type: SimpleFile + match: (?P<year>[0-9]{2,2})(?P<month>[0-9]{2,2})(?P<day>[0-9]{2,2})_data.dat + records: + Experiment: + parents: + - Experiment + - Month_$month # This adds a special parent as record type + date: 20$year-$month-$day + + ExperimentSeries: + Experiment: $Experiment + + Project: + Experiments: +$Experiment + dates: +20$year-$month-$day + diff --git a/unittests/test_validation.py b/unittests/test_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..686c66f72f55b66344322e0c6f3b9d1a2b76b3f9 --- /dev/null +++ b/unittests/test_validation.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 Alexander Schlemmer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +""" +Test the validation of cfood definition files. +""" + +from caoscrawler.crawl import Crawler + +from tempfile import NamedTemporaryFile + +import yaml +import pytest diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py index 071bf4646d20e35ed05dafaf5fabf786dc182dcc..1e4aa21b3c5c848a27051633b5f8f214d1ca8cd5 100644 --- a/unittests/test_variable_substitutions.py +++ b/unittests/test_variable_substitutions.py @@ -39,6 +39,14 @@ def crawler(): rfp("test_directories", "example_substitutions", "substitutions.yml")) return crawler +@pytest.fixture +def crawler_2(): + crawler = Crawler(debug=True) + crawler.crawl_directory(rfp("test_directories", "example_substitutions", "ExperimentalData"), + rfp("test_directories", "example_substitutions", + "substitutions_parents.yml")) + return crawler + def test_substitutions(crawler): # @review Florian Spreckelsen 2022-05-13 @@ -59,3 +67,17 @@ def test_substitutions(crawler): assert isinstance(subd[i]["Project"].get_property("dates").value, list) assert subd[i]["Project"].get_property( "dates").value[0] == "2022-05-12" + +def test_substitutions_parents(crawler_2): + # This is a test for: + # https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/35 + # ... testing whether variable substitutions can be used in parent declarations. + subd = crawler_2.debug_tree[dircheckstr( + "File", "ExperimentalData", "220512_data.dat")] + # subd[0] <- generalStore + # subd[1] <- recordStore + + parents = subd[1]["Experiment"].get_parents() + assert len(parents) == 2 + assert parents[0].name == "Experiment" + assert parents[1].name == "Month_05"