diff --git a/CHANGELOG.md b/CHANGELOG.md index e401aec7c87b06a6d656b7c26de6ca432a568668..118eebe00ffae7941fdb9b03efbcbc641639ed1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * FIX: #35 Parent cannot be set from value * [#6](https://gitlab.com/caosdb/caosdb-crawler/-/issues/6): Fixed many type hints to be compatible to python 3.8 +* [#9](https://gitlab.com/caosdb/caosdb-crawler/-/issues/9): Sclaras of types + different than string can now be given in cfood definitions ### Security diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py index 97b81cd2e28faf2310e84abd5bb98aba9b60b308..f316eba6096356511192005d5628ae4657a07454 100644 --- a/src/caoscrawler/converters.py +++ b/src/caoscrawler/converters.py @@ -156,15 +156,21 @@ def handle_value(value: Union[dict, str, list], values: GeneralStore): propvalue = value # variables replacement: - propvalue = [replace_variables(i, values) for i in propvalue] + propvalue = list() + for element in value: + # Do the element-wise replacement only, when its type is string: + if type(element) == str: + propvalue.append(replace_variables(element, values)) + else: + propvalue.append(element) return (propvalue, collection_mode) else: # value is another simple type - # collection_mode = "single" - # propvalue = value["value"] - # return (propvalue, collection_mode) - raise RuntimeError() + collection_mode = "single" + propvalue = value + # Return it immediately, otherwise variable substitution would be done and fail: + return (propvalue, collection_mode) propvalue = replace_variables(propvalue, values) return (propvalue, collection_mode) diff --git a/unittests/cfoods_scalar.yml b/unittests/cfoods_scalar.yml new file mode 100644 index 0000000000000000000000000000000000000000..d0a728c35c27e331114cc5c18ebcfd1aa0905e31 --- /dev/null +++ b/unittests/cfoods_scalar.yml @@ -0,0 +1,14 @@ +# This is a test cfood for: +# https://gitlab.com/caosdb/caosdb-crawler/-/issues/9 + +Data: # name of the converter + type: Directory + match: (.*) + subtree: + DataAnalysis: # name of the converter + type: Directory + match: DataAnalysis + records: + RecordThatGetsParentsLater: + someId: 23 # <- this scalar causes problems + diff --git a/unittests/test_scalars_cfood.py b/unittests/test_scalars_cfood.py new file mode 100644 index 0000000000000000000000000000000000000000..1bf8f0b7d67f00f2018b5b68424d6b9cc17602eb --- /dev/null +++ b/unittests/test_scalars_cfood.py @@ -0,0 +1,57 @@ +#!/bin/python +# Tests for: +# https://gitlab.com/caosdb/caosdb-crawler/-/issues/9 +# A. Schlemmer, 06/2021 + +import pytest + +# The main function that is affected by this issue: +from caoscrawler.converters import handle_value +from caoscrawler.crawl import Crawler +# We need the store for the above function +from caoscrawler.stores import GeneralStore + +from test_tool import dircheckstr, rfp + + +@pytest.fixture +def crawler(): + crawler = Crawler(debug=True) + crawler.crawl_directory(rfp("test_directories", "examples_article"), + rfp("cfoods_scalar.yml")) + return crawler + + +def test_handle_value(): + # Note that we will need this store only, if we also want to test variables substitution: + store = GeneralStore() + + # This one should work: + assert handle_value("bla", store) == ("bla", "single") + + # These failed: + assert handle_value(4, store) == (4, "single") + assert handle_value(4.2, store) == (4.2, "single") + assert handle_value(True, store) == (True, "single") + + # List test: + assert handle_value([4, 3, 2], store) == ([4, 3, 2], "single") + + +def test_record_structure_generation(crawler): + subd = crawler.debug_tree[dircheckstr("DataAnalysis")] + assert len(subd) == 2 + # variables store on Data Analysis node of debug tree + assert len(subd[0]) == 3 + assert "Data" in subd[0] + assert "DataAnalysis" in subd[0] + assert "RecordThatGetsParentsLater" in subd[0] + + prop = subd[0]["RecordThatGetsParentsLater"].get_property("someId") + assert type(prop.value) == int + assert prop.value == 23 + + # record store on Data Analysis node of debug tree + assert len(subd[1]) == 1 + prop2 = subd[1]["RecordThatGetsParentsLater"].get_property("someId") + assert prop == prop2