diff --git a/CHANGELOG.md b/CHANGELOG.md
index efb3c04c853ac8b03c7b72fbd5fe27d4317d75f2..69d938aec8dab10d3d9961b0321beec3dbf8cff9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added some StructureElements: BooleanElement, FloatElement, IntegerElement,
ListElement, DictElement
- String representation for Identifiables
+- [#43](https://gitlab.com/caosdb/caosdb-crawler/-/issues/43) the crawler
+ version can now be specified in the `metadata` section of the cfood
+ definition. It is checked against the installed version upon loading of the
+ definition.
- JSON schema validation can also be used in the DictElementConverter
### Changed ###
diff --git a/src/caoscrawler/__init__.py b/src/caoscrawler/__init__.py
index b65b9fd9d24b9519a52ca13d07e46c9d8f791a73..044d8f0bf53c4c80dab9b492919fa64ab321a60d 100644
--- a/src/caoscrawler/__init__.py
+++ b/src/caoscrawler/__init__.py
@@ -1 +1,2 @@
from .crawl import Crawler, SecurityMode
+from .version import CfoodRequiredVersionError, version as __version__
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index 945050720f7a08e3e59145efd9350dbbf0595405..644782d78cd4060677e04cb039f7a3f679e3ccb6 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -64,6 +64,7 @@ from .identified_cache import IdentifiedCache
from .macros import defmacro_constructor, macro_constructor
from .stores import GeneralStore, RecordStore
from .structure_elements import StructureElement, Directory, NoneElement
+from .version import check_cfood_version
logger = logging.getLogger(__name__)
@@ -255,12 +256,17 @@ class Crawler(object):
if len(crawler_definitions) == 1:
# Simple case, just one document:
crawler_definition = crawler_definitions[0]
+ metadata = {}
elif len(crawler_definitions) == 2:
+ metadata = crawler_definitions[0]["metadata"] if "metadata" in crawler_definitions[0] else {
+ }
crawler_definition = crawler_definitions[1]
else:
raise RuntimeError(
"Crawler definition must not contain more than two documents.")
+ check_cfood_version(metadata)
+
# TODO: at this point this function can already load the cfood schema extensions
# from the crawler definition and add them to the yaml schema that will be
# tested in the next lines of code:
@@ -275,8 +281,8 @@ class Crawler(object):
schema["cfood"]["$defs"]["converter"]["properties"]["type"]["enum"].append(
key)
if len(crawler_definitions) == 2:
- if "Converters" in crawler_definitions[0]["metadata"]:
- for key in crawler_definitions[0]["metadata"]["Converters"]:
+ if "Converters" in metadata:
+ for key in metadata["Converters"]:
schema["cfood"]["$defs"]["converter"]["properties"]["type"]["enum"].append(
key)
diff --git a/src/caoscrawler/version.py b/src/caoscrawler/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..de604bbed94616cf5685825453d210fd713db1ef
--- /dev/null
+++ b/src/caoscrawler/version.py
@@ -0,0 +1,84 @@
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import importlib.metadata
+
+from packaging.version import parse as parse_version
+from warnings import warn
+
+# Read in version of locally installed caoscrawler package
+version = importlib.metadata.version("caoscrawler")
+
+
+class CfoodRequiredVersionError(RuntimeError):
+ """The installed crawler version is older than the version specified in the
+ cfood's metadata.
+
+ """
+
+
+def check_cfood_version(metadata: dict):
+
+ if not metadata or "crawler-version" not in metadata:
+
+ msg = """
+No crawler version specified in cfood definition, so there is now guarantee that
+the cfood definition matches the installed crawler version.
+
+Specifying a version is highly recommended to ensure that the definition works
+as expected with the installed version of the crawler.
+ """
+
+ warn(msg, UserWarning)
+ return
+
+ installed_version = parse_version(version)
+ cfood_version = parse_version(metadata["crawler-version"])
+
+ if cfood_version > installed_version:
+ msg = f"""
+Your cfood definition requires a newer version of the CaosDB crawler. Please
+update the crawler to the required version.
+
+Crawler version specified in cfood: {cfood_version}
+Crawler version installed on your system: {installed_version}
+ """
+ raise CfoodRequiredVersionError(msg)
+
+ elif cfood_version < installed_version:
+ # only warn if major or minor of installed version are newer than
+ # specified in cfood
+ if (cfood_version.major < installed_version.major) or (cfood_version.minor < installed_version.minor):
+ msg = f"""
+The cfood was written for a previous crawler version. Running the crawler in a
+newer version than specified in the cfood definition may lead to unwanted or
+unexpected behavior. Please visit the CHANGELOG
+(https://gitlab.com/caosdb/caosdb-crawler/-/blob/main/CHANGELOG.md) and check
+for any relevant changes.
+
+Crawler version specified in cfood: {cfood_version}
+Crawler version installed on your system: {installed_version}
+ """
+ warn(msg, UserWarning)
+ return
+
+ # At this point, the version is either equal or the installed crawler
+ # version is newer just by an increase in the patch version, so still
+ # compatible. We can safely ...
+ return
diff --git a/src/doc/cfood.rst b/src/doc/cfood.rst
index 677cadc55709c6c25d16ff547b311102ee78699a..37f6a8c7d3be9298ec965c50a4ec29110988ddc6 100644
--- a/src/doc/cfood.rst
+++ b/src/doc/cfood.rst
@@ -16,6 +16,9 @@ document together with the metadata and :doc:`macro<macros>` definitions (see :r
If metadata and macro definitions are provided, there **must** be a second document preceeding the
converter tree specification, including these definitions.
+It is highly recommended to specify the version of the CaosDB crawler for which
+the cfood is written in the metadata section, see :ref:`below<example_3>`.
+
Examples
++++++++
@@ -69,6 +72,7 @@ two custom converters in the second document (**not recommended**, see the recom
metadata:
name: Datascience CFood
description: CFood for data from the local data science work group
+ crawler-version: 0.2.1
macros:
- !defmacro
name: SimulationDatasetFile
@@ -108,6 +112,7 @@ The **recommended way** of defining metadata, custom converters, macros and the
metadata:
name: Datascience CFood
description: CFood for data from the local data science work group
+ crawler-version: 0.2.1
macros:
- !defmacro
name: SimulationDatasetFile
diff --git a/unittests/test_cfood_metadata.py b/unittests/test_cfood_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..09d6c88bdc27e1066ed18a9c5865cbfb95270c3a
--- /dev/null
+++ b/unittests/test_cfood_metadata.py
@@ -0,0 +1,199 @@
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import pytest
+import yaml
+
+from tempfile import NamedTemporaryFile
+
+import caoscrawler
+
+CRAWLER_VERSION = ""
+
+
+def setup_function(function):
+ """Store original crawler version in case it is altered for tests."""
+ CRAWLER_VERSION = caoscrawler.version.version
+
+
+def teardown_function(function):
+ """Reset version"""
+ caoscrawler.version.version = CRAWLER_VERSION
+
+
+def _temp_file_load(txt: str):
+ """
+ Create a temporary file with txt and load the crawler
+ definition using load_definition from Crawler.
+ """
+ definition = None
+ with NamedTemporaryFile() as f:
+ f.write(txt.encode())
+ f.flush()
+ c = caoscrawler.Crawler()
+ definition = c.load_definition(f.name)
+ return definition
+
+
+def test_warning_if_no_version_specified():
+ """Warn if no version is specified in the cfood."""
+
+ # metadata section exists but doesn't specify a version
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
+ assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+
+ # metadata section is missing alltogether
+ definition_text = """
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
+ assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+
+
+def test_warning_if_version_too_old():
+ """Warn if the cfood was written for an older crawler version."""
+
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 0.2.0
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+
+ # higher minor
+ caoscrawler.version.version = "0.3.0"
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
+ assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
+ assert "version installed on your system: 0.3.0" in uw[0].message.args[0]
+
+ # higher major
+ caoscrawler.version.version = "1.1.0"
+ with pytest.warns(UserWarning) as uw:
+ _temp_file_load(definition_text)
+
+ assert len(uw) == 1
+ assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
+ assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
+ assert "version installed on your system: 1.1.0" in uw[0].message.args[0]
+
+
+def test_error_if_version_too_new():
+ """Raise error if the cfood requires a newer crawler version."""
+
+ # minor too old
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 0.2.1
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ caoscrawler.version.version = "0.1.5"
+ with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+ _temp_file_load(definition_text)
+
+ assert "cfood definition requires a newer version" in str(cre.value)
+ assert "version specified in cfood: 0.2.1" in str(cre.value)
+ assert "version installed on your system: 0.1.5" in str(cre.value)
+
+ # major too old
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 1.0.1
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+ _temp_file_load(definition_text)
+
+ assert "cfood definition requires a newer version" in str(cre.value)
+ assert "version specified in cfood: 1.0.1" in str(cre.value)
+ assert "version installed on your system: 0.1.5" in str(cre.value)
+
+ # patch to old
+ caoscrawler.version.version = "1.0.0"
+
+ with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+ _temp_file_load(definition_text)
+
+ assert "cfood definition requires a newer version" in str(cre.value)
+ assert "version specified in cfood: 1.0.1" in str(cre.value)
+ assert "version installed on your system: 1.0.0" in str(cre.value)
+
+
+def test_matching_version():
+ """Test that there is no warning or error in case the version matches."""
+
+ definition_text = """
+---
+metadata:
+ name: Something
+ description: A cfood that does something
+ crawler-version: 0.2.1
+---
+SimulationData:
+ type: Directory
+ match: SimulationData
+ """
+ caoscrawler.version.version = "0.2.1"
+ assert _temp_file_load(definition_text)
+
+ # The version is also considered a match if the patch version of the
+ # installed crawler is newer than the one specified in the cfood metadata
+ caoscrawler.version.version = "0.2.7"
+ assert _temp_file_load(definition_text)