Merge branch 'f-version-in-cfood' into 'dev'

ENH: Implement version check of cfood metadata

See merge request !78

CHANGELOG.md

@@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added some StructureElements: BooleanElement, FloatElement, IntegerElement, 
   ListElement, DictElement
 - String representation for Identifiables
+- [#43](https://gitlab.com/caosdb/caosdb-crawler/-/issues/43) the crawler
+  version can now be specified in the `metadata` section of the cfood
+  definition. It is checked against the installed version upon loading of the
+  definition.
 - JSON schema validation can also be used in the DictElementConverter
 
 ### Changed ###

src/caoscrawler/__init__.py

 from .crawl import Crawler, SecurityMode
+from .version import CfoodRequiredVersionError, version as __version__

src/caoscrawler/crawl.py

@@ -64,6 +64,7 @@ from .identified_cache import IdentifiedCache
 from .macros import defmacro_constructor, macro_constructor
 from .stores import GeneralStore, RecordStore
 from .structure_elements import StructureElement, Directory, NoneElement
+from .version import check_cfood_version
 
 logger = logging.getLogger(__name__)
 
@@ -255,12 +256,17 @@ class Crawler(object):
         if len(crawler_definitions) == 1:
             # Simple case, just one document:
             crawler_definition = crawler_definitions[0]
+            metadata = {}
         elif len(crawler_definitions) == 2:
+            metadata = crawler_definitions[0]["metadata"] if "metadata" in crawler_definitions[0] else {
+            }
             crawler_definition = crawler_definitions[1]
         else:
             raise RuntimeError(
                 "Crawler definition must not contain more than two documents.")
 
+        check_cfood_version(metadata)
+
         # TODO: at this point this function can already load the cfood schema extensions
         #       from the crawler definition and add them to the yaml schema that will be
         #       tested in the next lines of code:
@@ -275,8 +281,8 @@ class Crawler(object):
                 schema["cfood"]["$defs"]["converter"]["properties"]["type"]["enum"].append(
                     key)
         if len(crawler_definitions) == 2:
-            if "Converters" in crawler_definitions[0]["metadata"]:
-                for key in crawler_definitions[0]["metadata"]["Converters"]:
+            if "Converters" in metadata:
+                for key in metadata["Converters"]:
                     schema["cfood"]["$defs"]["converter"]["properties"]["type"]["enum"].append(
                         key)
 

src/caoscrawler/version.py

+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import importlib.metadata
+
+from packaging.version import parse as parse_version
+from warnings import warn
+
+# Read in version of locally installed caoscrawler package
+version = importlib.metadata.version("caoscrawler")
+
+
+class CfoodRequiredVersionError(RuntimeError):
+    """The installed crawler version is older than the version specified in the
+    cfood's metadata.
+
+    """
+
+
+def check_cfood_version(metadata: dict):
+
+    if not metadata or "crawler-version" not in metadata:
+
+        msg = """
+No crawler version specified in cfood definition, so there is now guarantee that
+the cfood definition matches the installed crawler version.
+
+Specifying a version is highly recommended to ensure that the definition works
+as expected with the installed version of the crawler.
+        """
+
+        warn(msg, UserWarning)
+        return
+
+    installed_version = parse_version(version)
+    cfood_version = parse_version(metadata["crawler-version"])
+
+    if cfood_version > installed_version:
+        msg = f"""
+Your cfood definition requires a newer version of the CaosDB crawler. Please
+update the crawler to the required version.
+
+Crawler version specified in cfood: {cfood_version}
+Crawler version installed on your system: {installed_version}
+        """
+        raise CfoodRequiredVersionError(msg)
+
+    elif cfood_version < installed_version:
+        # only warn if major or minor of installed version are newer than
+        # specified in cfood
+        if (cfood_version.major < installed_version.major) or (cfood_version.minor < installed_version.minor):
+            msg = f"""
+The cfood was written for a previous crawler version. Running the crawler in a
+newer version than specified in the cfood definition may lead to unwanted or
+unexpected behavior. Please visit the CHANGELOG
+(https://gitlab.com/caosdb/caosdb-crawler/-/blob/main/CHANGELOG.md) and check
+for any relevant changes.
+
+Crawler version specified in cfood: {cfood_version}
+Crawler version installed on your system: {installed_version}
+            """
+            warn(msg, UserWarning)
+            return
+
+    # At this point, the version is either equal or the installed crawler
+    # version is newer just by an increase in the patch version, so still
+    # compatible. We can safely ...
+    return

src/doc/cfood.rst

@@ -16,6 +16,9 @@ document together with the metadata and :doc:`macro<macros>` definitions (see :r
 If metadata and macro definitions are provided, there **must** be a second document preceeding the
 converter tree specification, including these definitions.
 
+It is highly recommended to specify the version of the CaosDB crawler for which
+the cfood is written in the metadata section, see :ref:`below<example_3>`.
+
 Examples
 ++++++++
 
@@ -69,6 +72,7 @@ two custom converters in the second document (**not recommended**, see the recom
    metadata:
      name: Datascience CFood
      description: CFood for data from the local data science work group
+     crawler-version: 0.2.1
      macros:
      - !defmacro
        name: SimulationDatasetFile
@@ -108,6 +112,7 @@ The **recommended way** of defining metadata, custom converters, macros and the
    metadata:
      name: Datascience CFood
      description: CFood for data from the local data science work group
+     crawler-version: 0.2.1
      macros:
      - !defmacro
        name: SimulationDatasetFile

unittests/test_cfood_metadata.py

+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import pytest
+import yaml
+
+from tempfile import NamedTemporaryFile
+
+import caoscrawler
+
+CRAWLER_VERSION = ""
+
+
+def setup_function(function):
+    """Store original crawler version in case it is altered for tests."""
+    CRAWLER_VERSION = caoscrawler.version.version
+
+
+def teardown_function(function):
+    """Reset version"""
+    caoscrawler.version.version = CRAWLER_VERSION
+
+
+def _temp_file_load(txt: str):
+    """
+    Create a temporary file with txt and load the crawler
+    definition using load_definition from Crawler.
+    """
+    definition = None
+    with NamedTemporaryFile() as f:
+        f.write(txt.encode())
+        f.flush()
+        c = caoscrawler.Crawler()
+        definition = c.load_definition(f.name)
+    return definition
+
+
+def test_warning_if_no_version_specified():
+    """Warn if no version is specified in the cfood."""
+
+    # metadata section exists but doesn't specify a version
+    definition_text = """
+---
+metadata:
+  name: Something
+  description: A cfood that does something
+---
+SimulationData:
+  type: Directory
+  match: SimulationData
+    """
+    with pytest.warns(UserWarning) as uw:
+        _temp_file_load(definition_text)
+
+    assert len(uw) == 1
+    assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
+    assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+
+    # metadata section is missing alltogether
+    definition_text = """
+SimulationData:
+  type: Directory
+  match: SimulationData
+    """
+
+    with pytest.warns(UserWarning) as uw:
+        _temp_file_load(definition_text)
+
+    assert len(uw) == 1
+    assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
+    assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+
+
+def test_warning_if_version_too_old():
+    """Warn if the cfood was written for an older crawler version."""
+
+    definition_text = """
+---
+metadata:
+  name: Something
+  description: A cfood that does something
+  crawler-version: 0.2.0
+---
+SimulationData:
+  type: Directory
+  match: SimulationData
+    """
+
+    # higher minor
+    caoscrawler.version.version = "0.3.0"
+    with pytest.warns(UserWarning) as uw:
+        _temp_file_load(definition_text)
+
+    assert len(uw) == 1
+    assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
+    assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
+    assert "version installed on your system: 0.3.0" in uw[0].message.args[0]
+
+    # higher major
+    caoscrawler.version.version = "1.1.0"
+    with pytest.warns(UserWarning) as uw:
+        _temp_file_load(definition_text)
+
+    assert len(uw) == 1
+    assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
+    assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
+    assert "version installed on your system: 1.1.0" in uw[0].message.args[0]
+
+
+def test_error_if_version_too_new():
+    """Raise error if the cfood requires a newer crawler version."""
+
+    # minor too old
+    definition_text = """
+---
+metadata:
+  name: Something
+  description: A cfood that does something
+  crawler-version: 0.2.1
+---
+SimulationData:
+  type: Directory
+  match: SimulationData
+    """
+    caoscrawler.version.version = "0.1.5"
+    with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+        _temp_file_load(definition_text)
+
+    assert "cfood definition requires a newer version" in str(cre.value)
+    assert "version specified in cfood: 0.2.1" in str(cre.value)
+    assert "version installed on your system: 0.1.5" in str(cre.value)
+
+    # major too old
+    definition_text = """
+---
+metadata:
+  name: Something
+  description: A cfood that does something
+  crawler-version: 1.0.1
+---
+SimulationData:
+  type: Directory
+  match: SimulationData
+    """
+    with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+        _temp_file_load(definition_text)
+
+    assert "cfood definition requires a newer version" in str(cre.value)
+    assert "version specified in cfood: 1.0.1" in str(cre.value)
+    assert "version installed on your system: 0.1.5" in str(cre.value)
+
+    # patch to old
+    caoscrawler.version.version = "1.0.0"
+
+    with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
+        _temp_file_load(definition_text)
+
+    assert "cfood definition requires a newer version" in str(cre.value)
+    assert "version specified in cfood: 1.0.1" in str(cre.value)
+    assert "version installed on your system: 1.0.0" in str(cre.value)
+
+
+def test_matching_version():
+    """Test that there is no warning or error in case the version matches."""
+
+    definition_text = """
+---
+metadata:
+  name: Something
+  description: A cfood that does something
+  crawler-version: 0.2.1
+---
+SimulationData:
+  type: Directory
+  match: SimulationData
+    """
+    caoscrawler.version.version = "0.2.1"
+    assert _temp_file_load(definition_text)
+
+    # The version is also considered a match if the patch version of the
+    # installed crawler is newer than the one specified in the cfood metadata
+    caoscrawler.version.version = "0.2.7"
+    assert _temp_file_load(definition_text)