TST: added tests for validation of schema for crawler definition yaml files

src/newcrawler/crawl.py

@@ -154,11 +154,10 @@ class Crawler(object):
             self.debug_metadata["provenance"] = defaultdict(lambda: dict())
             self.debug_metadata["usage"] = defaultdict(lambda: set())
 
-    def crawl_directory(self, dirname: str, crawler_definition_path: str):
-        """ Crawl a single directory.
-
-        Convenience function that starts the crawler (calls start_crawling)
-        with a single cirectory as the StructureElement.
+    def load_definition(self, crawler_definition_path: str):
+        """
+        Load a cfood from a crawler definition defined by
+        crawler definition path and validate it using cfood-schema.yml.
         """
         
         # Load the cfood from a yaml file:
@@ -170,6 +169,17 @@ class Crawler(object):
             schema = yaml.safe_load(f)
         validate(instance=crawler_definition, schema=schema["cfood"])
 
+        return crawler_definition
+
+    def crawl_directory(self, dirname: str, crawler_definition_path: str):
+        """ Crawl a single directory.
+
+        Convenience function that starts the crawler (calls start_crawling)
+        with a single directory as the StructureElement.
+        """
+
+        crawler_definition = self.load_definition(crawler_definition_path)
+
         self.start_crawling(Directory(os.path.basename(dirname),
                                       dirname),
                             crawler_definition)

unittests/test_schema.py

+#!/bin/python
+# Tests for schema validation
+# A. Schlemmer, 06/2021
+
+import caosdb as db
+
+from os.path import join, dirname
+from newcrawler import Crawler
+
+import pytest
+from pytest import raises
+
+from jsonschema.exceptions import ValidationError
+
+def rfp(*pathcomponents):
+    """
+    Return full path.
+    Shorthand convenience function.
+    """
+    return join(dirname(__file__), *pathcomponents)
+
+
+def test_schema_validation():
+    cr = Crawler()
+    cr.load_definition(rfp("scifolder_cfood.yml"))
+    cr.load_definition(rfp("scifolder_extended.yml"))
+
+    with raises(ValidationError, match=".*enum.*"):
+        cr.load_definition(rfp("broken_cfoods", "broken1.yml"))