TST: Add test for validating custom converter definitions

unittests/test_converters.py

@@ -23,12 +23,17 @@
 """
 test the converters module
 """
+import importlib
+import os
+import pytest
+import yaml
 
-from caoscrawler.converters import Converter
+from caoscrawler.converters import (Converter, ConverterValidationError,
+                                    DictConverter, DirectoryConverter,
+                                    handle_value, MarkdownFileConverter,
+                                    JSONFileConverter)
+from caoscrawler.crawl import Crawler
 from caoscrawler.stores import GeneralStore
-from caoscrawler.converters import (ConverterValidationError, DictConverter,
-                                    DirectoryConverter, handle_value,
-                                    MarkdownFileConverter, JSONFileConverter)
 from caoscrawler.structure_elements import (File, DictTextElement,
                                             DictListElement, DictElement,
                                             DictBooleanElement, DictDictElement,
@@ -37,10 +42,6 @@ from caoscrawler.structure_elements import (File, DictTextElement,
 
 from test_tool import rfp
 
-import pytest
-import os
-import importlib
-
 
 @pytest.fixture
 def converter_registry():
@@ -348,3 +349,39 @@ def test_filter_children_of_directory(converter_registry):
 
     with pytest.raises(RuntimeError):
         children = dc.create_children(None, test_dir)
+
+
+def test_validate_custom_converters():
+    one_doc_yaml = """
+Converters:
+  MyNewType:
+    converter: MyNewTypeConverter
+    package: some_package.my_converters
+MyElement:
+  type: MyNewType
+  match: something
+    """
+    crawler1 = Crawler()
+    one_doc_definitions = crawler1._load_definition_from_yaml_dict(
+        [yaml.load(one_doc_yaml, Loader=yaml.SafeLoader)])
+    assert "MyElement" in one_doc_definitions
+    assert one_doc_definitions["MyElement"]["type"] == "MyNewType"
+
+    # this has to be equivalent
+    two_doc_yaml = """
+---
+metadata:
+  Converters:
+    MyNewType:
+      converter: MyNewTypeConverter
+      package: some_package.my_converters
+---
+MyElement:
+  type: MyNewType
+  match: something
+    """
+    crawler2 = Crawler()
+    two_doc_definitions = crawler2._load_definition_from_yaml_dict(
+        list(yaml.safe_load_all(two_doc_yaml)))
+    assert "MyElement" in two_doc_definitions
+    assert two_doc_definitions["MyElement"]["type"] == one_doc_definitions["MyElement"]["type"]