From 80cc116c00f62e8825870b1ecee65da7d999f496 Mon Sep 17 00:00:00 2001
From: florian <f.spreckelsen@inidscale.com>
Date: Fri, 7 Oct 2022 10:10:45 +0200
Subject: [PATCH] TST: Add test for validating custom converter definitions
---
unittests/test_converters.py | 53 ++++++++++++++++++++++++++++++------
1 file changed, 45 insertions(+), 8 deletions(-)
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index 15a39b72..30c5972c 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -23,12 +23,17 @@
"""
test the converters module
"""
+import importlib
+import os
+import pytest
+import yaml
-from caoscrawler.converters import Converter
+from caoscrawler.converters import (Converter, ConverterValidationError,
+ DictConverter, DirectoryConverter,
+ handle_value, MarkdownFileConverter,
+ JSONFileConverter)
+from caoscrawler.crawl import Crawler
from caoscrawler.stores import GeneralStore
-from caoscrawler.converters import (ConverterValidationError, DictConverter,
- DirectoryConverter, handle_value,
- MarkdownFileConverter, JSONFileConverter)
from caoscrawler.structure_elements import (File, DictTextElement,
DictListElement, DictElement,
DictBooleanElement, DictDictElement,
@@ -37,10 +42,6 @@ from caoscrawler.structure_elements import (File, DictTextElement,
from test_tool import rfp
-import pytest
-import os
-import importlib
-
@pytest.fixture
def converter_registry():
@@ -348,3 +349,39 @@ def test_filter_children_of_directory(converter_registry):
with pytest.raises(RuntimeError):
children = dc.create_children(None, test_dir)
+
+
+def test_validate_custom_converters():
+ one_doc_yaml = """
+Converters:
+ MyNewType:
+ converter: MyNewTypeConverter
+ package: some_package.my_converters
+MyElement:
+ type: MyNewType
+ match: something
+ """
+ crawler1 = Crawler()
+ one_doc_definitions = crawler1._load_definition_from_yaml_dict(
+ [yaml.load(one_doc_yaml, Loader=yaml.SafeLoader)])
+ assert "MyElement" in one_doc_definitions
+ assert one_doc_definitions["MyElement"]["type"] == "MyNewType"
+
+ # this has to be equivalent
+ two_doc_yaml = """
+---
+metadata:
+ Converters:
+ MyNewType:
+ converter: MyNewTypeConverter
+ package: some_package.my_converters
+---
+MyElement:
+ type: MyNewType
+ match: something
+ """
+ crawler2 = Crawler()
+ two_doc_definitions = crawler2._load_definition_from_yaml_dict(
+ list(yaml.safe_load_all(two_doc_yaml)))
+ assert "MyElement" in two_doc_definitions
+ assert two_doc_definitions["MyElement"]["type"] == one_doc_definitions["MyElement"]["type"]
--
GitLab