TST: add unittest

src/newcrawler/converters.py

@@ -26,10 +26,11 @@
 import os
 import re
 import caosdb as db
+import json
 from .utils import has_parent
 from .stores import GeneralStore, RecordStore
 from .structure_elements import (StructureElement, Directory, File,
-                                 TextElement, DictTextElement, DictListElement)
+                                 TextElement, DictTextElement, DictElement, DictListElement)
 from typing import Optional, Union
 from abc import abstractmethod
 import yaml_header_tools
@@ -397,17 +398,18 @@ class JSONFileConverter(Converter):
             # DictXXXElements for numbers, booleans, enums, ...
             if type(entry) == list:
                 children.append(DictListElement(name, entry))
-            elif type(entry) == string:
+            elif type(entry) == str:
                 children.append(DictTextElement(name, entry))
             else:
-                raise RuntimeError(f"JSON entry {name} has incompatible type.")
+                children.append(DictElement(name, entry))
+                print(f"JSON entry {name} has incompatible type.")
+                # raise RuntimeError(f"JSON entry {name} has incompatible type.")
 
         return children
 
 
 class DictTextElementConverter(Converter):
-    def create_children(self, generalStore: GeneralStore,
-                        element: StructureElement):
+    def create_children(self, generalStore: GeneralStore, element: StructureElement):
         return []
 
     def typecheck(self, element: StructureElement):

src/newcrawler/structure_elements.py

@@ -60,6 +60,12 @@ class File(FileSystemStructureElement):
     pass
 
 
+class DictElement(StructureElement):
+    def __init__(self, name: str, value: str):
+        super().__init__(name)
+        self.value = value
+
+
 class DictTextElement(StructureElement):
     def __init__(self, name: str, value: str):
         super().__init__(name)

unittests/test_converters.py

@@ -29,15 +29,16 @@ test the converters module
 
 from newcrawler.converters import Converter
 from newcrawler.stores import GeneralStore
-from newcrawler.converters import MarkdownFileConverter
+from newcrawler.converters import MarkdownFileConverter, JSONFileConverter
 from newcrawler.structure_elements import Directory
-from newcrawler.structure_elements import File, DictTextElement, DictListElement
+from newcrawler.structure_elements import File, DictTextElement, DictListElement, DictElement
 
 from test_tool import rfp
 
 import pytest
 import importlib
 
+
 @pytest.fixture
 def converter_registry():
     converter_registry: dict[str, dict[str, str]] = {
@@ -55,7 +56,10 @@ def converter_registry():
             "package": "newcrawler.converters"},
         "TextElement": {
             "converter": "TextElementConverter",
-            "package": "newcrawler.converters"}
+            "package": "newcrawler.converters"},
+        "JSONFile": {
+            "converter": "JSONFileConverter",
+            "package": "newcrawler.converters"},
     }
 
     for key, value in converter_registry.items():
@@ -158,3 +162,54 @@ def test_markdown_converter(converter_registry):
     assert children[0].__class__ == DictListElement
     assert children[0].name == "responsible"
     assert children[0].value.__class__ == list
+
+
+def test_json_converter(converter_registry):
+    test_json = File("testjson.json", rfp(
+        "test_directories", "single_file_test_data", "testjson.json"))
+
+    converter = JSONFileConverter(
+        definition={"match": "(.*)"},
+        name="TestJSONFileConverter",
+        converter_registry=converter_registry)
+
+    m = converter.match(test_json)
+    assert m is not None
+    assert len(m) == 0
+
+    children = converter.create_children(None, test_json)
+    assert len(children) == 8
+    assert children[0].__class__ == DictTextElement
+    assert children[0].name == "name"
+    assert children[0].value.__class__ == str
+    assert children[0].value == "DEMO"
+
+    assert children[1].__class__ == DictElement
+    assert children[1].name == "id"
+    assert children[1].value.__class__ == int
+    assert children[1].value == 10002
+
+    assert children[2].__class__ == DictElement
+    assert children[2].name == "archived"
+    assert children[2].value.__class__ == bool
+
+    assert children[3].__class__ == DictElement
+    assert children[3].name == "coordinator"
+    assert children[3].value.__class__ == dict
+
+    assert children[4].__class__ == DictTextElement
+    assert children[4].name == "start_date"
+    assert children[4].value.__class__ == str
+
+    assert children[5].__class__ == DictListElement
+    assert children[5].name == "candidates"
+    assert children[5].value.__class__ == list
+    assert children[5].value == ["Mouse", "Penguine"]
+
+    assert children[6].__class__ == DictElement
+    assert children[6].name == "rvalue"
+    assert children[6].value.__class__ == float
+
+    assert children[7].__class__ == DictTextElement
+    assert children[7].name == "url"
+    assert children[7].value.__class__ == str