FIX: Use correct imports in unit tests

src/caoscrawler/__init__.py

-from . import converters, utils, xml_converter
+from . import converters, utils
 from .crawl import Crawler, SecurityMode
 from .version import CfoodRequiredVersionError, get_caoscrawler_version
 

src/caoscrawler/converters/__init__.py

@@ -22,11 +22,11 @@
 
 from .. import utils
 from .converters import *
+from .xml_converter import *
 
 try:
-    from .converters.spss import SPSSConverter
+    from .spss import SPSSConverter
 except ImportError as err:
     SPSSConverter: type = utils.MissingImport(
         name="SPSSConverter", hint="Try installing with the `spss` extra option.",
         err=err)
-

src/caoscrawler/converters/converters.py

@@ -43,9 +43,9 @@ from jsonschema import ValidationError, validate
 
 from ..stores import GeneralStore, RecordStore
 from ..structure_elements import (BooleanElement, DictElement, Directory, File,
-                                 FloatElement, IntegerElement, JSONFile,
-                                 ListElement, NoneElement, StructureElement,
-                                 TextElement)
+                                  FloatElement, IntegerElement, JSONFile,
+                                  ListElement, NoneElement, StructureElement,
+                                  TextElement)
 from ..utils import has_parent
 
 # These are special properties which are (currently) treated differently

src/caoscrawler/converters/spss.py

@@ -28,8 +28,7 @@ import pandas as pd
 import pyreadstat
 import yaml
 
-import .converters
-
+from . import converters
 from ..stores import GeneralStore
 from ..structure_elements import (File, StructureElement)
 from typing import Optional, Any

src/caoscrawler/xml_converter.py → src/caoscrawler/converters/xml_converter.py

@@ -35,12 +35,12 @@ from typing import Any, Callable, Optional, Union
 import linkahead as db
 from jsonschema import ValidationError, validate
 
-from .stores import GeneralStore, RecordStore
-from .structure_elements import (BooleanElement, DictElement, Directory, File,
+from ..stores import GeneralStore, RecordStore
+from ..structure_elements import (BooleanElement, DictElement, Directory, File,
                                  FloatElement, IntegerElement, JSONFile,
                                  ListElement, NoneElement, StructureElement,
                                  TextElement, XMLTagElement, XMLTextNode, XMLAttributeNode)
-from .utils import has_parent
+from ..utils import has_parent
 
 import lxml.etree
 from .converters import SimpleFileConverter, ConverterValidationError, Converter

src/caoscrawler/default_converters.yml

@@ -102,12 +102,12 @@ XLSXTableConverter:
 
 XMLFile:
   converter: XMLFileConverter
-  package: caoscrawler.xml_converter
+  package: caoscrawler.converters
 
 XMLTag:
   converter: XMLTagConverter
-  package: caoscrawler.xml_converter
+  package: caoscrawler.converters
 
 XMLTextNode:
   converter: XMLTextNodeConverter
-  package: caoscrawler.xml_converter
+  package: caoscrawler.converters

unittests/h5_cfood.yml

@@ -5,16 +5,16 @@ metadata:
 Converters:
   H5Dataset:
     converter: H5DatasetConverter
-    package: caoscrawler.hdf5_converter
+    package: caoscrawler.converters.hdf5_converter
   H5File:
     converter: H5FileConverter
-    package: caoscrawler.hdf5_converter
+    package: caoscrawler.converters.hdf5_converter
   H5Group:
     converter: H5GroupConverter
-    package: caoscrawler.hdf5_converter
+    package: caoscrawler.converters.hdf5_converter
   H5Ndarray:
     converter: H5NdarrayConverter
-    package: caoscrawler.hdf5_converter
+    package: caoscrawler.converters.hdf5_converter
 # Top-level, we have just the HDF5 file.
 ParentDirectory:
   type: Directory

unittests/test_converters.py

@@ -46,8 +46,8 @@ from caoscrawler.converters import (Converter, ConverterValidationError,
                                     ListElementConverter, MarkdownFileConverter,
                                     PropertiesFromDictConverter,
                                     YAMLFileConverter,
-                                    _AbstractScalarValueElementConverter,
                                     handle_value, replace_variables)
+from caoscrawler.converters.converters import _AbstractScalarValueElementConverter
 from caoscrawler.crawl import Crawler
 from caoscrawler.scanner import (_load_definition_from_yaml_dict,
                                  create_converter_registry,
@@ -648,10 +648,7 @@ def test_load_converters():
     # All of them are contained in caoscrawler.converters
     # except for the xml converters:
     for conv_key, conv in converter_registry.items():
-        if conv_key in ("XMLTag", "XMLFile", "XMLTextNode"):
-            assert conv["package"] == "caoscrawler.xml_converter"
-        else:
-            assert conv["package"] == "caoscrawler.converters"
+        assert conv["package"] == "caoscrawler.converters"
         # ... and their names all end in "Converter"
         assert conv["converter"].endswith("Converter")
 

unittests/test_h5_converter.py

@@ -26,9 +26,9 @@ from pytest import fixture, importorskip
 import linkahead as db
 
 from caoscrawler.debug_tree import DebugTree
-from caoscrawler.hdf5_converter import (convert_basic_element_with_nd_array,
-                                        convert_h5_element, H5GroupElement,
-                                        H5DatasetElement, H5NdarrayElement)
+from caoscrawler.converters.hdf5_converter import (convert_basic_element_with_nd_array,
+                                                   convert_h5_element, H5GroupElement,
+                                                   H5DatasetElement, H5NdarrayElement)
 from caoscrawler.scanner import scan_directory
 from caoscrawler.structure_elements import (FloatElement, ListElement,
                                             TextElement)

unittests/test_xml_converter.py

@@ -21,40 +21,22 @@
 #
 
 """
-test the converters module
+test the XML converters
 """
-import datetime
 import importlib
 import json
-import logging
-import os
+import pytest
 import sys
-from itertools import product
+import yaml
+
+from lxml.etree import fromstring
 from pathlib import Path
 
-import pytest
-import yaml
-from caoscrawler.converters import (Converter, ConverterValidationError,
-                                    DateElementConverter, DictElementConverter,
-                                    DictIntegerElementConverter,
-                                    DirectoryConverter, FloatElementConverter,
-                                    IntegerElementConverter, JSONFileConverter,
-                                    ListElementConverter,
-                                    MarkdownFileConverter, YAMLFileConverter,
-                                    _AbstractScalarValueElementConverter,
-                                    handle_value, replace_variables)
-from caoscrawler.crawl import Crawler
-from caoscrawler.scanner import (_load_definition_from_yaml_dict,
-                                 create_converter_registry,
-                                 create_transformer_registry, load_definition)
+from caoscrawler.converters import XMLTagConverter
+from caoscrawler.scanner import load_definition
 from caoscrawler.stores import GeneralStore
-from caoscrawler.structure_elements import (BooleanElement, DictElement,
-                                            Directory, File, FloatElement,
-                                            IntegerElement, ListElement,
-                                            TextElement, XMLTagElement)
-from caoscrawler.xml_converter import XMLTagConverter
+from caoscrawler.structure_elements import XMLTagElement
 
-from lxml.etree import fromstring
 
 UNITTESTDIR = Path(__file__).parent
 
@@ -62,19 +44,13 @@ UNITTESTDIR = Path(__file__).parent
 @pytest.fixture
 def converter_registry():
     converter_registry: dict[str, dict[str, str]] = {
-        "Directory": {
-            "converter": "DirectoryConverter",
-            "package": "caoscrawler.converters"},
-        "TextElement": {
-            "converter": "TextElementConverter",
-            "package": "caoscrawler.converters"},
         "XMLTag": {
             "converter": "XMLTagConverter",
-            "package": "caoscrawler.xml_converter"},
+            "package": "caoscrawler.converters"},
 
         "XMLTextNode": {
             "converter": "XMLTextNodeConverter",
-            "package": "caoscrawler.xml_converter"},
+            "package": "caoscrawler.converters"},
     }
 
     for key, value in converter_registry.items():