WIP: Extend h5 test

unittests/h5_cfood.yml

@@ -2,16 +2,33 @@
 metadata:
   crawler-version: 0.6.1
 ---
+Converters:
+  H5Dataset:
+    converter: H5DatasetConverter
+    package: caoscrawler.hdf5_converter
+  H5File:
+    converter: H5FileConverter
+    package: caoscrawler.hdf5_converter
+  H5Group:
+    converter: H5GroupConverter
+    package: caoscrawler.hdf5_converter
+  H5Ndarray:
+    converter: H5NdarrayConverter
+    package: caoscrawler.hdf5_converter
 # Top-level, we have just the HDF5 file.
+ParentDirectory:
+  type: Directory
+  match: (.*)
+  subtree:
     H5FileElement:
       type: H5File
       match: (.*)\.(hdf5|h5)$
       subtree:
-    # Here, we have the groups, the top-level dataset, and possible attributes
-    # (empty for now).
+        # Here, we have the groups, the top-level dataset, and possible
+        # attributes (empty for now).
         RootIntegerElement:
           type: H5Dataset
-      match: ^root_integers$
+          match_name: ^root_integers$
           records:
             H5Dataset:
               parents:
@@ -20,14 +37,14 @@ H5FileElement:
             # included NDArray in this dataset
             TopLevelIntNDElement:
               type: H5Ndarray
-          match: (.*)
+              match_name: (.*)
               recordname: this
               records:
                 H5Dataset: +$this
             # There is one more list-valued attribute to this dataset.
             TopLevelDataAttribute:
               type: ListElement
-          match: ^attr_data_root$
+              match_name: ^attr_data_root$
               subtree:
                 AttributeListEntries:
                   type: FloatElement

unittests/test_h5_converter.py

@@ -19,14 +19,16 @@
 #
 import numpy as np
 
+from pathlib import Path
+from pytest import fixture, importorskip
 
-from caoscrawler.hdf5_converter import (__convert_basic_element_with_nd_array,
-                                        __convert_h5_element, H5GroupElement,
+from caoscrawler.debug_tree import DebugTree
+from caoscrawler.hdf5_converter import (convert_basic_element_with_nd_array,
+                                        convert_h5_element, H5GroupElement,
                                         H5DatasetElement, H5NdarrayElement)
+from caoscrawler.scanner import scan_directory
 from caoscrawler.structure_elements import (FloatElement, ListElement,
                                             TextElement)
-from pathlib import Path
-from pytest import fixture, importorskip
 
 # Skip the whole module if h5py hasn't been installed
 h5py = importorskip("h5py")
@@ -45,10 +47,10 @@ def h5_dummy_file():
 
 def test_h5_elements(h5_dummy_file):
 
-    elt = __convert_h5_element(h5_dummy_file["group_level1_a"], "test")
+    elt = convert_h5_element(h5_dummy_file["group_level1_a"], "test")
     assert isinstance(elt, H5GroupElement)
 
-    elt = __convert_h5_element(h5_dummy_file["root_integers"], "test")
+    elt = convert_h5_element(h5_dummy_file["root_integers"], "test")
     assert isinstance(elt, H5DatasetElement)
 
 
@@ -57,30 +59,38 @@ def test_nd_array_conversion():
     # Only test array handling here, `convert_basic_element` is tested
     # elsewhere.
     arr = np.array([[["something"]]])
-    elt = __convert_basic_element_with_nd_array(arr)
+    elt = convert_basic_element_with_nd_array(arr)
     assert isinstance(elt, TextElement)
     assert elt.value == "something"
 
     arr = np.zeros((1, 1))
-    elt = __convert_basic_element_with_nd_array(arr)
+    elt = convert_basic_element_with_nd_array(arr)
     assert isinstance(elt, FloatElement)
     assert elt.value == 0
 
     arr = np.zeros((1, 3, 1))
-    elt = __convert_basic_element_with_nd_array(arr)
+    elt = convert_basic_element_with_nd_array(arr)
     assert isinstance(elt, ListElement)
     assert elt.value == [0, 0, 0]
 
     arr = np.array([[1, 2, 3], [4, 5, 6]])
-    elt = __convert_basic_element_with_nd_array(arr, internal_path="some/path")
+    elt = convert_basic_element_with_nd_array(arr, internal_path="some/path")
     assert isinstance(elt, H5NdarrayElement)
     assert elt.internal_path == "some/path"
 
     # Non-arrays should be forwarded correctly
-    elt = __convert_basic_element_with_nd_array("something")
+    elt = convert_basic_element_with_nd_array("something")
     assert isinstance(elt, TextElement)
     assert elt.value == "something"
 
-    elt = __convert_basic_element_with_nd_array([0, 0, 0])
+    elt = convert_basic_element_with_nd_array([0, 0, 0])
     assert isinstance(elt, ListElement)
     assert elt.value == [0, 0, 0]
+
+def test_record_creation():
+
+    dbtr = DebugTree()
+    records = scan_directory(UNITTESTDIR, UNITTESTDIR / "h5_cfood.yml", debug_tree=dbtr)
+    print(dbtr.debug_tree)
+    print(dbtr.debug_metadata)
+    assert False