WIP: Filling XLSX: Seems to be working.

We should still have a few more tests.

src/caosadvancedtools/table_json_conversion/fill_xlsx.py

@@ -20,6 +20,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
+from __future__ import annotations
+
 import json
 from collections import OrderedDict
 from types import SimpleNamespace
@@ -126,7 +128,6 @@ class TemplateFiller:
     def __init__(self, workbook: Workbook):
         self._workbook = workbook
         self._create_index()
-        self._context: Optional[dict[str, Any]] = None
 
     @property
     def workbook(self):
@@ -134,9 +135,49 @@ class TemplateFiller:
 
     def fill_data(self, data: dict):
         """Fill the data into the workbook."""
-        self._context = data
         self._handle_data(data=data)
-        self._context = None
+
+    class Context:
+        """Context for an entry: simple properties of all ancestors, organized in a dict.
+
+        This is similar to a dictionary with all scalar element properties at the tree nodes up to
+        the root.  Siblings in lists and dicts are ignored.  Additionally the context knows where
+        its current position is.
+
+        Lookup of elements can easily be achieved by giving the path (as ``list[str]`` or
+        stringified path).
+
+        """
+
+        def __init__(self, current_path: List[str] = None, props: Dict[str, Any] = None):
+            self._current_path = current_path if current_path is not None else []
+            self._props = props if props is not None else {}  # this is flat
+
+        def copy(self) -> TemplateFiller.Context:
+            """Deep copy."""
+            result = TemplateFiller.Context(current_path=self._current_path.copy(),
+                                            props=self._props.copy())
+            return result
+
+        def next_level(self, next_level: str) -> TemplateFiller.Context:
+            result = self.copy()
+            result._current_path.append(next_level)
+            return result
+
+        def __getitem__(self, path: Union[List[str], str], owner=None) -> Any:
+            if isinstance(path, list):
+                path = p2s(path)
+            return self._props[path]
+
+        def __setitem__(self, propname: str, value):
+            fullpath = p2s(self._current_path + [propname])
+            self._props[fullpath] = value
+
+        def fill_from_data(self, data: Dict[str, Any]):
+            """Fill current level with all scalar elements of ``data``."""
+            for name, value in data.items():
+                if not isinstance(value, (dict, list)):
+                    self[name] = value
 
     def _create_index(self):
         """Create a sheet index for the workbook.
@@ -175,6 +216,7 @@ class TemplateFiller:
                     col_type=col[coltype_idx].value)
 
     def _handle_data(self, data: dict, current_path: List[str] = None,
+                     context: TemplateFiller.Context = None,
                      only_collect_insertables: bool = False,
                      ) -> Optional[Dict[str, Any]]:
         """Handle the data and write it into ``workbook``.
@@ -186,7 +228,13 @@ data: dict
 
 current_path: list[str], optional
   If this is None or empty, we are at the top level.  This means that all children shall be entered
-  into their respective sheets and not into a sheet at this level.
+  into their respective sheets and not into a sheet at this level.  ``current_path`` and ``context``
+  must either both be given, or none of them.
+
+context: TemplateFiller.Context, optional
+  Directopry of scalar element properties at the tree nodes up to the root.  Siblings in lists
+  and dicts are ignored.  ``context`` and ``current_path`` must either both be given, or none of
+  them.
 
 only_collect_insertables: bool, optional
   If True, do not insert anything on this level, but return a dict with entries to be inserted.
@@ -194,16 +242,21 @@ only_collect_insertables: bool, optional
 
 Returns
 -------
-
 out: union[dict, None]
   If ``only_collect_insertables`` is True, return a dict (path string -> value)
         """
+        assert (current_path is None) is (context is None), (
+            "`current_path` and `context` must either both be given, or none of them.")
         if current_path is None:
             current_path = []
-        assert self._context is not None
+        if context is None:
+            context = TemplateFiller.Context()
+        context.fill_from_data(data)
+
         insertables: Dict[str, Any] = {}
         for name, content in data.items():
             path = current_path + [name]
+            next_context = context.next_level(name)
             # preprocessing
             if isinstance(content, list):
                 if not content:
@@ -213,13 +266,13 @@ out: union[dict, None]
                 if isinstance(content[0], dict):
                     # An array of objects: must go into exploded sheet
                     for entry in content:
-                        self._handle_data(data=entry, current_path=path)
+                        self._handle_data(data=entry, current_path=path, context=next_context)
                     continue
             elif isinstance(content, dict):
                 if not current_path:  # Special handling for top level
-                    self._handle_data(content, current_path=path)
+                    self._handle_data(content, current_path=path, context=next_context)
                     continue
-                insert = self._handle_data(content, current_path=path,
+                insert = self._handle_data(content, current_path=path, context=next_context.copy(),
                                            only_collect_insertables=True)
                 assert isinstance(insert, dict)
                 assert not any(key in insertables for key in insert)
@@ -260,7 +313,7 @@ out: union[dict, None]
         if insert_row is not None and sheet is not None and _is_exploded_sheet(sheet):
             foreigns = _get_foreign_key_columns(sheet)
             for index, path in ((f.index, f.path) for f in foreigns.values()):
-                value = _get_deep_value(self._context, path)
+                value = context[path]
                 sheet.cell(row=insert_row+1, column=index+1, value=value)
 
         return None

unittests/table_json_conversion/data/multiple_refs_data.json

+{
+  "Training": {
+    "trainer": [],
+    "participant": [
+      {
+        "full_name": "Petra Participant",
+        "email": "petra@indiscale.com"
+      },
+      {
+        "full_name": "Peter",
+        "email": "peter@getlinkahead.com"
+      }
+    ],
+    "Organisation": [
+      {
+        "Person": [
+          {
+            "full_name": "Henry Henderson",
+            "email": "henry@organization.org"
+          },
+          {
+            "full_name": "Harry Hamburg",
+            "email": "harry@organization.org"
+          }
+        ],
+        "name": "World Training Organization",
+        "Country": "US"
+      },
+      {
+        "Person": [
+          {
+            "full_name": "Hermione Harvard",
+            "email": "hermione@organisation.org.uk"
+          },
+          {
+            "full_name": "Hazel Harper",
+            "email": "hazel@organisation.org.uk"
+          }
+        ],
+        "name": "European Training Organisation",
+        "Country": "UK"
+      }
+    ],
+    "date": "2024-03-21T14:12:00.000Z",
+    "url": "www.indiscale.com",
+    "name": "Example training with multiple organizations."
+  }
+}

unittests/table_json_conversion/test_fill_xlsx.py

@@ -56,9 +56,6 @@ custom_output: str, optional
         assert os.path.exists(outfile)
         generated = load_workbook(outfile)  # workbook can be read
     known_good_wb = load_workbook(known_good)
-    # if custom_output is not None:
-    #     from IPython import embed
-    #     embed()
     compare_workbooks(generated, known_good_wb)
 
 
@@ -71,5 +68,6 @@ def test_detect():
 def test_fill_xlsx():
     fill_and_compare(json_file="data/simple_data.json", template_file="data/simple_template.xlsx",
                      known_good="data/simple_data.xlsx")
-    # fill_and_compare(json_file="data/example.json", template_file="data/example_template.xlsx",
-    #                  known_good="data/example_single_data.xlsx")
+    fill_and_compare(json_file="data/multiple_refs_data.json",
+                     template_file="data/multiple_refs_template.xlsx",
+                     known_good="data/multiple_refs_data.xlsx")

unittests/table_json_conversion/utils.py

@@ -33,7 +33,9 @@ Parameters
 hidden: bool, optional
   Test if the "hidden" status of rows and columns is the same.
     """
-    assert wb1.sheetnames == wb2.sheetnames, "Sheet names are different."
+    assert wb1.sheetnames == wb2.sheetnames, (
+        f"Sheet names are different: \n{wb1.sheetnames}\n   !=\n{wb2.sheetnames}"
+    )
     for sheetname in wb2.sheetnames:
         sheet_1 = wb1[sheetname]
         sheet_2 = wb2[sheetname]