Filling XLSX: Everything except multiple choice.

Summary

For caosdb/customers/dimr/management#219:

Filling XLSX templates from JSON works now.

Focus

• Correct handling of foreign keys
• Correct handling of other lists

Test Environment

Run the unit tests.

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Up-to-date JSON schema (or not necessary)
• Appropriate user and developer documentation (or not necessary)
  • How do I use the software? Assume "stupid" users.
  • How do I develop or debug the software? Assume novice developers.
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• Appropriate user and developer documentation (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/caosadvancedtools/table_json_conversion/fill_xlsx.py

@@ -27,10 +27,12 @@ import pathlib
 from collections import OrderedDict
 from types import SimpleNamespace
 from typing import Any, Dict, List, Optional, TextIO, Union
+from warnings import warn
 
 from jsonschema import FormatChecker, validate
 from jsonschema.exceptions import ValidationError
 from openpyxl import Workbook, load_workbook
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
 from openpyxl.worksheet.worksheet import Worksheet
 
 from .table_generator import ColumnType, RowType
@@ -134,8 +136,9 @@ def _read_or_dict(data: Union[dict, str, TextIO]) -> dict:
 class TemplateFiller:
     """Class to fill XLSX templates.  Has an index for all relevant columns."""
 
-    def __init__(self, workbook: Workbook):
+    def __init__(self, workbook: Workbook, graceful: bool = False):
         self._workbook = workbook
+        self._graceful = graceful
         self._create_index()
 
     @property
@@ -282,7 +285,7 @@ out: union[dict, None]
             if isinstance(content, list):
                 if not content:  # empty list
                     continue
-                # list elements must be all of the same type.
+                # List elements must be all of the same type.
                 assert len(set(type(entry) for entry in content)) == 1
 
                 if isinstance(content[0], dict):  # all elements are dicts
@@ -304,24 +307,29 @@ out: union[dict, None]
                 content = [content]  # make list for unified treatment below
 
             # collecting the data
-            assert isinstance(content, list)  # TODO do we want this??? make list non-lists?
-            if len(content) == 1:
-                value = content[0]
-            else:
+            assert isinstance(content, list)
+            if len(content) > 1:
+                content = [ILLEGAL_CHARACTERS_RE.sub("", str(x)) for x in content]
                 value = ";".join(content)  # TODO we need escaping of values
+            else:
+                value = content[0]
+                if isinstance(value, str):
+                    value = ILLEGAL_CHARACTERS_RE.sub("", value)
             path_str = p2s(path)
             assert path_str not in insertables
             insertables[path_str] = value
         if only_collect_insertables:
             return insertables
-        if not current_path:  # top level returns (?)
+        if not current_path:  # Top level returns, because there are only sheets for the children.
             return None
 
         # actual data insertion
         insert_row = None
         sheet = None
         for path_str, value in insertables.items():
-
+            if self._graceful and path_str not in self._sheet_index:
+                warn(f"Ignoring path with missing sheet index: {path_str}")
+                continue
             sheet_meta = self._sheet_index[path_str]
             if sheet is None:
                 sheet = sheet_meta.sheet
@@ -363,7 +371,8 @@ result: str
   Path for the result XLSX.
 validation_schema: dict, optional
   If given, validate the date against this schema first.  This raises an exception if the validation
-  fails.
+  fails.  If no validation schema is given, try to ignore more errors in the data when filling the
+  XLSX template.
 """
     data = _read_or_dict(data)
     assert isinstance(data, dict)
@@ -376,10 +385,12 @@ validation_schema: dict, optional
         except ValidationError as ve:
             print(ve.message)
             raise RuntimeError("Validation failed")
+    else:
+        print("No validation schema given, continue at your own risk.")
 
     # Filling the data
     result_wb = load_workbook(template)
-    template_filler = TemplateFiller(result_wb)
+    template_filler = TemplateFiller(result_wb, graceful=(validation_schema is None))
     template_filler.fill_data(data=data)
 
     parentpath = pathlib.Path(result).parent