ENH: Handling illegal characters.

src/caosadvancedtools/table_json_conversion/fill_xlsx.py

@@ -32,6 +32,7 @@ from warnings import warn
 from jsonschema import FormatChecker, validate
 from jsonschema.exceptions import ValidationError
 from openpyxl import Workbook, load_workbook
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
 from openpyxl.worksheet.worksheet import Worksheet
 
 from .table_generator import ColumnType, RowType
@@ -332,7 +333,8 @@ out: union[dict, None]
             if insert_row is None:
                 insert_row = _next_row_index(sheet)
 
-            sheet.cell(row=insert_row+1, column=col_index+1, value=value)
+            sheet.cell(row=insert_row+1, column=col_index+1,
+                       value=ILLEGAL_CHARACTERS_RE.sub("", value))
 
         # Insert foreign keys
         if insert_row is not None and sheet is not None and _is_exploded_sheet(sheet):
@@ -343,7 +345,8 @@ out: union[dict, None]
                 raise
             for index, path in ((f.index, f.path) for f in foreigns.values()):
                 value = context[path]
-                sheet.cell(row=insert_row+1, column=index+1, value=value)
+                sheet.cell(row=insert_row+1, column=index+1,
+                           value=ILLEGAL_CHARACTERS_RE.sub("", value))
 
         return None
 

unittests/table_json_conversion/data/simple_data_ascii_chars.json

 {
   "Training": {
     "date": "2023-01-01",
-    "url": "char: \u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009",
+    "url": "char: >\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009<",
     "subjects": [
-      "\u000a\u000b\u000c\u000d\u000e\u000f",
-      "\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017",
-      "\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f",
-      "\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027",
-      "\u0028\u0029\u002a\u002b\u002c\u002d\u002e\u002f"
+      ">\u000a\u000b\u000c\u000e\u000f<",
+      ">\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017<",
+      ">\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f<",
+      ">\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027<",
+      ">\u0028\u0029\u002a\u002b\u002c\u002d\u002e\u002f<"
     ]
   },
   "Person": {
-    "family_name": "Steve:",
+    "family_name": "Steve",
     "given_name": "Stevie",
     "Organisation": "IMF"
   }

unittests/table_json_conversion/test_fill_xlsx.py

@@ -134,10 +134,9 @@ def test_fill_xlsx():
                      template_file=rfp("data/indirect_template.xlsx"),
                      known_good=rfp("data/indirect_data.xlsx"),
                      schema=rfp("data/indirect_schema.json"))
-
     fill_and_compare(json_file=rfp("data/simple_data_ascii_chars.json"),
                      template_file=rfp("data/simple_template.xlsx"),
-                     known_good=rfp("data/simple_data.xlsx"),
+                     known_good=rfp("data/simple_data_ascii_chars.xlsx"),
                      schema=rfp("data/simple_schema.json"))