Skip to content
Snippets Groups Projects
Commit 3931c9bd authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

WIP

parent eea46658
No related branches found
No related tags found
2 merge requests!100WIP: Filling XLSX: Seems to be working.,!93Filling XLSX: Everything except multiple choice.
Pipeline #48437 failed
...@@ -26,10 +26,11 @@ import json ...@@ -26,10 +26,11 @@ import json
import pathlib import pathlib
from collections import OrderedDict from collections import OrderedDict
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any, Dict, List, Optional, Union, TextIO from typing import Any, Dict, List, Optional, TextIO, Union
from jsonschema import FormatChecker, validate from jsonschema import FormatChecker, validate
from openpyxl import load_workbook, Workbook from jsonschema.exceptions import ValidationError
from openpyxl import Workbook, load_workbook
from openpyxl.worksheet.worksheet import Worksheet from openpyxl.worksheet.worksheet import Worksheet
from .table_generator import ColumnType, RowType from .table_generator import ColumnType, RowType
...@@ -183,6 +184,7 @@ class TemplateFiller: ...@@ -183,6 +184,7 @@ class TemplateFiller:
self._props[fullpath] = value self._props[fullpath] = value
def fill_from_data(self, data: Dict[str, Any]): def fill_from_data(self, data: Dict[str, Any]):
# TODO recursive for dicts and list?
"""Fill current level with all scalar elements of ``data``.""" """Fill current level with all scalar elements of ``data``."""
for name, value in data.items(): for name, value in data.items():
if not isinstance(value, (dict, list)): if not isinstance(value, (dict, list)):
...@@ -272,6 +274,9 @@ out: union[dict, None] ...@@ -272,6 +274,9 @@ out: union[dict, None]
insertables: Dict[str, Any] = {} insertables: Dict[str, Any] = {}
for name, content in data.items(): for name, content in data.items():
# TODO is this the best way to do it????
if name == "file":
continue
path = current_path + [name] path = current_path + [name]
next_context = context.next_level(name) next_context = context.next_level(name)
# preprocessing # preprocessing
...@@ -328,7 +333,11 @@ out: union[dict, None] ...@@ -328,7 +333,11 @@ out: union[dict, None]
# Insert foreign keys # Insert foreign keys
if insert_row is not None and sheet is not None and _is_exploded_sheet(sheet): if insert_row is not None and sheet is not None and _is_exploded_sheet(sheet):
try:
foreigns = _get_foreign_key_columns(sheet) foreigns = _get_foreign_key_columns(sheet)
except ValueError:
print(f"Sheet: {sheet}")
raise
for index, path in ((f.index, f.path) for f in foreigns.values()): for index, path in ((f.index, f.path) for f in foreigns.values()):
value = context[path] value = context[path]
sheet.cell(row=insert_row+1, column=index+1, value=value) sheet.cell(row=insert_row+1, column=index+1, value=value)
...@@ -361,7 +370,11 @@ validation_schema: dict, optional ...@@ -361,7 +370,11 @@ validation_schema: dict, optional
# Validation # Validation
if validation_schema is not None: if validation_schema is not None:
validation_schema = _read_or_dict(validation_schema) validation_schema = _read_or_dict(validation_schema)
try:
validate(data, validation_schema, format_checker=FormatChecker()) validate(data, validation_schema, format_checker=FormatChecker())
except ValidationError as ve:
print(ve.message)
raise RuntimeError("Validation failed")
# Filling the data # Filling the data
result_wb = load_workbook(template) result_wb = load_workbook(template)
......
...@@ -196,11 +196,13 @@ class TableTemplateGenerator(ABC): ...@@ -196,11 +196,13 @@ class TableTemplateGenerator(ABC):
if sheetname in sheets: if sheetname in sheets:
raise ValueError("The schema would lead to two sheets with the same name, " raise ValueError("The schema would lead to two sheets with the same name, "
f"which is forbidden: {sheetname}") f"which is forbidden: {sheetname}")
sheets[sheetname] = self._treat_schema_element( col_def = self._treat_schema_element(
schema=schema['items'], sheets=sheets, path=path, foreign_keys=foreign_keys, schema=schema['items'], sheets=sheets, path=path, foreign_keys=foreign_keys,
level_in_sheet_name=len(path), level_in_sheet_name=len(path),
array_paths=array_paths+[path] # since this level is an array extend the list array_paths=array_paths+[path] # since this level is an array extend the list
) )
if col_def:
sheets[sheetname] = col_def
# and add the foreign keys that are necessary up to this point # and add the foreign keys that are necessary up to this point
for array_path in array_paths: for array_path in array_paths:
foreigns = self._get_foreign_keys(foreign_keys, array_path) foreigns = self._get_foreign_keys(foreign_keys, array_path)
...@@ -326,6 +328,8 @@ class XLSXTemplateGenerator(TableTemplateGenerator): ...@@ -326,6 +328,8 @@ class XLSXTemplateGenerator(TableTemplateGenerator):
del wb['Sheet'] del wb['Sheet']
for sheetname, sheetdef in sheets.items(): for sheetname, sheetdef in sheets.items():
if not sheetdef:
continue
ws = wb.create_sheet(re.sub(INVALID_TITLE_REGEX, '_', sheetname)) ws = wb.create_sheet(re.sub(INVALID_TITLE_REGEX, '_', sheetname))
# First row will by the COL_TYPE row. # First row will by the COL_TYPE row.
# First column will be the indicator row with values COL_TYPE, PATH, IGNORE. # First column will be the indicator row with values COL_TYPE, PATH, IGNORE.
......
...@@ -19,7 +19,9 @@ ...@@ -19,7 +19,9 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import json
import os import os
import re
import tempfile import tempfile
import jsonschema.exceptions as schema_exc import jsonschema.exceptions as schema_exc
...@@ -69,6 +71,56 @@ def test_detect(): ...@@ -69,6 +71,56 @@ def test_detect():
assert [1, 2] == _get_path_rows(example['Person']) assert [1, 2] == _get_path_rows(example['Person'])
def test_temporary():
# TODO: remove the following after manual testing
di = '/home/henrik/CaosDB/management/external/dimr/eingabemaske/crawler/schemas'
dd = '/home/henrik/CaosDB/management/external/dimr/eingabemaske/django/laforms/persistent/'
allreadydone = [
"Präventionsmaßnahmen",
"Beratungsstellen",
"Schutzeinrichtungen",
"Einzelfallversorgung",
"Strategiedokumente",
"Kooperationsvereinbarungen",
"Gremien",
"Verwaltungsvorschriften",
"Gewaltschutzkonzepte und -maßnahmen",
"Polizeilicher Opferschutz",
"Feedback",
]
for prefix, _, files in os.walk(dd):
for fi in files:
match = re.match(r"(?P<teilb>.*)_2024-.*\.json", fi)
if match:
print(match.group('teilb'))
tb = match.group('teilb')
if tb in allreadydone:
continue
# allreadydone.append(tb)
template = os.path.join(di, "template_"+tb+".xlsx")
schema = os.path.join(di, "schema_"+tb+".json")
if not os.path.exists(template):
print(template)
assert False
jfi = os.path.join(prefix, fi)
print(jfi)
if not fi.startswith("Art"):
continue
# if jfi != "/home/henrik/CaosDB/management/external/dimr/eingabemaske/django/laforms/persistent/data/datenhalterin_gg/he_gg_2/Art__13_Bewusstseinsbildung_2024-01-11T10:22:26.json":
# continue
with open(jfi, encoding="utf-8") as infile:
data = json.load(infile)
data = data["form_data"]
if "__version__" in data:
del data["__version__"]
with tempfile.TemporaryDirectory() as tmpdir:
outfile = os.path.join(tmpdir, 'test.xlsx')
fill_template(data=data, template=template, result=outfile,
validation_schema=schema)
os.system(f'libreoffice {outfile}')
def test_fill_xlsx(): def test_fill_xlsx():
fill_and_compare(json_file=rfp("data/simple_data.json"), fill_and_compare(json_file=rfp("data/simple_data.json"),
template_file=rfp("data/simple_template.xlsx"), template_file=rfp("data/simple_template.xlsx"),
...@@ -78,6 +130,7 @@ def test_fill_xlsx(): ...@@ -78,6 +130,7 @@ def test_fill_xlsx():
template_file=rfp("data/multiple_refs_template.xlsx"), template_file=rfp("data/multiple_refs_template.xlsx"),
known_good=rfp("data/multiple_refs_data.xlsx"), known_good=rfp("data/multiple_refs_data.xlsx"),
schema=rfp("data/multiple_refs_schema.json")) schema=rfp("data/multiple_refs_schema.json"))
fill_and_compare(json_file=rfp("data/indirect_data.json"), fill_and_compare(json_file=rfp("data/indirect_data.json"),
template_file=rfp("data/indirect_template.xlsx"), template_file=rfp("data/indirect_template.xlsx"),
known_good=rfp("data/indirect_data.xlsx"), known_good=rfp("data/indirect_data.xlsx"),
......
...@@ -206,21 +206,44 @@ def test_template_generator(): ...@@ -206,21 +206,44 @@ def test_template_generator():
assert ws.column_dimensions['A'].hidden is True assert ws.column_dimensions['A'].hidden is True
# TODO: remove the following after manual testing # TODO: remove the following after manual testing
di = '/home/professional/CaosDB/management/external/dimr/eingabemaske/crawler/schemas' di = '/home/henrik/CaosDB/management/external/dimr/eingabemaske/crawler/schemas'
if not os.path.exists(di): if not os.path.exists(di):
return return
for fi in os.listdir(di): for fi in os.listdir(di):
rp = os.path.join(di, fi) rp = os.path.join(di, fi)
if not fi.startswith("schema_"):
continue
with open(rp) as sfi: with open(rp) as sfi:
schema = json.load(sfi) schema = json.load(sfi)
fk_path = os.path.join(di, "foreign_keys"+fi[len('schema'):]) fk_path = os.path.join(di, "foreign_keys"+fi[len('schema'):])
if not os.path.exists(fk_path): path =os.path.join(di, "template"+fi[len('schema'):-4]+"xlsx")
print(f"No foreign keys file for:\n{rp}") allreadydone = [
"Präventionsmaßnahmen" ,
"Beratungsstellen" ,
"Schutzeinrichtungen",
"Einzelfallversorgung" ,
"Strategiedokumente" ,
"Kooperationsvereinbarungen" ,
"Gremien" ,
"Verwaltungsvorschriften" ,
"Gewaltschutzkonzepte und -maßnahmen",
"Polizeilicher Opferschutz",
"Feedback",
]
if any([path.startswith("template_"+k) for k in allreadydone]):
continue continue
if not os.path.exists(fk_path):
print(f"No foreign keys file for:\n{fk_path}")
assert False
with open(fk_path) as sfi: with open(fk_path) as sfi:
fk = json.load(sfi) fk = json.load(sfi)
generator = XLSXTemplateGenerator()
if not os.path.exists(path):
generator.generate(schema=schema, foreign_keys=fk, filepath=path) generator.generate(schema=schema, foreign_keys=fk, filepath=path)
os.system(f'libreoffice {path}') os.system(f'libreoffice {path}')
else:
print(f"Not creating template because it exists:\n{path}")
# TODO test collisions of sheet or colnames # TODO test collisions of sheet or colnames
# TODO test escaping of values # TODO test escaping of values
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment