Skip to content
Snippets Groups Projects
Verified Commit b1573034 authored by Daniel Hornung's avatar Daniel Hornung
Browse files

ENH: More extensive testing data, better foreign key naming.

parent ccda768b
Branches f-more-jsonschema-export
No related tags found
3 merge requests!100WIP: Filling XLSX: Seems to be working.,!94ENH: add framework for converting json schema into table templates,!93Filling XLSX: Everything except multiple choice.
Pipeline #48172 failed
Stage: setup
Stage: cert
Stage: style
Stage: unittest
Stage: integrationtest
Hide whitespace changes
Inline Side-by-side
Showing
with 105 additions and 135 deletions
src/caosadvancedtools/table_json_conversion/table_generator.py
+ 8
− 6
View file @ b1573034
......@@ -198,13 +198,15 @@ class TableTemplateGenerator(ABC):
)
# and add the foreign keys that are necessary up to this point
for array_path in array_paths:
keys = self._get_foreign_keys(foreign_keys, array_path)
for key in keys:
if key in sheets[sheetname]:
foreigns = self._get_foreign_keys(foreign_keys, array_path)
for foreign in foreigns:
internal_key = ".".join(array_path + [foreign])
if internal_key in sheets[sheetname]:
raise ValueError("The schema would lead to two columns with the same "
f"name which is forbidden: {key}")
sheets[sheetname][key] = (ColumnType.FOREIGN, f"see sheet '{path[0]}'",
array_path + [key])
f"name, which is forbidden: {internal_key}")
ref_sheet = ".".join(array_path)
sheets[sheetname][internal_key] = (
ColumnType.FOREIGN, f"see sheet '{ref_sheet}'", array_path + [foreign])
# Columns are added to the new sheet, thus we do not return any columns for the
# current sheet.
return {}
......
unittests/table_json_conversion/example_template.xlsx
+ 0
− 0
View file @ b1573034
No preview for this file type
unittests/table_json_conversion/model_multiple_refs.yml
+ 4
− 1
View file @ b1573034
......@@ -4,7 +4,6 @@ Person:
datatype: TEXT
email:
datatype: TEXT
Organisation:
Training:
recommended_properties:
date:
......@@ -21,6 +20,8 @@ Training:
datatype: Person
responsible:
datatype: Person
Organisation:
datatype: LIST<Organisation>
supervisor_inherit:
inherit_from_suggested:
- Person
......@@ -31,3 +32,5 @@ Organisation:
recommended_properties:
Country:
datatype: TEXT
Person:
datatype: LIST<Person>
unittests/table_json_conversion/multiple_refs.xlsx 0 → 100644
+ 0
− 0
View file @ b1573034
File added
unittests/table_json_conversion/schema_multiple_refs.json
+ 37
− 103
View file @ b1573034
......@@ -45,21 +45,6 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
}
}
}
......@@ -81,21 +66,6 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
}
}
}
......@@ -115,21 +85,6 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
}
}
},
......@@ -148,19 +103,43 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
},
"Organisation": {
"type": "array",
"items": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
},
"Person": {
"type": "array",
"items": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Person",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"full_name": {
"type": "string"
},
"email": {
"type": "string"
}
}
}
}
}
......@@ -181,21 +160,6 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
}
}
},
......@@ -214,21 +178,6 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
}
}
}
......@@ -250,21 +199,6 @@
},
"email": {
"type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema"
......
unittests/table_json_conversion/test_table_template_generator.py
+ 56
− 25
View file @ b1573034
......@@ -22,6 +22,7 @@
import json
import os
import tempfile
from typing import Tuple
import pytest
from caosadvancedtools.table_json_conversion.table_generator import (
......@@ -37,6 +38,47 @@ def rfp(*pathcomponents):
return os.path.join(os.path.dirname(__file__), *pathcomponents)
def _compare_generated_to_known_good(schema_file: str, known_good: str, foreign_keys: dict = None,
outfile: str = None) -> Tuple:
"""Generate an XLSX from the schema, then compare to known good output.
Returns
-------
out: tuple
The generated and known good workbook objects.
"""
generator = XLSXTemplateGenerator()
if foreign_keys is None:
foreign_keys = {}
with open(schema_file, encoding="utf-8") as schema_input:
schema = json.load(schema_input)
if outfile is None:
outpath = os.path.join(tempfile.mkdtemp(), 'generated.xlsx')
else:
outpath = outfile
assert not os.path.exists(outpath)
generator.generate(schema=schema,
foreign_keys=foreign_keys,
filepath=outpath)
assert os.path.exists(outpath)
generated = load_workbook(outpath) # workbook can be read
good = load_workbook(known_good)
assert generated.sheetnames == good.sheetnames
for sheetname in good.sheetnames:
gen_sheet = generated[sheetname]
good_sheet = good[sheetname]
for irow, (erow, grow) in enumerate(zip(good_sheet.iter_rows(), gen_sheet.iter_rows())):
assert (good_sheet.row_dimensions[irow].hidden
== gen_sheet.row_dimensions[irow].hidden), f"row: {sheetname}, {irow}"
for icol, (ecol, gcol) in enumerate(zip(erow, grow)):
assert (good_sheet.column_dimensions[ecol.column_letter].hidden
== gen_sheet.column_dimensions[ecol.column_letter].hidden), (
f"col: {sheetname}, {icol}")
cell = gen_sheet.cell(irow+1, icol+1)
assert ecol.value == gcol.value, f"Sheet: {sheetname}, cell: {cell.coordinate}"
return generated, good
def test_generate_sheets_from_schema():
# trivial case; we do not support this
schema = {}
......@@ -133,8 +175,8 @@ def test_generate_sheets_from_schema():
assert cdef['given_name'] == (ColumnType.SCALAR, None, ["Training", 'coach', 'given_name'])
assert cdef['Organisation'] == (ColumnType.SCALAR, None, ["Training", 'coach',
'Organisation'])
assert cdef['date'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'date'])
assert cdef['url'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'url'])
assert cdef['Training.date'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'date'])
assert cdef['Training.url'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'url'])
def test_get_foreign_keys():
......@@ -163,29 +205,10 @@ def test_get_max_path_length():
def test_template_generator():
generator = XLSXTemplateGenerator()
with open(rfp("model_schema.json")) as sfi:
schema = json.load(sfi)
path = os.path.join(tempfile.mkdtemp(), 'test.xlsx')
assert not os.path.exists(path)
generator.generate(schema=schema,
foreign_keys={'Training': {"__this__": ['date', 'url']}},
filepath=path)
assert os.path.exists(path)
generated = load_workbook(path) # workbook can be read
example = load_workbook(rfp("example_template.xlsx"))
assert generated.sheetnames == example.sheetnames
for sheetname in example.sheetnames:
gen_sheet = generated[sheetname]
ex_sheet = example[sheetname]
for irow, (erow, grow) in enumerate(zip(ex_sheet.iter_rows(), gen_sheet.iter_rows())):
assert ex_sheet.row_dimensions[irow].hidden == gen_sheet.row_dimensions[irow].hidden
for icol, (ecol, gcol) in enumerate(zip(erow, grow)):
assert (ex_sheet.column_dimensions[ecol.column_letter].hidden
== gen_sheet.column_dimensions[ecol.column_letter].hidden)
cell = gen_sheet.cell(irow+1, icol+1)
assert ecol.value == gcol.value, f"Sheet: {sheetname}, cell: {cell.coordinate}"
generated, _ = _compare_generated_to_known_good(
schema_file=rfp("model_schema.json"), known_good=rfp("example_template.xlsx"),
foreign_keys={'Training': {"__this__": ['date', 'url']}},
outfile=None)
# test some hidden
ws = generated.active
assert ws.row_dimensions[1].hidden is True
......@@ -212,3 +235,11 @@ def test_template_generator():
# TODO test escaping of values
# TODO finish enum example
def test_model_with_multiple_refs():
_compare_generated_to_known_good(
schema_file=rfp("schema_multiple_refs.json"), known_good=rfp("multiple_refs.xlsx"),
foreign_keys={'Training': {"__this__": ['date', 'url'],
"Organisation": ["name"]}},
outfile=None)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment