Skip to content
Snippets Groups Projects
Verified Commit b1573034 authored by Daniel Hornung's avatar Daniel Hornung
Browse files

ENH: More extensive testing data, better foreign key naming.

parent ccda768b
Branches f-more-jsonschema-export
No related tags found
3 merge requests!100WIP: Filling XLSX: Seems to be working.,!94ENH: add framework for converting json schema into table templates,!93Filling XLSX: Everything except multiple choice.
Pipeline #48172 failed
Stage: setup
Stage: cert
Stage: style
Stage: unittest
Stage: integrationtest
Hide whitespace changes
Inline Side-by-side
Showing
with 105 additions and 135 deletions
src/caosadvancedtools/table_json_conversion/table_generator.py
+ 8
− 6
View file @ b1573034
...@@ -198,13 +198,15 @@ class TableTemplateGenerator(ABC): ...@@ -198,13 +198,15 @@ class TableTemplateGenerator(ABC):
) )
# and add the foreign keys that are necessary up to this point # and add the foreign keys that are necessary up to this point
for array_path in array_paths: for array_path in array_paths:
keys = self._get_foreign_keys(foreign_keys, array_path) foreigns = self._get_foreign_keys(foreign_keys, array_path)
for key in keys: for foreign in foreigns:
if key in sheets[sheetname]: internal_key = ".".join(array_path + [foreign])
if internal_key in sheets[sheetname]:
raise ValueError("The schema would lead to two columns with the same " raise ValueError("The schema would lead to two columns with the same "
f"name which is forbidden: {key}") f"name, which is forbidden: {internal_key}")
sheets[sheetname][key] = (ColumnType.FOREIGN, f"see sheet '{path[0]}'", ref_sheet = ".".join(array_path)
array_path + [key]) sheets[sheetname][internal_key] = (
ColumnType.FOREIGN, f"see sheet '{ref_sheet}'", array_path + [foreign])
# Columns are added to the new sheet, thus we do not return any columns for the # Columns are added to the new sheet, thus we do not return any columns for the
# current sheet. # current sheet.
return {} return {}
......
unittests/table_json_conversion/example_template.xlsx
+ 0
− 0
View file @ b1573034
No preview for this file type
unittests/table_json_conversion/model_multiple_refs.yml
+ 4
− 1
View file @ b1573034
...@@ -4,7 +4,6 @@ Person: ...@@ -4,7 +4,6 @@ Person:
datatype: TEXT datatype: TEXT
email: email:
datatype: TEXT datatype: TEXT
Organisation:
Training: Training:
recommended_properties: recommended_properties:
date: date:
...@@ -21,6 +20,8 @@ Training: ...@@ -21,6 +20,8 @@ Training:
datatype: Person datatype: Person
responsible: responsible:
datatype: Person datatype: Person
Organisation:
datatype: LIST<Organisation>
supervisor_inherit: supervisor_inherit:
inherit_from_suggested: inherit_from_suggested:
- Person - Person
...@@ -31,3 +32,5 @@ Organisation: ...@@ -31,3 +32,5 @@ Organisation:
recommended_properties: recommended_properties:
Country: Country:
datatype: TEXT datatype: TEXT
Person:
datatype: LIST<Person>
unittests/table_json_conversion/multiple_refs.xlsx 0 → 100644
+ 0
− 0
View file @ b1573034
File added
unittests/table_json_conversion/schema_multiple_refs.json
+ 37
− 103
View file @ b1573034
...@@ -45,21 +45,6 @@ ...@@ -45,21 +45,6 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
} }
} }
} }
...@@ -81,21 +66,6 @@ ...@@ -81,21 +66,6 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
} }
} }
} }
...@@ -115,21 +85,6 @@ ...@@ -115,21 +85,6 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
} }
} }
}, },
...@@ -148,19 +103,43 @@ ...@@ -148,19 +103,43 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
}, }
"Organisation": { }
"type": "object", },
"required": [], "Organisation": {
"additionalProperties": false, "type": "array",
"title": "Organisation", "items": {
"properties": { "type": "object",
"name": { "required": [],
"type": "string", "additionalProperties": false,
"description": "The name of the Record to be created" "title": "Organisation",
}, "properties": {
"Country": { "name": {
"type": "string" "type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
},
"Person": {
"type": "array",
"items": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Person",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"full_name": {
"type": "string"
},
"email": {
"type": "string"
}
}
} }
} }
} }
...@@ -181,21 +160,6 @@ ...@@ -181,21 +160,6 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
} }
} }
}, },
...@@ -214,21 +178,6 @@ ...@@ -214,21 +178,6 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
} }
} }
} }
...@@ -250,21 +199,6 @@ ...@@ -250,21 +199,6 @@
}, },
"email": { "email": {
"type": "string" "type": "string"
},
"Organisation": {
"type": "object",
"required": [],
"additionalProperties": false,
"title": "Organisation",
"properties": {
"name": {
"type": "string",
"description": "The name of the Record to be created"
},
"Country": {
"type": "string"
}
}
} }
}, },
"$schema": "https://json-schema.org/draft/2020-12/schema" "$schema": "https://json-schema.org/draft/2020-12/schema"
......
unittests/table_json_conversion/test_table_template_generator.py
+ 56
− 25
View file @ b1573034
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
import json import json
import os import os
import tempfile import tempfile
from typing import Tuple
import pytest import pytest
from caosadvancedtools.table_json_conversion.table_generator import ( from caosadvancedtools.table_json_conversion.table_generator import (
...@@ -37,6 +38,47 @@ def rfp(*pathcomponents): ...@@ -37,6 +38,47 @@ def rfp(*pathcomponents):
return os.path.join(os.path.dirname(__file__), *pathcomponents) return os.path.join(os.path.dirname(__file__), *pathcomponents)
def _compare_generated_to_known_good(schema_file: str, known_good: str, foreign_keys: dict = None,
outfile: str = None) -> Tuple:
"""Generate an XLSX from the schema, then compare to known good output.
Returns
-------
out: tuple
The generated and known good workbook objects.
"""
generator = XLSXTemplateGenerator()
if foreign_keys is None:
foreign_keys = {}
with open(schema_file, encoding="utf-8") as schema_input:
schema = json.load(schema_input)
if outfile is None:
outpath = os.path.join(tempfile.mkdtemp(), 'generated.xlsx')
else:
outpath = outfile
assert not os.path.exists(outpath)
generator.generate(schema=schema,
foreign_keys=foreign_keys,
filepath=outpath)
assert os.path.exists(outpath)
generated = load_workbook(outpath) # workbook can be read
good = load_workbook(known_good)
assert generated.sheetnames == good.sheetnames
for sheetname in good.sheetnames:
gen_sheet = generated[sheetname]
good_sheet = good[sheetname]
for irow, (erow, grow) in enumerate(zip(good_sheet.iter_rows(), gen_sheet.iter_rows())):
assert (good_sheet.row_dimensions[irow].hidden
== gen_sheet.row_dimensions[irow].hidden), f"row: {sheetname}, {irow}"
for icol, (ecol, gcol) in enumerate(zip(erow, grow)):
assert (good_sheet.column_dimensions[ecol.column_letter].hidden
== gen_sheet.column_dimensions[ecol.column_letter].hidden), (
f"col: {sheetname}, {icol}")
cell = gen_sheet.cell(irow+1, icol+1)
assert ecol.value == gcol.value, f"Sheet: {sheetname}, cell: {cell.coordinate}"
return generated, good
def test_generate_sheets_from_schema(): def test_generate_sheets_from_schema():
# trivial case; we do not support this # trivial case; we do not support this
schema = {} schema = {}
...@@ -133,8 +175,8 @@ def test_generate_sheets_from_schema(): ...@@ -133,8 +175,8 @@ def test_generate_sheets_from_schema():
assert cdef['given_name'] == (ColumnType.SCALAR, None, ["Training", 'coach', 'given_name']) assert cdef['given_name'] == (ColumnType.SCALAR, None, ["Training", 'coach', 'given_name'])
assert cdef['Organisation'] == (ColumnType.SCALAR, None, ["Training", 'coach', assert cdef['Organisation'] == (ColumnType.SCALAR, None, ["Training", 'coach',
'Organisation']) 'Organisation'])
assert cdef['date'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'date']) assert cdef['Training.date'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'date'])
assert cdef['url'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'url']) assert cdef['Training.url'] == (ColumnType.FOREIGN, "see sheet 'Training'", ["Training", 'url'])
def test_get_foreign_keys(): def test_get_foreign_keys():
...@@ -163,29 +205,10 @@ def test_get_max_path_length(): ...@@ -163,29 +205,10 @@ def test_get_max_path_length():
def test_template_generator(): def test_template_generator():
generator = XLSXTemplateGenerator() generated, _ = _compare_generated_to_known_good(
with open(rfp("model_schema.json")) as sfi: schema_file=rfp("model_schema.json"), known_good=rfp("example_template.xlsx"),
schema = json.load(sfi) foreign_keys={'Training': {"__this__": ['date', 'url']}},
path = os.path.join(tempfile.mkdtemp(), 'test.xlsx') outfile=None)
assert not os.path.exists(path)
generator.generate(schema=schema,
foreign_keys={'Training': {"__this__": ['date', 'url']}},
filepath=path)
assert os.path.exists(path)
generated = load_workbook(path) # workbook can be read
example = load_workbook(rfp("example_template.xlsx"))
assert generated.sheetnames == example.sheetnames
for sheetname in example.sheetnames:
gen_sheet = generated[sheetname]
ex_sheet = example[sheetname]
for irow, (erow, grow) in enumerate(zip(ex_sheet.iter_rows(), gen_sheet.iter_rows())):
assert ex_sheet.row_dimensions[irow].hidden == gen_sheet.row_dimensions[irow].hidden
for icol, (ecol, gcol) in enumerate(zip(erow, grow)):
assert (ex_sheet.column_dimensions[ecol.column_letter].hidden
== gen_sheet.column_dimensions[ecol.column_letter].hidden)
cell = gen_sheet.cell(irow+1, icol+1)
assert ecol.value == gcol.value, f"Sheet: {sheetname}, cell: {cell.coordinate}"
# test some hidden # test some hidden
ws = generated.active ws = generated.active
assert ws.row_dimensions[1].hidden is True assert ws.row_dimensions[1].hidden is True
...@@ -212,3 +235,11 @@ def test_template_generator(): ...@@ -212,3 +235,11 @@ def test_template_generator():
# TODO test escaping of values # TODO test escaping of values
# TODO finish enum example # TODO finish enum example
def test_model_with_multiple_refs():
_compare_generated_to_known_good(
schema_file=rfp("schema_multiple_refs.json"), known_good=rfp("multiple_refs.xlsx"),
foreign_keys={'Training': {"__this__": ['date', 'url'],
"Organisation": ["name"]}},
outfile=None)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment