Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
C
caosdb-advanced-user-tools
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
caosdb
Software
caosdb-advanced-user-tools
Commits
1a34df84
Verified
Commit
1a34df84
authored
1 year ago
by
Daniel Hornung
Browse files
Options
Downloads
Patches
Plain Diff
ENH: JsonSchemaExporter accepts do_not_create parameter.
Also moved it into a class.
parent
85757cf3
No related branches found
No related tags found
2 merge requests
!89
ENH: JsonSchemaExporter accepts do_not_create parameter.
,
!82
ENH: JsonSchemaExporter accepts do_not_create parameter.
Pipeline
#43061
failed
1 year ago
Stage: setup
Stage: cert
Stage: style
Stage: unittest
Stage: integrationtest
Changes
2
Pipelines
1
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
src/caosadvancedtools/json_schema_exporter.py
+241
-195
241 additions, 195 deletions
src/caosadvancedtools/json_schema_exporter.py
unittests/test_json_schema_exporter.py
+2
-0
2 additions, 0 deletions
unittests/test_json_schema_exporter.py
with
243 additions
and
195 deletions
src/caosadvancedtools/json_schema_exporter.py
+
241
−
195
View file @
1a34df84
...
@@ -19,55 +19,82 @@
...
@@ -19,55 +19,82 @@
# You should have received a copy of the GNU Affero General Public License along
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.
# with this program. If not, see <https://www.gnu.org/licenses/>.
#
#
"""
Module for converting a data model into a json schema compatible dictionary.
"""
import
re
from
typing
import
Any
,
Optional
from
typing
import
Optional
import
linkahead
as
db
import
linkahead
as
db
from
linkahead.common.datatype
import
get_list_datatype
,
is_list_datatype
from
linkahead.common.datatype
import
get_list_datatype
,
is_list_datatype
def
_make_required_list
(
rt
:
db
.
RecordType
):
class
JsonSchemaExporter
:
"""
Return the list of names of properties with importance db.OBLIGATORY.
"""
"""
A class which collects everything needed for the conversion.
return
[
prop
.
name
for
prop
in
rt
.
properties
"""
if
rt
.
get_importance
(
prop
.
name
)
==
db
.
OBLIGATORY
]
def
_make_segment_from_prop
(
prop
:
db
.
Property
,
additional_properties
:
bool
,
def
__init__
(
self
,
additional_properties
:
bool
=
True
,
name_and_description_in_properties
:
bool
,
name_and_description_in_properties
:
bool
=
False
,
additional_options_for_text_props
:
Optional
[
dict
],
additional_options_for_text_props
:
dict
=
None
,
units_in_description
:
bool
):
units_in_description
:
bool
=
True
,
"""
Return the JSON Schema segment for the given property
do_not_create
:
list
[
str
]
=
None
,
):
"""
Set up a JsonSchemaExporter, which can then be applied on RecordTypes.
Parameters
Parameters
----------
----------
prop : db.Property
the property to be transformed
additional_properties : bool, optional
additional_properties : bool, optional
Whether additional properties will be admitted in the resulting
Whether additional properties will be admitted in the resulting
schema. Optional, default is True.
schema. Optional, default is True.
name_and_description_in_properties : bool, optional
name_and_description_in_properties : bool, optional
Whether to include name and description in the `properties` section of
Whether to include name and description in the `properties` section of
the schema to be exported. Optional, default is False.
the schema to be exported. Optional, default is False.
additional_options_for_text_props : Optional[dict]
additional_options_for_text_props : dict, optional
dict of dicts that may contain the keys
'
pattern
'
and
'
format
'
to
Dictionary containing additional
"
pattern
"
or
"
format
"
options for
further define the rules for the JSON Schema segment
string-typed properties. Optional, default is empty.
units_in_description : bool
units_in_description : bool, optional
Whether to store the unit of a LinkAhead property in the description of
Whether to add the unit of a LinkAhead property (if it has any) to the
the corresponding json schema item or to create a separate `unit` key
description of the corresponding schema entry. If set to false, an
instead.
additional `unit` key is added to the schema itself which is purely
annotational and ignored, e.g., in validation. Default is True.
do_not_create : list[str]
A list of RedcordType names, for which there should be no option
to create them. Instead, only the choice of existing elements should
be given.
"""
"""
if
not
additional_options_for_text_props
:
if
not
additional_options_for_text_props
:
additional_options_for_text_props
=
{}
additional_options_for_text_props
=
{}
if
not
do_not_create
:
do_not_create
=
[]
self
.
_additional_properties
=
additional_properties
self
.
_name_and_description_in_properties
=
name_and_description_in_properties
self
.
_additional_options_for_text_props
=
additional_options_for_text_props
self
.
_units_in_description
=
units_in_description
self
.
_do_not_create
=
do_not_create
@staticmethod
def
_make_required_list
(
rt
:
db
.
RecordType
):
"""
Return the list of names of properties with importance db.OBLIGATORY.
"""
return
[
prop
.
name
for
prop
in
rt
.
properties
if
rt
.
get_importance
(
prop
.
name
)
==
db
.
OBLIGATORY
]
def
_make_segment_from_prop
(
self
,
prop
:
db
.
Property
):
"""
Return the JSON Schema segment for the given property
Parameters
----------
prop : db.Property
The property to be transformed.
"""
if
prop
.
datatype
==
db
.
TEXT
or
prop
.
datatype
==
db
.
DATETIME
:
if
prop
.
datatype
==
db
.
TEXT
or
prop
.
datatype
==
db
.
DATETIME
:
text_format
=
None
text_format
=
None
text_pattern
=
None
text_pattern
=
None
if
prop
.
name
in
additional_options_for_text_props
:
if
prop
.
name
in
self
.
_
additional_options_for_text_props
:
if
"
pattern
"
in
additional_options_for_text_props
[
prop
.
name
]:
if
"
pattern
"
in
self
.
_
additional_options_for_text_props
[
prop
.
name
]:
text_pattern
=
additional_options_for_text_props
[
prop
.
name
][
"
pattern
"
]
text_pattern
=
self
.
_
additional_options_for_text_props
[
prop
.
name
][
"
pattern
"
]
if
"
format
"
in
additional_options_for_text_props
[
prop
.
name
]:
if
"
format
"
in
self
.
_
additional_options_for_text_props
[
prop
.
name
]:
text_format
=
additional_options_for_text_props
[
prop
.
name
][
"
format
"
]
text_format
=
self
.
_
additional_options_for_text_props
[
prop
.
name
][
"
format
"
]
elif
prop
.
datatype
==
db
.
DATETIME
:
elif
prop
.
datatype
==
db
.
DATETIME
:
# Set the date or datetime format if only a pattern is given ...
# Set the date or datetime format if only a pattern is given ...
text_format
=
[
"
date
"
,
"
date-time
"
]
text_format
=
[
"
date
"
,
"
date-time
"
]
...
@@ -76,12 +103,12 @@ def _make_segment_from_prop(prop: db.Property, additional_properties: bool,
...
@@ -76,12 +103,12 @@ def _make_segment_from_prop(prop: db.Property, additional_properties: bool,
# options list.
# options list.
text_format
=
[
"
date
"
,
"
date-time
"
]
text_format
=
[
"
date
"
,
"
date-time
"
]
return
_make_text_property
(
prop
.
description
,
text_format
,
text_pattern
)
return
self
.
_make_text_property
(
prop
.
description
,
text_format
,
text_pattern
)
json_prop
=
{}
json_prop
=
{}
if
prop
.
description
:
if
prop
.
description
:
json_prop
[
"
description
"
]
=
prop
.
description
json_prop
[
"
description
"
]
=
prop
.
description
if
units_in_description
and
prop
.
unit
:
if
self
.
_
units_in_description
and
prop
.
unit
:
if
"
description
"
in
json_prop
:
if
"
description
"
in
json_prop
:
json_prop
[
"
description
"
]
+=
f
"
Unit is
{
prop
.
unit
}
.
"
json_prop
[
"
description
"
]
+=
f
"
Unit is
{
prop
.
unit
}
.
"
else
:
else
:
...
@@ -99,32 +126,31 @@ def _make_segment_from_prop(prop: db.Property, additional_properties: bool,
...
@@ -99,32 +126,31 @@ def _make_segment_from_prop(prop: db.Property, additional_properties: bool,
json_prop
[
"
type
"
]
=
"
array
"
json_prop
[
"
type
"
]
=
"
array
"
list_element_prop
=
db
.
Property
(
list_element_prop
=
db
.
Property
(
name
=
prop
.
name
,
datatype
=
get_list_datatype
(
prop
.
datatype
,
strict
=
True
))
name
=
prop
.
name
,
datatype
=
get_list_datatype
(
prop
.
datatype
,
strict
=
True
))
json_prop
[
"
items
"
]
=
_make_segment_from_prop
(
json_prop
[
"
items
"
]
=
self
.
_make_segment_from_prop
(
list_element_prop
)
list_element_prop
,
additional_properties
,
name_and_description_in_properties
,
additional_options_for_text_props
,
units_in_description
)
elif
prop
.
is_reference
():
elif
prop
.
is_reference
():
if
prop
.
datatype
==
db
.
REFERENCE
:
if
prop
.
datatype
==
db
.
REFERENCE
:
# No Record creation since no RT is specified and we don't know what
# No Record creation since no RT is specified and we don't know what
# schema to use, so only enum of all Records and all Files.
# schema to use, so only enum of all Records and all Files.
values
=
_retrieve_enum_values
(
"
RECORD
"
)
+
_retrieve_enum_values
(
"
FILE
"
)
values
=
self
.
_retrieve_enum_values
(
"
RECORD
"
)
+
self
.
_retrieve_enum_values
(
"
FILE
"
)
json_prop
[
"
enum
"
]
=
values
json_prop
[
"
enum
"
]
=
values
elif
prop
.
datatype
==
db
.
FILE
:
elif
prop
.
datatype
==
db
.
FILE
:
# TODO: different issue
# TODO: different issue
raise
NotImplementedError
(
"
Files have not been implemented yet.
"
)
raise
NotImplementedError
(
"
Files have not been implemented yet.
"
)
else
:
else
:
values
=
_retrieve_enum_values
(
f
"
RECORD
'
{
prop
.
datatype
}
'"
)
values
=
self
.
_retrieve_enum_values
(
f
"
RECORD
'
{
prop
.
datatype
}
'"
)
rt
=
db
.
execute_query
(
f
"
FIND RECORDTYPE WITH name=
'
{
prop
.
datatype
}
'"
,
unique
=
True
)
if
prop
.
datatype
in
self
.
_do_not_create
:
subschema
=
_make_segment_from_recordtype
(
rt
,
additional_properties
,
# Only a simple list of values
name_and_description_in_properties
,
json_prop
[
"
enum
"
]
=
values
additional_options_for_text_props
,
else
:
units_in_description
)
rt
=
db
.
execute_query
(
f
"
FIND RECORDTYPE WITH name=
'
{
prop
.
datatype
}
'"
,
unique
=
True
)
subschema
=
self
.
_make_segment_from_recordtype
(
rt
)
subschema
[
"
title
"
]
=
"
Create new
"
subschema
[
"
title
"
]
=
"
Create new
"
json_prop
[
"
oneOf
"
]
=
[
json_prop
[
"
oneOf
"
]
=
[
{
{
"
title
"
:
"
Existing entries
"
,
"
title
"
:
"
Existing entries
"
,
"
enum
"
:
values
},
"
enum
"
:
values
,
},
subschema
subschema
]
]
...
@@ -134,9 +160,8 @@ def _make_segment_from_prop(prop: db.Property, additional_properties: bool,
...
@@ -134,9 +160,8 @@ def _make_segment_from_prop(prop: db.Property, additional_properties: bool,
return
json_prop
return
json_prop
@staticmethod
def
_make_text_property
(
description
=
""
,
text_format
=
None
,
text_pattern
=
None
):
def
_make_text_property
(
description
=
""
,
text_format
=
None
,
text_pattern
=
None
):
prop
=
{
prop
=
{
"
type
"
:
"
string
"
"
type
"
:
"
string
"
}
}
...
@@ -164,7 +189,7 @@ def _make_text_property(description="", text_format=None, text_pattern=None):
...
@@ -164,7 +189,7 @@ def _make_text_property(description="", text_format=None, text_pattern=None):
return
prop
return
prop
@staticmethod
def
_retrieve_enum_values
(
role
:
str
):
def
_retrieve_enum_values
(
role
:
str
):
possible_values
=
db
.
execute_query
(
f
"
SELECT name, id FROM
{
role
}
"
)
possible_values
=
db
.
execute_query
(
f
"
SELECT name, id FROM
{
role
}
"
)
...
@@ -178,24 +203,21 @@ def _retrieve_enum_values(role: str):
...
@@ -178,24 +203,21 @@ def _retrieve_enum_values(role: str):
return
vals
return
vals
def
_make_segment_from_recordtype
(
self
,
rt
:
db
.
RecordType
):
def
_make_segment_from_recordtype
(
rt
:
db
.
RecordType
,
additional_properties
:
bool
=
True
,
name_and_description_in_properties
:
bool
=
False
,
additional_options_for_text_props
:
Optional
[
dict
]
=
None
,
units_in_description
:
bool
=
True
):
"""
Return a Json schema segment for the given RecordType.
"""
Return a Json schema segment for the given RecordType.
"""
"""
schema
=
{
schema
:
dict
[
str
,
Any
]
=
{
"
type
"
:
"
object
"
"
type
"
:
"
object
"
}
}
schema
[
"
required
"
]
=
_make_required_list
(
rt
)
schema
[
"
required
"
]
=
self
.
_make_required_list
(
rt
)
schema
[
"
additionalProperties
"
]
=
additional_properties
schema
[
"
additionalProperties
"
]
=
self
.
_
additional_properties
props
=
{}
props
=
{}
if
name_and_description_in_properties
:
if
self
.
_name_and_description_in_properties
:
props
[
"
name
"
]
=
_make_text_property
(
"
The name of the Record to be created
"
)
props
[
"
name
"
]
=
self
.
_make_text_property
(
"
The name of the Record to be created
"
)
props
[
"
description
"
]
=
_make_text_property
(
"
The description of the Record to be created
"
)
props
[
"
description
"
]
=
self
.
_make_text_property
(
"
The description of the Record to be created
"
)
for
prop
in
rt
.
properties
:
for
prop
in
rt
.
properties
:
if
prop
.
name
in
props
:
if
prop
.
name
in
props
:
...
@@ -204,28 +226,54 @@ def _make_segment_from_recordtype(rt: db.RecordType, additional_properties: bool
...
@@ -204,28 +226,54 @@ def _make_segment_from_recordtype(rt: db.RecordType, additional_properties: bool
"
Creating a schema for multi-properties is not specified.
"
"
Creating a schema for multi-properties is not specified.
"
f
"
Property
{
prop
.
name
}
occurs more than once.
"
f
"
Property
{
prop
.
name
}
occurs more than once.
"
)
)
props
[
prop
.
name
]
=
_make_segment_from_prop
(
props
[
prop
.
name
]
=
self
.
_make_segment_from_prop
(
prop
)
prop
,
additional_properties
,
name_and_description_in_properties
,
additional_options_for_text_props
,
units_in_description
)
schema
[
"
properties
"
]
=
props
schema
[
"
properties
"
]
=
props
return
schema
return
schema
def
recordtype_to_json_schema
(
self
,
rt
:
db
.
RecordType
):
"""
Create a jsonschema from a given RecordType that can be used, e.g., to
validate a json specifying a record of the given type.
Parameters
----------
rt : RecordType
The RecordType from which a json schema will be created.
Returns
-------
schema : dict
A dict containing the json schema created from the given RecordType
'
s properties.
"""
schema
=
self
.
_make_segment_from_recordtype
(
rt
)
schema
[
"
$schema
"
]
=
"
https://json-schema.org/draft/2019-09/schema
"
if
rt
.
name
:
schema
[
"
title
"
]
=
rt
.
name
if
rt
.
description
:
schema
[
"
description
"
]
=
rt
.
description
return
schema
def
recordtype_to_json_schema
(
rt
:
db
.
RecordType
,
additional_properties
:
bool
=
True
,
def
recordtype_to_json_schema
(
rt
:
db
.
RecordType
,
additional_properties
:
bool
=
True
,
name_and_description_in_properties
:
bool
=
False
,
name_and_description_in_properties
:
bool
=
False
,
additional_options_for_text_props
:
Optional
[
dict
]
=
None
,
additional_options_for_text_props
:
Optional
[
dict
]
=
None
,
units_in_description
:
bool
=
True
):
units_in_description
:
bool
=
True
,
do_not_create
:
list
[
str
]
=
None
):
"""
Create a jsonschema from a given RecordType that can be used, e.g., to
"""
Create a jsonschema from a given RecordType that can be used, e.g., to
validate a json specifying a record of the given type.
validate a json specifying a record of the given type.
This is a standalone function which works without manually creating a
JsonSchemaExporter object.
Parameters
Parameters
----------
----------
rt : RecordType
rt : RecordType
The RecordType from which a json schema will be created.
The RecordType from which a json schema will be created.
additional_properties : bool, optional
additional_properties : bool, optional
Whether additional propeties will be admitted in the resulting
Whether additional prope
r
ties will be admitted in the resulting
schema. Optional, default is True.
schema. Optional, default is True.
name_and_description_in_properties : bool, optional
name_and_description_in_properties : bool, optional
Whether to include name and description in the `properties` section of
Whether to include name and description in the `properties` section of
...
@@ -238,6 +286,10 @@ def recordtype_to_json_schema(rt: db.RecordType, additional_properties: bool = T
...
@@ -238,6 +286,10 @@ def recordtype_to_json_schema(rt: db.RecordType, additional_properties: bool = T
description of the corresponding schema entry. If set to false, an
description of the corresponding schema entry. If set to false, an
additional `unit` key is added to the schema itself which is purely
additional `unit` key is added to the schema itself which is purely
annotational and ignored, e.g., in validation. Default is True.
annotational and ignored, e.g., in validation. Default is True.
do_not_create : list[str]
A list of RedcordType names, for which there should be no option
to create them. Instead, only the choice of existing elements should
be given.
Returns
Returns
-------
-------
...
@@ -246,17 +298,11 @@ def recordtype_to_json_schema(rt: db.RecordType, additional_properties: bool = T
...
@@ -246,17 +298,11 @@ def recordtype_to_json_schema(rt: db.RecordType, additional_properties: bool = T
"""
"""
if
additional_options_for_text_props
is
None
:
exporter
=
JsonSchemaExporter
(
additional_options_for_text_props
=
{}
additional_properties
=
additional_properties
,
name_and_description_in_properties
=
name_and_description_in_properties
,
schema
=
_make_segment_from_recordtype
(
rt
,
additional_properties
,
additional_options_for_text_props
=
additional_options_for_text_props
,
name_and_description_in_properties
,
units_in_description
=
units_in_description
,
additional_options_for_text_props
,
do_not_create
=
do_not_create
,
units_in_description
)
)
schema
[
"
$schema
"
]
=
"
https://json-schema.org/draft/2019-09/schema
"
return
exporter
.
recordtype_to_json_schema
(
rt
)
if
rt
.
name
:
schema
[
"
title
"
]
=
rt
.
name
if
rt
.
description
:
schema
[
"
description
"
]
=
rt
.
description
return
schema
This diff is collapsed.
Click to expand it.
unittests/test_json_schema_exporter.py
+
2
−
0
View file @
1a34df84
...
@@ -20,6 +20,8 @@
...
@@ -20,6 +20,8 @@
# with this program. If not, see <https://www.gnu.org/licenses/>.
# with this program. If not, see <https://www.gnu.org/licenses/>.
#
#
"""
Tests the Json schema exporter.
"""
import
linkahead
as
db
import
linkahead
as
db
from
jsonschema
import
FormatChecker
,
validate
,
ValidationError
from
jsonschema
import
FormatChecker
,
validate
,
ValidationError
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment