Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
C
caosdb-advanced-user-tools
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
caosdb
Software
caosdb-advanced-user-tools
Commits
d3e493ae
Commit
d3e493ae
authored
2 years ago
by
florian
Browse files
Options
Downloads
Patches
Plain Diff
ENH: Treat array entries without items property
parent
0893890f
No related branches found
No related tags found
2 merge requests
!73
MAINT: change wording of TableImporter argument and allow converters and...
,
!72
Extend json-schema model parser
Pipeline
#36820
failed
2 years ago
Stage: setup
Stage: cert
Stage: style
Stage: unittest
Stage: integrationtest
Changes
2
Pipelines
1
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
src/caosadvancedtools/models/parser.py
+47
-22
47 additions, 22 deletions
src/caosadvancedtools/models/parser.py
unittests/test_json_schema_model_parser.py
+27
-0
27 additions, 0 deletions
unittests/test_json_schema_model_parser.py
with
74 additions
and
22 deletions
src/caosadvancedtools/models/parser.py
+
47
−
22
View file @
d3e493ae
...
@@ -154,7 +154,12 @@ def parse_model_from_string(string):
...
@@ -154,7 +154,12 @@ def parse_model_from_string(string):
return
parser
.
parse_model_from_string
(
string
)
return
parser
.
parse_model_from_string
(
string
)
def
parse_model_from_json_schema
(
filename
:
str
,
top_level_recordtype
:
bool
=
True
):
def
parse_model_from_json_schema
(
filename
:
str
,
top_level_recordtype
:
bool
=
True
,
types_for_missing_array_items
:
dict
=
{},
ignore_unspecified_array_items
:
bool
=
False
):
"""
Return a datamodel parsed from a json schema definition.
"""
Return a datamodel parsed from a json schema definition.
Parameters
Parameters
...
@@ -164,7 +169,14 @@ def parse_model_from_json_schema(filename: str, top_level_recordtype: bool = Tru
...
@@ -164,7 +169,14 @@ def parse_model_from_json_schema(filename: str, top_level_recordtype: bool = Tru
top_level_recordtype : bool, optional
top_level_recordtype : bool, optional
Whether there is a record type defined at the top level of the
Whether there is a record type defined at the top level of the
schema. Default is true.
schema. Default is true.
types_for_missing_array_items : dict, optional
dictionary containing fall-back types for json entries with `type:
array` but without `items` specification. Default is an empty dict.
ignore_unspecified_array_items : bool, optional
Whether to ignore `type: array` entries the type of which is not
specified by their `items` property or given in
`types_for_missing_array_items`. An error is raised if they are not
ignored. Default is False.
Returns
Returns
-------
-------
...
@@ -181,7 +193,7 @@ def parse_model_from_json_schema(filename: str, top_level_recordtype: bool = Tru
...
@@ -181,7 +193,7 @@ def parse_model_from_json_schema(filename: str, top_level_recordtype: bool = Tru
# @author Florian Spreckelsen
# @author Florian Spreckelsen
# @date 2022-02-17
# @date 2022-02-17
# @review Daniel Hornung 2022-02-18
# @review Daniel Hornung 2022-02-18
parser
=
JsonSchemaParser
()
parser
=
JsonSchemaParser
(
types_for_missing_array_items
,
ignore_unspecified_array_items
)
return
parser
.
parse_model_from_json_schema
(
filename
,
top_level_recordtype
)
return
parser
.
parse_model_from_json_schema
(
filename
,
top_level_recordtype
)
...
@@ -629,6 +641,11 @@ class JsonSchemaParser(Parser):
...
@@ -629,6 +641,11 @@ class JsonSchemaParser(Parser):
# @date 2022-02-17
# @date 2022-02-17
# @review Timm Fitschen 2022-02-30
# @review Timm Fitschen 2022-02-30
def
__init__
(
self
,
types_for_missing_array_items
=
{},
ignore_unspecified_array_items
=
False
):
super
().
__init__
()
self
.
types_for_missing_array_items
=
types_for_missing_array_items
self
.
ignore_unspecified_array_items
=
ignore_unspecified_array_items
def
parse_model_from_json_schema
(
self
,
filename
:
str
,
top_level_recordtype
:
bool
=
True
):
def
parse_model_from_json_schema
(
self
,
filename
:
str
,
top_level_recordtype
:
bool
=
True
):
"""
Return a datamodel created from the definition in the json schema in
"""
Return a datamodel created from the definition in the json schema in
`filename`.
`filename`.
...
@@ -755,7 +772,7 @@ class JsonSchemaParser(Parser):
...
@@ -755,7 +772,7 @@ class JsonSchemaParser(Parser):
if
name
==
"
name
"
:
if
name
==
"
name
"
:
# This is identified with the CaosDB name property as long as the
# This is identified with the CaosDB name property as long as the
# type is correct.
# type is correct.
if
not
elt
[
"
type
"
]
==
"
string
"
:
if
not
elt
[
"
type
"
]
==
"
string
"
and
"
string
"
not
in
elt
[
"
type
"
]
:
raise
JsonSchemaDefinitionError
(
raise
JsonSchemaDefinitionError
(
"
The
'
name
'
property must be string-typed, otherwise it cannot
"
"
The
'
name
'
property must be string-typed, otherwise it cannot
"
"
be identified with CaosDB
'
s name property.
"
"
be identified with CaosDB
'
s name property.
"
...
@@ -786,6 +803,7 @@ class JsonSchemaParser(Parser):
...
@@ -786,6 +803,7 @@ class JsonSchemaParser(Parser):
# treat_something function
# treat_something function
ent
.
description
=
elt
[
"
description
"
]
ent
.
description
=
elt
[
"
description
"
]
if
ent
is
not
None
:
self
.
model
[
name
]
=
ent
self
.
model
[
name
]
=
ent
return
ent
,
force_list
return
ent
,
force_list
...
@@ -846,9 +864,12 @@ class JsonSchemaParser(Parser):
...
@@ -846,9 +864,12 @@ class JsonSchemaParser(Parser):
def
_treat_list
(
self
,
elt
:
dict
,
name
:
str
):
def
_treat_list
(
self
,
elt
:
dict
,
name
:
str
):
# @review Timm Fitschen 2022-02-30
# @review Timm Fitschen 2022-02-30
if
"
items
"
not
in
elt
:
if
"
items
"
not
in
elt
and
name
not
in
self
.
types_for_missing_array_items
:
if
self
.
ignore_unspecified_array_items
:
return
None
,
False
raise
JsonSchemaDefinitionError
(
raise
JsonSchemaDefinitionError
(
f
"
The definition of the list items is missing in
{
elt
}
.
"
)
f
"
The definition of the list items is missing in
{
elt
}
.
"
)
if
"
items
"
in
elt
:
items
=
elt
[
"
items
"
]
items
=
elt
[
"
items
"
]
if
"
enum
"
in
items
:
if
"
enum
"
in
items
:
return
self
.
_treat_enum
(
items
,
name
),
True
return
self
.
_treat_enum
(
items
,
name
),
True
...
@@ -866,6 +887,10 @@ class JsonSchemaParser(Parser):
...
@@ -866,6 +887,10 @@ class JsonSchemaParser(Parser):
items
,
self
.
_stringify
(
items
[
"
title
"
]))
items
,
self
.
_stringify
(
items
[
"
title
"
]))
self
.
model
[
ref_rt
.
name
]
=
ref_rt
self
.
model
[
ref_rt
.
name
]
=
ref_rt
return
db
.
Property
(
name
=
name
,
datatype
=
db
.
LIST
(
ref_rt
)),
False
return
db
.
Property
(
name
=
name
,
datatype
=
db
.
LIST
(
ref_rt
)),
False
else
:
# Use predefined type:
datatype
=
db
.
LIST
(
self
.
types_for_missing_array_items
[
name
])
return
db
.
Property
(
name
=
name
,
datatype
=
datatype
),
False
def
_get_pattern_prop
(
self
):
def
_get_pattern_prop
(
self
):
if
"
__pattern_property_pattern_property
"
in
self
.
model
:
if
"
__pattern_property_pattern_property
"
in
self
.
model
:
...
...
This diff is collapsed.
Click to expand it.
unittests/test_json_schema_model_parser.py
+
27
−
0
View file @
d3e493ae
...
@@ -414,3 +414,30 @@ def test_no_toplevel_entity():
...
@@ -414,3 +414,30 @@ def test_no_toplevel_entity():
assert
pattern_object_rt
.
get_importance
(
pp
.
name
)
==
db
.
OBLIGATORY
assert
pattern_object_rt
.
get_importance
(
pp
.
name
)
==
db
.
OBLIGATORY
date_id_prop
=
pattern_object_rt
.
get_property
(
"
date_id
"
)
date_id_prop
=
pattern_object_rt
.
get_property
(
"
date_id
"
)
assert
date_id_prop
.
datatype
==
db
.
TEXT
assert
date_id_prop
.
datatype
==
db
.
TEXT
def
test_missing_array_items
():
# strict behavior
with
pytest
.
raises
(
JsonSchemaDefinitionError
)
as
err
:
parse_model_from_json_schema
(
os
.
path
.
join
(
FILEPATH
,
"
datamodel_missing_array_items.schema.json
"
))
assert
"
{
'
type
'
:
'
array
'
}
"
in
str
(
err
)
# ignore all problems, so a RT is created that does not have the property
model
=
parse_model_from_json_schema
(
os
.
path
.
join
(
FILEPATH
,
"
datamodel_missing_array_items.schema.json
"
),
ignore_unspecified_array_items
=
True
)
assert
"
something_with_missing_array_items
"
in
model
rt
=
model
[
"
something_with_missing_array_items
"
]
assert
isinstance
(
rt
,
db
.
RecordType
)
assert
rt
.
get_property
(
"
missing
"
)
is
None
# specify the type:
type_dict
=
{
"
missing
"
:
db
.
FILE
}
model
=
parse_model_from_json_schema
(
os
.
path
.
join
(
FILEPATH
,
"
datamodel_missing_array_items.schema.json
"
),
types_for_missing_array_items
=
type_dict
)
assert
"
something_with_missing_array_items
"
in
model
rt
=
model
[
"
something_with_missing_array_items
"
]
assert
rt
.
get_property
(
"
missing
"
)
is
not
None
assert
rt
.
get_property
(
"
missing
"
).
datatype
==
db
.
LIST
(
db
.
FILE
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment