Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
CaosDB Crawler
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
caosdb
Software
CaosDB Crawler
Commits
cb37a085
Commit
cb37a085
authored
2 years ago
by
florian
Browse files
Options
Downloads
Patches
Plain Diff
WIP: Extend h5 test
parent
94d4e55e
No related branches found
No related tags found
2 merge requests
!160
STY: styling
,
!143
ENH: HDF5 Converter
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
unittests/h5_cfood.yml
+44
-27
44 additions, 27 deletions
unittests/h5_cfood.yml
unittests/test_h5_converter.py
+22
-12
22 additions, 12 deletions
unittests/test_h5_converter.py
with
66 additions
and
39 deletions
unittests/h5_cfood.yml
+
44
−
27
View file @
cb37a085
...
...
@@ -2,16 +2,33 @@
metadata
:
crawler-version
:
0.6.1
---
Converters
:
H5Dataset
:
converter
:
H5DatasetConverter
package
:
caoscrawler.hdf5_converter
H5File
:
converter
:
H5FileConverter
package
:
caoscrawler.hdf5_converter
H5Group
:
converter
:
H5GroupConverter
package
:
caoscrawler.hdf5_converter
H5Ndarray
:
converter
:
H5NdarrayConverter
package
:
caoscrawler.hdf5_converter
# Top-level, we have just the HDF5 file.
ParentDirectory
:
type
:
Directory
match
:
(.*)
subtree
:
H5FileElement
:
type
:
H5File
match
:
(.*)\.(hdf5|h5)$
subtree
:
# Here, we have the groups, the top-level dataset, and possible
attributes
# (empty for now).
# Here, we have the groups, the top-level dataset, and possible
#
attributes
(empty for now).
RootIntegerElement
:
type
:
H5Dataset
match
:
^root_integers$
match
_name
:
^root_integers$
records
:
H5Dataset
:
parents
:
...
...
@@ -20,14 +37,14 @@ H5FileElement:
# included NDArray in this dataset
TopLevelIntNDElement
:
type
:
H5Ndarray
match
:
(.*)
match
_name
:
(.*)
recordname
:
this
records
:
H5Dataset
:
+$this
# There is one more list-valued attribute to this dataset.
TopLevelDataAttribute
:
type
:
ListElement
match
:
^attr_data_root$
match
_name
:
^attr_data_root$
subtree
:
AttributeListEntries
:
type
:
FloatElement
...
...
This diff is collapsed.
Click to expand it.
unittests/test_h5_converter.py
+
22
−
12
View file @
cb37a085
...
...
@@ -19,14 +19,16 @@
#
import
numpy
as
np
from
pathlib
import
Path
from
pytest
import
fixture
,
importorskip
from
caoscrawler.hdf5_converter
import
(
__convert_basic_element_with_nd_array
,
__convert_h5_element
,
H5GroupElement
,
from
caoscrawler.debug_tree
import
DebugTree
from
caoscrawler.hdf5_converter
import
(
convert_basic_element_with_nd_array
,
convert_h5_element
,
H5GroupElement
,
H5DatasetElement
,
H5NdarrayElement
)
from
caoscrawler.scanner
import
scan_directory
from
caoscrawler.structure_elements
import
(
FloatElement
,
ListElement
,
TextElement
)
from
pathlib
import
Path
from
pytest
import
fixture
,
importorskip
# Skip the whole module if h5py hasn't been installed
h5py
=
importorskip
(
"
h5py
"
)
...
...
@@ -45,10 +47,10 @@ def h5_dummy_file():
def
test_h5_elements
(
h5_dummy_file
):
elt
=
__
convert_h5_element
(
h5_dummy_file
[
"
group_level1_a
"
],
"
test
"
)
elt
=
convert_h5_element
(
h5_dummy_file
[
"
group_level1_a
"
],
"
test
"
)
assert
isinstance
(
elt
,
H5GroupElement
)
elt
=
__
convert_h5_element
(
h5_dummy_file
[
"
root_integers
"
],
"
test
"
)
elt
=
convert_h5_element
(
h5_dummy_file
[
"
root_integers
"
],
"
test
"
)
assert
isinstance
(
elt
,
H5DatasetElement
)
...
...
@@ -57,30 +59,38 @@ def test_nd_array_conversion():
# Only test array handling here, `convert_basic_element` is tested
# elsewhere.
arr
=
np
.
array
([[[
"
something
"
]]])
elt
=
__
convert_basic_element_with_nd_array
(
arr
)
elt
=
convert_basic_element_with_nd_array
(
arr
)
assert
isinstance
(
elt
,
TextElement
)
assert
elt
.
value
==
"
something
"
arr
=
np
.
zeros
((
1
,
1
))
elt
=
__
convert_basic_element_with_nd_array
(
arr
)
elt
=
convert_basic_element_with_nd_array
(
arr
)
assert
isinstance
(
elt
,
FloatElement
)
assert
elt
.
value
==
0
arr
=
np
.
zeros
((
1
,
3
,
1
))
elt
=
__
convert_basic_element_with_nd_array
(
arr
)
elt
=
convert_basic_element_with_nd_array
(
arr
)
assert
isinstance
(
elt
,
ListElement
)
assert
elt
.
value
==
[
0
,
0
,
0
]
arr
=
np
.
array
([[
1
,
2
,
3
],
[
4
,
5
,
6
]])
elt
=
__
convert_basic_element_with_nd_array
(
arr
,
internal_path
=
"
some/path
"
)
elt
=
convert_basic_element_with_nd_array
(
arr
,
internal_path
=
"
some/path
"
)
assert
isinstance
(
elt
,
H5NdarrayElement
)
assert
elt
.
internal_path
==
"
some/path
"
# Non-arrays should be forwarded correctly
elt
=
__
convert_basic_element_with_nd_array
(
"
something
"
)
elt
=
convert_basic_element_with_nd_array
(
"
something
"
)
assert
isinstance
(
elt
,
TextElement
)
assert
elt
.
value
==
"
something
"
elt
=
__
convert_basic_element_with_nd_array
([
0
,
0
,
0
])
elt
=
convert_basic_element_with_nd_array
([
0
,
0
,
0
])
assert
isinstance
(
elt
,
ListElement
)
assert
elt
.
value
==
[
0
,
0
,
0
]
def
test_record_creation
():
dbtr
=
DebugTree
()
records
=
scan_directory
(
UNITTESTDIR
,
UNITTESTDIR
/
"
h5_cfood.yml
"
,
debug_tree
=
dbtr
)
print
(
dbtr
.
debug_tree
)
print
(
dbtr
.
debug_metadata
)
assert
False
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment