Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
R
RuQaD
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
caosdb
Software
FAIR Data Spaces
RuQaD
Commits
2b7fd173
Verified
Commit
2b7fd173
authored
5 months ago
by
Timm Fitschen
Browse files
Options
Downloads
Patches
Plain Diff
STY: auto formatting
parent
7c5617d3
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!8
Code coverage
Pipeline
#59615
failed
5 months ago
Stage: info
Stage: setup
Stage: style
Stage: code-analysis
Stage: test
Stage: pages
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
src/ruqad/crawler.py
+1
-1
1 addition, 1 deletion
src/ruqad/crawler.py
src/ruqad/kadi.py
+5
-2
5 additions, 2 deletions
src/ruqad/kadi.py
src/ruqad/monitor.py
+2
-1
2 additions, 1 deletion
src/ruqad/monitor.py
with
8 additions
and
4 deletions
src/ruqad/crawler.py
+
1
−
1
View file @
2b7fd173
...
@@ -87,6 +87,6 @@ def trigger_crawler(target_dir: str) -> tuple[bool, list[db.Entity]]:
...
@@ -87,6 +87,6 @@ def trigger_crawler(target_dir: str) -> tuple[bool, list[db.Entity]]:
cfood_file_name
=
ruqad_crawler_settings
.
joinpath
(
'
cfood.yaml
'
),
cfood_file_name
=
ruqad_crawler_settings
.
joinpath
(
'
cfood.yaml
'
),
identifiables_definition_file
=
ruqad_crawler_settings
.
joinpath
(
identifiables_definition_file
=
ruqad_crawler_settings
.
joinpath
(
'
identifiables.yaml
'
),
'
identifiables.yaml
'
),
remove_prefix
=
"
/
"
+
os
.
path
.
basename
(
target_dir
))
remove_prefix
=
"
/
"
+
os
.
path
.
basename
(
target_dir
))
return
(
True
,
ent_qc
)
return
(
True
,
ent_qc
)
This diff is collapsed.
Click to expand it.
src/ruqad/kadi.py
+
5
−
2
View file @
2b7fd173
...
@@ -44,11 +44,12 @@ def _generate_pages(manager) -> dict:
...
@@ -44,11 +44,12 @@ def _generate_pages(manager) -> dict:
+
str
(
response
[
'
code
'
]))
+
str
(
response
[
'
code
'
]))
n_pages
=
response
[
"
_pagination
"
][
"
total_pages
"
]
n_pages
=
response
[
"
_pagination
"
][
"
total_pages
"
]
for
ii
in
range
(
n_pages
):
for
ii
in
range
(
n_pages
):
query_params
.
update
({
"
page
"
:
ii
+
1
})
query_params
.
update
({
"
page
"
:
ii
+
1
})
yield
manager
.
search
.
search_resources
(
"
record
"
,
**
query_params
).
json
()
yield
manager
.
search
.
search_resources
(
"
record
"
,
**
query_params
).
json
()
def
collect_records_created_after
(
manager
:
KadiManager
,
cut_off_date
:
datetime
.
datetime
)
->
list
(
int
):
def
collect_records_created_after
(
manager
:
KadiManager
,
cut_off_date
:
datetime
.
datetime
)
->
list
(
int
):
"""
"""
Iterates page-wise over the responses of the Kadi API until records are reached that are older
Iterates page-wise over the responses of the Kadi API until records are reached that are older
than the given cut_off_date.
than the given cut_off_date.
...
@@ -89,6 +90,7 @@ def download_eln_for(manager: KadiManager, rid: int, path: str) -> None:
...
@@ -89,6 +90,7 @@ def download_eln_for(manager: KadiManager, rid: int, path: str) -> None:
rec
=
manager
.
record
(
id
=
rid
)
rec
=
manager
.
record
(
id
=
rid
)
rec
.
export
(
path
=
path
,
export_type
=
'
ro-crate
'
)
rec
.
export
(
path
=
path
,
export_type
=
'
ro-crate
'
)
class
KadiManager
(
_KadiManager
):
class
KadiManager
(
_KadiManager
):
"""
Fix KadiManager to respect context root in url.
"""
"""
Fix KadiManager to respect context root in url.
"""
...
@@ -96,6 +98,7 @@ class KadiManager(_KadiManager):
...
@@ -96,6 +98,7 @@ class KadiManager(_KadiManager):
super
().
__init__
(
**
kwargs
)
super
().
__init__
(
**
kwargs
)
self
.
host
=
f
'
{
kwargs
[
"
host
"
]
}
/api/v1
'
self
.
host
=
f
'
{
kwargs
[
"
host
"
]
}
/api/v1
'
def
main
():
def
main
():
with
KadiManager
(
instance
=
'
demo
'
)
as
manager
:
with
KadiManager
(
instance
=
'
demo
'
)
as
manager
:
cut_off_date
=
datetime
.
fromisoformat
(
cut_off_date
=
datetime
.
fromisoformat
(
...
...
This diff is collapsed.
Click to expand it.
src/ruqad/monitor.py
+
2
−
1
View file @
2b7fd173
...
@@ -42,6 +42,7 @@ KADIARGS = {
...
@@ -42,6 +42,7 @@ KADIARGS = {
SKIP_QUALITY_CHECK
=
os
.
getenv
(
"
SKIP_QUALITY_CHECK
"
)
is
not
None
SKIP_QUALITY_CHECK
=
os
.
getenv
(
"
SKIP_QUALITY_CHECK
"
)
is
not
None
def
monitor
():
def
monitor
():
"""
Continuously monitor the Kadi instance given in the environment variables.
"""
Continuously monitor the Kadi instance given in the environment variables.
...
@@ -82,7 +83,7 @@ def monitor():
...
@@ -82,7 +83,7 @@ def monitor():
if
os
.
path
.
exists
(
os
.
path
.
join
(
cdir
,
"
artifacts.zip
"
)):
if
os
.
path
.
exists
(
os
.
path
.
join
(
cdir
,
"
artifacts.zip
"
)):
shutil
.
move
(
os
.
path
.
join
(
cdir
,
"
artifacts.zip
"
),
shutil
.
move
(
os
.
path
.
join
(
cdir
,
"
artifacts.zip
"
),
os
.
path
.
join
(
remote_dir_path
,
"
report.zip
"
))
os
.
path
.
join
(
remote_dir_path
,
"
report.zip
"
))
#else:
#
else:
# Path(os.path.join(remote_dir_path, "report.zip")).touch()
# Path(os.path.join(remote_dir_path, "report.zip")).touch()
shutil
.
move
(
os
.
path
.
join
(
cdir
,
"
export.eln
"
),
shutil
.
move
(
os
.
path
.
join
(
cdir
,
"
export.eln
"
),
os
.
path
.
join
(
remote_dir_path
,
"
export.eln
"
))
os
.
path
.
join
(
remote_dir_path
,
"
export.eln
"
))
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment