Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
C
caosdb-advanced-user-tools
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
caosdb
Software
caosdb-advanced-user-tools
Commits
563e3578
Commit
563e3578
authored
2 years ago
by
Henrik tom Wörden
Browse files
Options
Downloads
Patches
Plain Diff
DOC: add doc strings and remove debugging output
parent
5f096364
No related branches found
No related tags found
2 merge requests
!59
FIX: if multiple updates for one entity exist, the retrieve would result in an...
,
!55
ENH: include caosdbignore
Pipeline
#28020
failed
2 years ago
Stage: setup
Stage: cert
Stage: style
Stage: unittest
Stage: integrationtest
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
src/caosadvancedtools/loadFiles.py
+19
-4
19 additions, 4 deletions
src/caosadvancedtools/loadFiles.py
with
19 additions
and
4 deletions
src/caosadvancedtools/loadFiles.py
+
19
−
4
View file @
563e3578
...
@@ -51,6 +51,9 @@ def convert_size(size):
...
@@ -51,6 +51,9 @@ def convert_size(size):
def
combine_ignore_files
(
caosdbignore
,
localignore
,
dirname
=
None
):
def
combine_ignore_files
(
caosdbignore
,
localignore
,
dirname
=
None
):
"""
appends the contents of localignore to caosdbignore
and saves the result and returns the name
"""
tmp
=
NamedTemporaryFile
(
delete
=
False
,
mode
=
"
w
"
,
dir
=
dirname
,
prefix
=
"
.caosdbignore
"
)
tmp
=
NamedTemporaryFile
(
delete
=
False
,
mode
=
"
w
"
,
dir
=
dirname
,
prefix
=
"
.caosdbignore
"
)
with
open
(
caosdbignore
,
"
r
"
)
as
base
:
with
open
(
caosdbignore
,
"
r
"
)
as
base
:
tmp
.
write
(
base
.
read
())
tmp
.
write
(
base
.
read
())
...
@@ -61,21 +64,29 @@ def combine_ignore_files(caosdbignore, localignore, dirname=None):
...
@@ -61,21 +64,29 @@ def combine_ignore_files(caosdbignore, localignore, dirname=None):
def
compile_file_list
(
caosdbignore
,
localpath
):
def
compile_file_list
(
caosdbignore
,
localpath
):
"""
creates a list of files
that contain all files under localpath except those excluded by caosdbignore
"""
from
gitignore_parser
import
parse_gitignore
from
gitignore_parser
import
parse_gitignore
matches
=
parse_gitignore
(
caosdbignore
)
matches
=
parse_gitignore
(
caosdbignore
)
current_ignore
=
caosdbignore
current_ignore
=
caosdbignore
non_ignored_files
=
[]
non_ignored_files
=
[]
ignore_files
=
[]
ignore_files
=
[]
for
root
,
dirs
,
files
in
os
.
walk
(
localpath
):
for
root
,
dirs
,
files
in
os
.
walk
(
localpath
):
# remove local ignore files that do no longer apply to the current subtree (branch switch)
while
len
(
ignore_files
)
>
0
and
not
root
.
startswith
(
ignore_files
[
-
1
][
0
]):
while
len
(
ignore_files
)
>
0
and
not
root
.
startswith
(
ignore_files
[
-
1
][
0
]):
shutil
.
os
.
remove
(
ignore_files
[
-
1
][
1
])
shutil
.
os
.
remove
(
ignore_files
[
-
1
][
1
])
ignore_files
.
pop
()
ignore_files
.
pop
()
# use the global one if there are no more local ones
if
len
(
ignore_files
)
>
0
:
if
len
(
ignore_files
)
>
0
:
current_ignore
=
ignore_files
[
-
1
][
1
]
current_ignore
=
ignore_files
[
-
1
][
1
]
matches
=
parse_gitignore
(
current_ignore
)
matches
=
parse_gitignore
(
current_ignore
)
else
:
else
:
current_ignore
=
caosdbignore
current_ignore
=
caosdbignore
matches
=
parse_gitignore
(
current_ignore
)
matches
=
parse_gitignore
(
current_ignore
)
# create a new local ignore file
if
"
.caosdbignore
"
in
files
:
if
"
.caosdbignore
"
in
files
:
current_ignore
=
combine_ignore_files
(
current_ignore
,
current_ignore
=
combine_ignore_files
(
current_ignore
,
os
.
path
.
join
(
root
,
"
.caosdbignore
"
),
os
.
path
.
join
(
root
,
"
.caosdbignore
"
),
...
@@ -84,6 +95,8 @@ def compile_file_list(caosdbignore, localpath):
...
@@ -84,6 +95,8 @@ def compile_file_list(caosdbignore, localpath):
dirname
=
root
)
dirname
=
root
)
ignore_files
.
append
((
root
,
current_ignore
))
ignore_files
.
append
((
root
,
current_ignore
))
matches
=
parse_gitignore
(
current_ignore
)
matches
=
parse_gitignore
(
current_ignore
)
# actually append files that are not ignored
for
fi
in
files
:
for
fi
in
files
:
fullpath
=
os
.
path
.
join
(
root
,
fi
)
fullpath
=
os
.
path
.
join
(
root
,
fi
)
if
not
matches
(
fullpath
):
if
not
matches
(
fullpath
):
...
@@ -92,6 +105,10 @@ def compile_file_list(caosdbignore, localpath):
...
@@ -92,6 +105,10 @@ def compile_file_list(caosdbignore, localpath):
def
create_re_for_file_list
(
files
,
localroot
,
remoteroot
):
def
create_re_for_file_list
(
files
,
localroot
,
remoteroot
):
"""
creates a regular expression
that matches file paths contained in the files argument and all parent directories.
The prefix localroot is replaced by the prefix remoteroot.
"""
regexp
=
""
regexp
=
""
for
fi
in
files
:
for
fi
in
files
:
path
=
fi
path
=
fi
...
@@ -108,10 +125,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi
...
@@ -108,10 +125,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi
localpath
=
None
):
localpath
=
None
):
if
caosdbignore
:
if
caosdbignore
:
# create list of files and create regular expression for small chunks
filelist
=
compile_file_list
(
caosdbignore
,
localpath
)
filelist
=
compile_file_list
(
caosdbignore
,
localpath
)
for
fi
in
filelist
:
print
(
fi
)
fulllist
=
filelist
fulllist
=
filelist
index
=
0
index
=
0
...
@@ -124,8 +139,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi
...
@@ -124,8 +139,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi
else
:
else
:
includes
=
[
include
]
includes
=
[
include
]
# if no caosdbignore file is used, this iterates over a single include
for
include
in
includes
:
for
include
in
includes
:
print
(
include
)
if
dryrun
:
if
dryrun
:
logger
.
info
(
"
Performin a dryrun!
"
)
logger
.
info
(
"
Performin a dryrun!
"
)
files
=
db
.
Container
().
retrieve
(
files
=
db
.
Container
().
retrieve
(
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment