Skip to content
Snippets Groups Projects
Commit 6c1cfa3f authored by Daniel Hornung's avatar Daniel Hornung
Browse files

Merge branch 'f-sast-sca' into 'dev' 

ENH: SAST / SCA

See merge request caosdb/customers/f-fit/ruqad!6
parents 6a07f736 3bf4227d
No related branches found
No related tags found
1 merge request!6ENH: SAST / SCA
Pipeline #58673 passed
Stage: info
Stage: setup
Stage: style
Stage: code-analysis
Stage: test
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 73
− 50
View file @ 6c1cfa3f
......@@ -19,16 +19,25 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
# Taken from: https://forum.gitlab.com/t/clarification/54346
GITLAB_FEATURES: "$GITLAB_FEATURES,dependency_scanning"
stages:
- info
- setup
- cert
- style
- code-analysis
- test
- deploy
# TEMPLATES ==========================================================
include:
- template: Jobs/SAST.gitlab-ci.yml
- template: Jobs/Dependency-Scanning.gitlab-ci.yml
.env: &env
- echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
......@@ -52,51 +61,15 @@ e2e_tests:
- echo $KADITOKEN
- python -m pytest end-to-end-tests/test_kadi.py
unittest_py3.9:
tags: [cached-dind]
stage: test
image: python:3.9
script: &python_test_script
# TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
- pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
- pip install .[all]
# actual test
- pytest --cov=ruqad -vv ./unittests
unittest_py3.10:
tags: [cached-dind]
stage: test
image: python:3.10
script: *python_test_script
unittest_py3.11:
tags: [cached-dind]
stage: test
image: python:3.11
script: *python_test_script
unittest_py3.12:
tags: [cached-dind]
stage: test
image: python:3.12
script: *python_test_script
unittest_py3.13:
tags: [cached-dind]
stage: test
image: python:3.13
script: *python_test_script
build-testenv:
tags: [cached-dind]
image: docker:20.10
stage: setup
timeout: 2h
only:
- schedules
- web
- pushes
# rules:
# - if: $CI_PIPELINE_SOURCE == "push"
# - if: $CI_PIPELINE_SOURCE == "schedule"
# - if: $CI_PIPELINE_SOURCE == "web"
needs: []
script:
- df -h
......@@ -117,21 +90,71 @@ code-style:
tags: [docker]
stage: style
image: $CI_REGISTRY_IMAGE
needs:
- job: build-testenv
optional: true
needs: [build-testenv]
allow_failure: true
script:
- autopep8 -r --diff --exit-code .
allow_failure: true
pylint:
tags: [docker]
stage: style
image: $CI_REGISTRY_IMAGE
needs:
- job: build-testenv
optional: true
needs: [build-testenv]
allow_failure: true
script:
- pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
- pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
# SAST/SCA: Everything else is configured in the templates. This is only necessary so that the job
# finds a matching runner and is run in the correct stage.
# SAST: Static Application Security Testing
sast:
tags: [docker]
needs: [info]
stage: code-analysis
# Dependency scanning and vulnerabilities (SCA, Software Composition Analysis)
gemnasium-python-dependency_scanning:
tags: [docker]
needs: [info]
stage: code-analysis
unittest_py3.9:
tags: [cached-dind]
needs: [build-testenv]
stage: test
image: python:3.9
script: &python_test_script
# TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
- pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
- pip install .[all]
# actual test
- pytest --cov=ruqad -vv ./unittests
unittest_py3.10:
tags: [cached-dind]
needs: [build-testenv]
stage: test
image: python:3.10
script: *python_test_script
unittest_py3.11:
tags: [cached-dind]
needs: [build-testenv]
stage: test
image: python:3.11
script: *python_test_script
unittest_py3.12:
tags: [cached-dind]
needs: [build-testenv]
stage: test
image: python:3.12
script: *python_test_script
unittest_py3.13:
tags: [cached-dind]
needs: [build-testenv]
stage: test
image: python:3.13
script: *python_test_script
CHANGELOG.md
+ 1
− 0
View file @ 6c1cfa3f
......@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- LinkAhead crawler for the metadata check.
- Triggers the quality checker.
- Docker build file and instructions.
- CI pipeline.
### Changed ###
......
Makefile
+ 5
− 0
View file @ 6c1cfa3f
......@@ -46,3 +46,8 @@ lint:
unittests:
tox -r
.PHONY: unittests
bom:
echo '# generated by `pip freeze`' > requirements.txt
pip freeze --exclude-editable >> requirements.txt
.PHONY: bom
README.md
+ 11
− 9
View file @ 6c1cfa3f
......@@ -13,6 +13,9 @@ Simply install with:
`pip install .`
Note: You can safely ignore the `requirements.txt`, this file is used as a lock file for components
analysis. For more information, look at the section "SCA" below.
### Run locally ###
- Make sure that `qualitycheck_config.toml` and `secrets.sh` are filled with valied values.
......@@ -29,7 +32,9 @@ Simply install with:
record.
- Wait for the new record with the file to be digested by the Ruqad monitor.
### unit Tests
## Development ##
### Unit Tests
- For testing, install with the `test` extra, for example like so: `pip install .[test]`
- Then run `make unittest` or `pytest unittests/`.
......@@ -44,15 +49,12 @@ In order to run the E2E test, you need to create a personal access token (pat) i
Run `make style lint` after installing with the `dev` extra. (The `dev` extra includes the `test`
extra.)
<!-- ### Documentation -->
<!-- Run `make doc` after installing the dependencies listed below. -->
<!-- For building the documentation we require -->
### SCA (Software component analysis) ###
<!-- - `sphinx` -->
<!-- - `recommonmark` -->
<!-- - `sphinx-rtd-theme` -->
To run the SCA job in the pipeline, run `make bom` and commit the generated `requirements.txt` to
git. This will be used in the `gemnasium-python-dependency_scanning` job of the `code-analysis`
stage to generate `gl-sbom-pypi-pip.cdx.json` file in CycloneDX format inside the artifacts zip
file. This file can be visuealized for example with [Dependency Track](https://dependencytrack.org/).
## Docker deployment ##
......
requirements.txt 0 → 100644
+ 108
− 0
View file @ 6c1cfa3f
# generated by `pip freeze`
aiohappyeyeballs==2.4.3
aiohttp==3.11.8
aiosignal==1.3.1
arcp==0.2.1
arrow==1.3.0
astroid==3.3.5
asttokens==2.4.1
attrs==24.2.0
autopep8==2.3.1
bioblend==1.4.0
boto3==1.35.54
botocore==1.35.54
CacheControl==0.14.1
caosadvancedtools==0.12.0
certifi==2024.8.30
charset-normalizer==3.4.0
click==8.1.7
coverage==7.6.8
decorator==5.1.1
defusedxml==0.7.1
dill==0.3.9
et_xmlfile==2.0.0
executing==2.1.0
filelock==3.16.1
fqdn==1.5.1
frozenlist==1.5.0
future==1.0.0
galaxy2cwl==0.1.4
gxformat2==0.20.0
idna==3.10
importlib_resources==6.4.5
iniconfig==2.0.0
ipython==8.30.0
isoduration==20.11.0
isort==5.13.2
jedi==0.19.1
Jinja2==3.1.4
jmespath==1.0.1
jsonpointer==3.0.0
jsonref==1.1.0
jsonschema==4.23.0
jsonschema-specifications==2024.10.1
kadi-apy==0.43.0
linkahead==0.16.0
lxml==5.3.0
MarkupSafe==3.0.2
matplotlib-inline==0.1.7
mccabe==0.7.0
mistune==3.0.2
msgpack==1.1.0
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
odfpy==1.4.1
openpyxl==3.1.5
packaging==24.2
pandas==2.2.3
parso==0.8.4
pexpect==4.9.0
pip-autoremove==0.10.0
platformdirs==4.3.6
pluggy==1.5.0
prompt_toolkit==3.0.48
propcache==0.2.0
ptyprocess==0.7.0
pure_eval==0.2.3
pyarrow==18.0.0
pycodestyle==2.12.1
Pygments==2.18.0
pylint==3.3.1
pyparsing==3.2.0
PySocks==1.7.1
pytest==8.3.3
pytest-cov==6.0.0
python-dateutil==2.9.0.post0
pytz==2024.2
PyYAML==6.0.2
rdflib==7.1.1
referencing==0.35.1
requests==2.32.3
requests-toolbelt==1.0.0
rfc3339-validator==0.1.4
rfc3987==1.3.8
rocrate @ git+https://github.com/salexan2001/ro-crate-py.git@858e5ef84a2f0425f138175435f92d377dc8d26f
rpds-py==0.21.0
ruamel.yaml==0.18.6
ruamel.yaml.clib==0.2.12
s3transfer==0.10.3
schema-salad==8.7.20241021092521
six==1.16.0
stack-data==0.6.3
tinydb==4.8.2
toml==0.10.2
tomlkit==0.13.2
traitlets==5.14.3
tuspy==1.0.3
types-python-dateutil==2.9.0.20241003
typing_extensions==4.12.2
tzdata==2024.2
uri-template==1.3.0
urllib3==2.2.3
wcwidth==0.2.13
webcolors==24.11.1
xlrd==2.0.1
xmlhelpy==0.14.0
yaml-header-tools==0.2.1
yarl==1.18.0
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment