diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ea7a39c0e07d2997a09a6b713c2b49e97a9d1e78..34c94e38249b7354f6feea661fd79a435336ea8d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,16 +19,25 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
- CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
+ # Taken from: https://forum.gitlab.com/t/clarification/54346
+ GITLAB_FEATURES: "$GITLAB_FEATURES,dependency_scanning"
stages:
- info
- setup
- - cert
- style
+ - code-analysis
- test
- deploy
+
+# TEMPLATES ==========================================================
+include:
+ - template: Jobs/SAST.gitlab-ci.yml
+ - template: Jobs/Dependency-Scanning.gitlab-ci.yml
+
+
.env: &env
- echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
@@ -52,51 +61,15 @@ e2e_tests:
- echo $KADITOKEN
- python -m pytest end-to-end-tests/test_kadi.py
-
-unittest_py3.9:
- tags: [cached-dind]
- stage: test
- image: python:3.9
- script: &python_test_script
- # TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
- - pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
- - pip install .[all]
- # actual test
- - pytest --cov=ruqad -vv ./unittests
-
-unittest_py3.10:
- tags: [cached-dind]
- stage: test
- image: python:3.10
- script: *python_test_script
-
-unittest_py3.11:
- tags: [cached-dind]
- stage: test
- image: python:3.11
- script: *python_test_script
-
-unittest_py3.12:
- tags: [cached-dind]
- stage: test
- image: python:3.12
- script: *python_test_script
-
-unittest_py3.13:
- tags: [cached-dind]
- stage: test
- image: python:3.13
- script: *python_test_script
-
build-testenv:
tags: [cached-dind]
image: docker:20.10
stage: setup
timeout: 2h
- only:
- - schedules
- - web
- - pushes
+ # rules:
+ # - if: $CI_PIPELINE_SOURCE == "push"
+ # - if: $CI_PIPELINE_SOURCE == "schedule"
+ # - if: $CI_PIPELINE_SOURCE == "web"
needs: []
script:
- df -h
@@ -117,21 +90,71 @@ code-style:
tags: [docker]
stage: style
image: $CI_REGISTRY_IMAGE
- needs:
- - job: build-testenv
- optional: true
+ needs: [build-testenv]
+ allow_failure: true
script:
- autopep8 -r --diff --exit-code .
- allow_failure: true
pylint:
tags: [docker]
stage: style
image: $CI_REGISTRY_IMAGE
- needs:
- - job: build-testenv
- optional: true
+ needs: [build-testenv]
allow_failure: true
script:
- - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+ - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+
+# SAST/SCA: Everything else is configured in the templates. This is only necessary so that the job
+# finds a matching runner and is run in the correct stage.
+
+# SAST: Static Application Security Testing
+sast:
+ tags: [docker]
+ needs: [info]
+ stage: code-analysis
+
+# Dependency scanning and vulnerabilities (SCA, Software Composition Analysis)
+gemnasium-python-dependency_scanning:
+ tags: [docker]
+ needs: [info]
+ stage: code-analysis
+
+unittest_py3.9:
+ tags: [cached-dind]
+ needs: [build-testenv]
+ stage: test
+ image: python:3.9
+ script: &python_test_script
+ # TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
+ - pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
+ - pip install .[all]
+ # actual test
+ - pytest --cov=ruqad -vv ./unittests
+
+unittest_py3.10:
+ tags: [cached-dind]
+ needs: [build-testenv]
+ stage: test
+ image: python:3.10
+ script: *python_test_script
+
+unittest_py3.11:
+ tags: [cached-dind]
+ needs: [build-testenv]
+ stage: test
+ image: python:3.11
+ script: *python_test_script
+
+unittest_py3.12:
+ tags: [cached-dind]
+ needs: [build-testenv]
+ stage: test
+ image: python:3.12
+ script: *python_test_script
+unittest_py3.13:
+ tags: [cached-dind]
+ needs: [build-testenv]
+ stage: test
+ image: python:3.13
+ script: *python_test_script
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 45541164eb3d04203fe8152780608ba6960ac5a1..5b4f3f70c523fa57300322c74876553ad92ace20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- LinkAhead crawler for the metadata check.
- Triggers the quality checker.
- Docker build file and instructions.
+- CI pipeline.
### Changed ###
diff --git a/Makefile b/Makefile
index aac219654c7180b28c014c78a5fbe5cfffaf60dc..514d4bbf44fce5cb39f8d980e332ffbd77c4e3b5 100644
--- a/Makefile
+++ b/Makefile
@@ -46,3 +46,8 @@ lint:
unittests:
tox -r
.PHONY: unittests
+
+bom:
+ echo '# generated by `pip freeze`' > requirements.txt
+ pip freeze --exclude-editable >> requirements.txt
+.PHONY: bom
diff --git a/README.md b/README.md
index 8aa2b7b6d47be00f291f5bfe94abb82827974587..a09687e4877ba90c9216ac23f611285ecc911976 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,9 @@ Simply install with:
`pip install .`
+Note: You can safely ignore the `requirements.txt`, this file is used as a lock file for components
+analysis. For more information, look at the section "SCA" below.
+
### Run locally ###
- Make sure that `qualitycheck_config.toml` and `secrets.sh` are filled with valied values.
@@ -29,7 +32,9 @@ Simply install with:
record.
- Wait for the new record with the file to be digested by the Ruqad monitor.
-### unit Tests
+## Development ##
+
+### Unit Tests
- For testing, install with the `test` extra, for example like so: `pip install .[test]`
- Then run `make unittest` or `pytest unittests/`.
@@ -44,15 +49,12 @@ In order to run the E2E test, you need to create a personal access token (pat) i
Run `make style lint` after installing with the `dev` extra. (The `dev` extra includes the `test`
extra.)
-<!-- ### Documentation -->
-
-<!-- Run `make doc` after installing the dependencies listed below. -->
-
-<!-- For building the documentation we require -->
+### SCA (Software component analysis) ###
-<!-- - `sphinx` -->
-<!-- - `recommonmark` -->
-<!-- - `sphinx-rtd-theme` -->
+To run the SCA job in the pipeline, run `make bom` and commit the generated `requirements.txt` to
+git. This will be used in the `gemnasium-python-dependency_scanning` job of the `code-analysis`
+stage to generate `gl-sbom-pypi-pip.cdx.json` file in CycloneDX format inside the artifacts zip
+file. This file can be visuealized for example with [Dependency Track](https://dependencytrack.org/).
## Docker deployment ##
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..158f0cec97f328da39f19870f7f04e25e85c21ae
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,108 @@
+# generated by `pip freeze`
+aiohappyeyeballs==2.4.3
+aiohttp==3.11.8
+aiosignal==1.3.1
+arcp==0.2.1
+arrow==1.3.0
+astroid==3.3.5
+asttokens==2.4.1
+attrs==24.2.0
+autopep8==2.3.1
+bioblend==1.4.0
+boto3==1.35.54
+botocore==1.35.54
+CacheControl==0.14.1
+caosadvancedtools==0.12.0
+certifi==2024.8.30
+charset-normalizer==3.4.0
+click==8.1.7
+coverage==7.6.8
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.9
+et_xmlfile==2.0.0
+executing==2.1.0
+filelock==3.16.1
+fqdn==1.5.1
+frozenlist==1.5.0
+future==1.0.0
+galaxy2cwl==0.1.4
+gxformat2==0.20.0
+idna==3.10
+importlib_resources==6.4.5
+iniconfig==2.0.0
+ipython==8.30.0
+isoduration==20.11.0
+isort==5.13.2
+jedi==0.19.1
+Jinja2==3.1.4
+jmespath==1.0.1
+jsonpointer==3.0.0
+jsonref==1.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+kadi-apy==0.43.0
+linkahead==0.16.0
+lxml==5.3.0
+MarkupSafe==3.0.2
+matplotlib-inline==0.1.7
+mccabe==0.7.0
+mistune==3.0.2
+msgpack==1.1.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+numpy==1.26.4
+odfpy==1.4.1
+openpyxl==3.1.5
+packaging==24.2
+pandas==2.2.3
+parso==0.8.4
+pexpect==4.9.0
+pip-autoremove==0.10.0
+platformdirs==4.3.6
+pluggy==1.5.0
+prompt_toolkit==3.0.48
+propcache==0.2.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyarrow==18.0.0
+pycodestyle==2.12.1
+Pygments==2.18.0
+pylint==3.3.1
+pyparsing==3.2.0
+PySocks==1.7.1
+pytest==8.3.3
+pytest-cov==6.0.0
+python-dateutil==2.9.0.post0
+pytz==2024.2
+PyYAML==6.0.2
+rdflib==7.1.1
+referencing==0.35.1
+requests==2.32.3
+requests-toolbelt==1.0.0
+rfc3339-validator==0.1.4
+rfc3987==1.3.8
+rocrate @ git+https://github.com/salexan2001/ro-crate-py.git@858e5ef84a2f0425f138175435f92d377dc8d26f
+rpds-py==0.21.0
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.12
+s3transfer==0.10.3
+schema-salad==8.7.20241021092521
+six==1.16.0
+stack-data==0.6.3
+tinydb==4.8.2
+toml==0.10.2
+tomlkit==0.13.2
+traitlets==5.14.3
+tuspy==1.0.3
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+tzdata==2024.2
+uri-template==1.3.0
+urllib3==2.2.3
+wcwidth==0.2.13
+webcolors==24.11.1
+xlrd==2.0.1
+xmlhelpy==0.14.0
+yaml-header-tools==0.2.1
+yarl==1.18.0