Merge branch 'f-fairness-check' of...

Merge branch 'f-fairness-check' of gitlab.indiscale.com:caosdb/customers/f-fit/ruqad into f-fairness-check

.gitlab-ci.yml

@@ -19,16 +19,25 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
+  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
+  # Taken from: https://forum.gitlab.com/t/clarification/54346
+  GITLAB_FEATURES: "$GITLAB_FEATURES,dependency_scanning"
 
 stages:
   - info
   - setup
-  - cert
   - style
+  - code-analysis
   - test
   - deploy
 
+
+# TEMPLATES ==========================================================
+include:
+  - template: Jobs/SAST.gitlab-ci.yml
+  - template: Jobs/Dependency-Scanning.gitlab-ci.yml
+
+
 .env: &env
   - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
   - echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
@@ -52,51 +61,15 @@ e2e_tests:
     - echo $KADITOKEN
     - python -m pytest end-to-end-tests/test_kadi.py
 
-
-unittest_py3.9:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.9
-  script: &python_test_script
-    # TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
-    - pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
-    - pip install .[all]
-    # actual test
-    - pytest --cov=ruqad -vv ./unittests
-
-unittest_py3.10:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.10
-  script: *python_test_script
-
-unittest_py3.11:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.11
-  script: *python_test_script
-
-unittest_py3.12:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.12
-  script: *python_test_script
-
-unittest_py3.13:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.13
-  script: *python_test_script
-
 build-testenv:
   tags: [cached-dind]
   image: docker:20.10
   stage: setup
   timeout: 2h
-  only:
-    - schedules
-    - web
-    - pushes
+  # rules:
+  #  - if: $CI_PIPELINE_SOURCE == "push"
+  #  - if: $CI_PIPELINE_SOURCE == "schedule"
+  #  - if: $CI_PIPELINE_SOURCE == "web"
   needs: []
   script:
       - df -h
@@ -117,21 +90,71 @@ code-style:
   tags: [docker]
   stage: style
   image: $CI_REGISTRY_IMAGE
-  needs:
-    - job: build-testenv
-      optional: true
+  needs: [build-testenv]
+  allow_failure: true
   script:
       - autopep8 -r --diff --exit-code .
-  allow_failure: true
 
 pylint:
   tags: [docker]
   stage: style
   image: $CI_REGISTRY_IMAGE
-  needs:
-    - job: build-testenv
-      optional: true
+  needs: [build-testenv]
   allow_failure: true
   script:
-    - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+      - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+
+# SAST/SCA: Everything else is configured in the templates.  This is only necessary so that the job
+# finds a matching runner and is run in the correct stage.
+
+# SAST: Static Application Security Testing
+sast:
+  tags: [docker]
+  needs: [info]
+  stage: code-analysis
+
+# Dependency scanning and vulnerabilities (SCA, Software Composition Analysis)
+gemnasium-python-dependency_scanning:
+  tags: [docker]
+  needs: [info]
+  stage: code-analysis
+
+unittest_py3.9:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.9
+  script: &python_test_script
+    # TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
+    - pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
+    - pip install .[all]
+    # actual test
+    - pytest --cov=ruqad -vv ./unittests
+
+unittest_py3.10:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.10
+  script: *python_test_script
+
+unittest_py3.11:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.11
+  script: *python_test_script
+
+unittest_py3.12:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.12
+  script: *python_test_script
 
+unittest_py3.13:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.13
+  script: *python_test_script

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - LinkAhead crawler for the metadata check.
 - Triggers the quality checker.
 - Docker build file and instructions.
+- CI pipeline.
 
 ### Changed ###
 

Makefile

@@ -46,3 +46,8 @@ lint:
 unittests:
 	tox -r
 .PHONY: unittests
+
+bom:
+	echo '# generated by `pip freeze`' > requirements.txt
+	pip freeze --exclude-editable >> requirements.txt
+.PHONY: bom

README.md

@@ -13,6 +13,9 @@ Simply install with:
 
 `pip install .`
 
+Note: You can safely ignore the `requirements.txt`, this file is used as a lock file for components
+analysis.  For more information, look at the section "SCA" below.
+
 ### Run locally ###
 
 - Make sure that `qualitycheck_config.toml` and `secrets.sh` are filled with valied values.
@@ -29,7 +32,9 @@ Simply install with:
     record.
   - Wait for the new record with the file to be digested by the Ruqad monitor.
 
-### unit Tests
+## Development ##
+
+### Unit Tests
 
 - For testing, install with the `test` extra, for example like so: `pip install .[test]`
 - Then run `make unittest` or `pytest unittests/`.
@@ -44,15 +49,12 @@ In order to run the E2E test, you need to create a personal access token (pat) i
 Run `make style lint` after installing with the `dev` extra.  (The `dev` extra includes the `test`
 extra.)
 
-<!-- ### Documentation -->
-
-<!-- Run `make doc` after installing the dependencies listed below. -->
-
-<!-- For building the documentation we require -->
+### SCA (Software component analysis) ###
 
-<!-- - `sphinx` -->
-<!-- - `recommonmark`  -->
-<!-- - `sphinx-rtd-theme` -->
+To run the SCA job in the pipeline, run `make bom` and commit the generated `requirements.txt` to
+git.  This will be used in the `gemnasium-python-dependency_scanning` job of the `code-analysis`
+stage to generate `gl-sbom-pypi-pip.cdx.json` file in CycloneDX format inside the artifacts zip
+file.  This file can be visuealized for example with [Dependency Track](https://dependencytrack.org/).
 
 ## Docker deployment ##
 

requirements.txt

+# generated by `pip freeze`
+aiohappyeyeballs==2.4.3
+aiohttp==3.11.8
+aiosignal==1.3.1
+arcp==0.2.1
+arrow==1.3.0
+astroid==3.3.5
+asttokens==2.4.1
+attrs==24.2.0
+autopep8==2.3.1
+bioblend==1.4.0
+boto3==1.35.54
+botocore==1.35.54
+CacheControl==0.14.1
+caosadvancedtools==0.12.0
+certifi==2024.8.30
+charset-normalizer==3.4.0
+click==8.1.7
+coverage==7.6.8
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.9
+et_xmlfile==2.0.0
+executing==2.1.0
+filelock==3.16.1
+fqdn==1.5.1
+frozenlist==1.5.0
+future==1.0.0
+galaxy2cwl==0.1.4
+gxformat2==0.20.0
+idna==3.10
+importlib_resources==6.4.5
+iniconfig==2.0.0
+ipython==8.30.0
+isoduration==20.11.0
+isort==5.13.2
+jedi==0.19.1
+Jinja2==3.1.4
+jmespath==1.0.1
+jsonpointer==3.0.0
+jsonref==1.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+kadi-apy==0.43.0
+linkahead==0.16.0
+lxml==5.3.0
+MarkupSafe==3.0.2
+matplotlib-inline==0.1.7
+mccabe==0.7.0
+mistune==3.0.2
+msgpack==1.1.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+numpy==1.26.4
+odfpy==1.4.1
+openpyxl==3.1.5
+packaging==24.2
+pandas==2.2.3
+parso==0.8.4
+pexpect==4.9.0
+pip-autoremove==0.10.0
+platformdirs==4.3.6
+pluggy==1.5.0
+prompt_toolkit==3.0.48
+propcache==0.2.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyarrow==18.0.0
+pycodestyle==2.12.1
+Pygments==2.18.0
+pylint==3.3.1
+pyparsing==3.2.0
+PySocks==1.7.1
+pytest==8.3.3
+pytest-cov==6.0.0
+python-dateutil==2.9.0.post0
+pytz==2024.2
+PyYAML==6.0.2
+rdflib==7.1.1
+referencing==0.35.1
+requests==2.32.3
+requests-toolbelt==1.0.0
+rfc3339-validator==0.1.4
+rfc3987==1.3.8
+rocrate @ git+https://github.com/salexan2001/ro-crate-py.git@858e5ef84a2f0425f138175435f92d377dc8d26f
+rpds-py==0.21.0
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.12
+s3transfer==0.10.3
+schema-salad==8.7.20241021092521
+six==1.16.0
+stack-data==0.6.3
+tinydb==4.8.2
+toml==0.10.2
+tomlkit==0.13.2
+traitlets==5.14.3
+tuspy==1.0.3
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+tzdata==2024.2
+uri-template==1.3.0
+urllib3==2.2.3
+wcwidth==0.2.13
+webcolors==24.11.1
+xlrd==2.0.1
+xmlhelpy==0.14.0
+yaml-header-tools==0.2.1
+yarl==1.18.0