ENH: SAST / SCA

• New pipeline jobs.
• New Makefile target for requirements.txt.
• Issue: caosdb/customers/f-fit/management#23

.gitlab-ci.yml

@@ -19,16 +19,25 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
+  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
+  # Taken from: https://forum.gitlab.com/t/clarification/54346
+  GITLAB_FEATURES: "$GITLAB_FEATURES,dependency_scanning"
 
 stages:
   - info
   - setup
-  - cert
   - style
+  - code-analysis
   - test
   - deploy
 
+
+# TEMPLATES ==========================================================
+include:
+  - template: Jobs/SAST.gitlab-ci.yml
+  - template: Jobs/Dependency-Scanning.gitlab-ci.yml
+
+
 .env: &env
   - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
   - echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
@@ -52,51 +61,15 @@ e2e_tests:
     - echo $KADITOKEN
     - python -m pytest end-to-end-tests/test_kadi.py
 
-
-unittest_py3.9:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.9
-  script: &python_test_script
-    # TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
-    - pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
-    - pip install .[all]
-    # actual test
-    - pytest --cov=ruqad -vv ./unittests
-
-unittest_py3.10:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.10
-  script: *python_test_script
-
-unittest_py3.11:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.11
-  script: *python_test_script
-
-unittest_py3.12:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.12
-  script: *python_test_script
-
-unittest_py3.13:
-  tags: [cached-dind]
-  stage: test
-  image: python:3.13
-  script: *python_test_script
-
 build-testenv:
   tags: [cached-dind]
   image: docker:20.10
   stage: setup
   timeout: 2h
-  only:
-    - schedules
-    - web
-    - pushes
+  # rules:
+  #  - if: $CI_PIPELINE_SOURCE == "push"
+  #  - if: $CI_PIPELINE_SOURCE == "schedule"
+  #  - if: $CI_PIPELINE_SOURCE == "web"
   needs: []
   script:
       - df -h
@@ -117,21 +90,71 @@ code-style:
   tags: [docker]
   stage: style
   image: $CI_REGISTRY_IMAGE
-  needs:
-    - job: build-testenv
-      optional: true
+  needs: [build-testenv]
+  allow_failure: true
   script:
       - autopep8 -r --diff --exit-code .
-  allow_failure: true
 
 pylint:
   tags: [docker]
   stage: style
   image: $CI_REGISTRY_IMAGE
-  needs:
-    - job: build-testenv
-      optional: true
+  needs: [build-testenv]
   allow_failure: true
   script:
-    - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+      - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+
+# SAST/SCA: Everything else is configured in the templates.  This is only necessary so that the job
+# finds a matching runner and is run in the correct stage.
+
+# SAST: Static Application Security Testing
+sast:
+  tags: [docker]
+  needs: [info]
+  stage: code-analysis
+
+# Dependency scanning and vulnerabilities (SCA, Software Composition Analysis)
+gemnasium-python-dependency_scanning:
+  tags: [docker]
+  needs: [info]
+  stage: code-analysis
+
+unittest_py3.9:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.9
+  script: &python_test_script
+    # TODO Remove this manual crawler installation after the crawler has been released in version 0.10.2
+    - pip install --root-user-action=ignore git+https://gitlab.indiscale.com/caosdb/src/caosdb-crawler.git@dev
+    - pip install .[all]
+    # actual test
+    - pytest --cov=ruqad -vv ./unittests
+
+unittest_py3.10:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.10
+  script: *python_test_script
+
+unittest_py3.11:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.11
+  script: *python_test_script
+
+unittest_py3.12:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.12
+  script: *python_test_script
 
+unittest_py3.13:
+  tags: [cached-dind]
+  needs: [build-testenv]
+  stage: test
+  image: python:3.13
+  script: *python_test_script