diff --git a/pyproject.toml b/pyproject.toml index b0739c1f4d6e8b0aa382407901eb806d881d9342..48d12970c26a6a4bbfdf96a8842a9093c6a3e26c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dev = [ test = [ "tox", "pytest", + "pytest-env", "pytest-cov", ] all = [ @@ -58,3 +59,11 @@ rq_qualitycheck = "ruqad.qualitycheck:main" [tool.setuptools.package-data] ruqad = ["resources/**/*"] + +[tool.pytest.ini_options] +env = [ + "S3_ACCESS_KEY_ID=1234", + "S3_SECRET_ACCESS_KEY=1234", + "GITLAB_PIPELINE_TOKEN=1234", + "GITLAB_API_TOKEN=1234", +] \ No newline at end of file diff --git a/src/ruqad/qualitycheck.py b/src/ruqad/qualitycheck.py index 9ba034590f94b12f9d2756be0eafb8fcc38fbb66..b846e593158aaf9b582006455e80bbcc98061e5e 100755 --- a/src/ruqad/qualitycheck.py +++ b/src/ruqad/qualitycheck.py @@ -155,7 +155,7 @@ This deletes all the objects in the bucket. zipf = ZipFile(filename) zipf.extractall(path=tmp) # TODO Zip bomb detection and prevention. for name in zipf.namelist(): - if name.endswith(".json"): + if name.endswith(".json") or name.endswith(os.path.sep): continue if upload: self._upload(os.path.join(tmp, name), remove_prefix=tmp) diff --git a/unittests/test_qualitycheck.py b/unittests/test_qualitycheck.py new file mode 100644 index 0000000000000000000000000000000000000000..dca6a761607aa5a3dd2ff51d18dbfaeb6e429665 --- /dev/null +++ b/unittests/test_qualitycheck.py @@ -0,0 +1,43 @@ +# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +"""Unit tests for the QualityChecker.""" + +from datetime import datetime +from pathlib import Path +from unittest.mock import patch, Mock + +from ruqad import qualitycheck + + +@patch("boto3.Session.client") +def test_qc_internal(mock_s3_client): + zipfile = (Path(__file__).parents[1] / "end-to-end-tests" / "data" / "crawler_data" / "ruqad" / + "1223" / "export.eln") + qc = qualitycheck.QualityChecker() + qc._extract_content(zipfile, upload=True) + correct_call = False + for call in mock_s3_client.mock_calls: + if not call[0] == '().upload_file': + continue + if (len(call.args) == 3 + and call.args[0].endswith("abalone2.csv") + and call.args[1] == "ruqad" + and call.args[2] == + "data/test-crawler-second/test-crawler-second/files/abalone2.csv"): + correct_call = True + break + assert correct_call