diff --git a/README_SETUP.md b/README_SETUP.md
index 9da548395073643c16539cef180c4d6412dd8d46..e58f934ceba176e4b5ba42239565f8e3bd48171a 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -11,6 +11,10 @@ typically be installed automatically):
- `PyYaml`
- `PySocks`
+Optional packages:
+- `keyring`
+- `jsonschema`
+
### How to install ###
#### Linux ####
@@ -30,12 +34,14 @@ packages you will ever need out of the box. If you prefer, you may also install
After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
installation](#generic-installation) section.
-#### iOS ####
+#### MacOS ####
-If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
-package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/). After installation
-from python.org, it is recommended to also update the TLS certificates for Python (this requires
-administrator rights for your user):
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
```sh
# Replace this with your Python version number:
@@ -45,7 +51,8 @@ cd /Applications/Python\ 3.9/
sudo ./Install\ Certificates.command
```
-After these steps, you may continue with the [Generic installation](#generic-installation).
+After these steps, you may continue with the [Generic
+installation](#generic-installation).
#### Generic installation ####
@@ -66,6 +73,13 @@ cd caosdb-pylib
pip3 install --user .
```
+For installation of optional packages, install with an additional option, e.g. for
+validating with the caosdb json schema:
+
+```sh
+pip3 install --user .[jsonschema]
+```
+
## Configuration ##
The configuration is done using `ini` configuration files.
diff --git a/setup.py b/setup.py
index e1d39458ea8d1b0b17ea12a82ebd7133b27b045a..d0e12f1a88f43883ede5e53c90512968ca310e93 100755
--- a/setup.py
+++ b/setup.py
@@ -159,11 +159,12 @@ def setup_package():
package_dir={'': 'src'},
install_requires=['lxml>=3.6.4',
'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
- extras_require={'keyring': ['keyring>=13.0.0']},
+ extras_require={'keyring': ['keyring>=13.0.0'],
+ 'jsonschema': ['jsonschema==4.0.1']},
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
+ tests_require=["pytest", "pytest-cov", "coverage>=4.4.2", "jsonschema==4.0.1"],
package_data={
- 'caosdb': ['cert/indiscale.ca.crt'],
+ 'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
scripts=["src/caosdb/utils/caosdb_admin.py"]
)
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 842f1ee62a5b3178b7305e4c1e0c281a2dbd3b38..314a1a1d0eb25091333c7b5528e32512a3679d96 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -21,6 +21,16 @@
#
# ** end header
#
+
+import os
+import yaml
+import warnings
+try:
+ optional_jsonschema_validate = None
+ from jsonschema import validate as optional_jsonschema_validate
+except ImportError:
+ pass
+
try:
# python2
from ConfigParser import ConfigParser
@@ -47,7 +57,9 @@ def configure(inifile):
_pycaosdbconf = None
if _pycaosdbconf is None:
_reset_config()
- return _pycaosdbconf.read(inifile)
+ read_config = _pycaosdbconf.read(inifile)
+ validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+ return read_config
def get_config():
@@ -55,6 +67,33 @@ def get_config():
return _pycaosdbconf
+def config_to_yaml(config):
+ valobj = {}
+ for s in config.sections():
+ valobj[s] = {}
+ for key, value in config[s].items():
+ # TODO: Can the type be inferred from the config object?
+ if key in ["timeout", "debug"]:
+ valobj[s][key] = int(value)
+ elif key in ["ssl_insecure"]:
+ valobj[s][key] = bool(value)
+ else:
+ valobj[s][key] = value
+
+ return valobj
+
+
+def validate_yaml_schema(valobj):
+ if optional_jsonschema_validate:
+ with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
+ schema = yaml.load(f, Loader=yaml.SafeLoader)
+ optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+ else:
+ warnings.warn("""
+ Warning: The validation could not be performed because `jsonschema` is not installed.
+ """)
+
+
def _read_config_files():
"""Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
new file mode 100644
index 0000000000000000000000000000000000000000..fd49c2ffa63e7bc334f8177ff79575f46718f38c
--- /dev/null
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -0,0 +1,87 @@
+schema-pycaosdb-ini:
+ type: object
+ additionalProperties: false
+ properties:
+ Container:
+ additionalProperties: false
+ properties:
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ Connection:
+ description: Settings for the connection to the CaosDB server
+ additionalProperties: false
+ properties:
+ url:
+ description: URL of the CaosDB server
+ type: string
+ pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: [https://demo.indiscale.com/, https://localhost:10443/]
+ username:
+ type: string
+ description: User name used for authentication with the server
+ examples: [admin]
+ password_method:
+ description: The password input method defines how the password is supplied that is used for authentication with the server.
+ type: string
+ default: input
+ enum: [input, plain, pass, keyring]
+ password_identifier:
+ type: string
+ password:
+ type: string
+ examples: [caosdb]
+ auth_token:
+ type: string
+ description: Using an authentication token to connect with the server. This setting is not recommended for users.
+ cacert:
+ type: string
+ description: If the server's SSL certificate cannot be validated by your installed certificates (default or installed by your admins), you may also need to supply the matching key (pem file)
+ examples: [/path/to/caosdb.ca.pem]
+ ssl_insecure:
+ description: If this option is set, the SSL certificate of the server will not be validated. This has the potential of a man-in-the-middle attack. Use with care!
+ type: boolean
+ default: false
+ ssl_version:
+ description: You may define the ssl version to be used. It has to be the name of the corresponding attribute in the Python ssl module.
+ examples: [PROTOCOL_TLS]
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ description: The debug key allows control the verbosity. Set it to 1 or 2 in case you want to see debugging output or if you want to learn more about the internals of the protocol. 0 disables debugging output.
+ socket_proxy:
+ examples: [localhost:12345]
+ type: string
+ description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ implementation:
+ description: This option is used internally and for testing. Do not override.
+ examples: [_DefaultCaosDBServerConnection]
+ timeout:
+ type: integer
+ allOf:
+ - if:
+ properties:
+ password_method:
+ const: input
+ then:
+ required: [url]
+ - if:
+ properties:
+ password_method:
+ const: plain
+ then:
+ required: [url, username, password]
+ - if:
+ properties:
+ password_method:
+ const: pass
+ then:
+ required: [url, username, password_identifier]
+ - if:
+ properties:
+ password_method:
+ const: keyring
+ then:
+ required: [url, username]
diff --git a/tox.ini b/tox.ini
index 94c2dc8affb280d3e7f6cff4536636432c9f7749..22c89f765c612ff78572ee2cab20dfab2e740e84 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,4 +7,5 @@ deps = .
nose
pytest
pytest-cov
+ jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
diff --git a/unittests/broken_configs/pycaosdb1.ini b/unittests/broken_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..71180286881399c35e251bba89a54a345cd948ac
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb1.ini
@@ -0,0 +1,5 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+username=username
+password_method=pass
diff --git a/unittests/broken_configs/pycaosdb2.ini b/unittests/broken_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6bdf58ea812b83a622e647b2a18b01bb1a1e3099
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=9
diff --git a/unittests/broken_configs/pycaosdb3.ini b/unittests/broken_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..62d1fed9497c0c258c13aa2ed8fecb23a3006849
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/broken_configs/pycaosdb4.ini b/unittests/broken_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e96604ac453ff2be4e2b419aa9ccbbf3598fa231
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb4.ini
@@ -0,0 +1,10 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+key=bla
\ No newline at end of file
diff --git a/unittests/test_configs/pycaosdb-indiscale-demo.ini b/unittests/test_configs/pycaosdb-indiscale-demo.ini
new file mode 100644
index 0000000000000000000000000000000000000000..9010343467f78c7c9c3e25ea3a57520deac18c8e
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-indiscale-demo.ini
@@ -0,0 +1,12 @@
+[Connection]
+url=https://demo.indiscale.com/
+username=admin
+password=caosdb
+password_method=plain
+cacert=/etc/ssl/cert.pem
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
\ No newline at end of file
diff --git a/unittests/test_configs/pycaosdb1.ini b/unittests/test_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..dcfa7c21fac735d81ab92b33f0abd31df25fc1ad
--- /dev/null
+++ b/unittests/test_configs/pycaosdb1.ini
@@ -0,0 +1,6 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+password_identifier=SECTION/caosdb
+username=username
+password_method=pass
diff --git a/unittests/test_configs/pycaosdb2.ini b/unittests/test_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e5493bde3be0f84f38e427edb4d42fba9c75482d
--- /dev/null
+++ b/unittests/test_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=0
diff --git a/unittests/test_configs/pycaosdb3.ini b/unittests/test_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6c4934039c99855566f38c69f7511d774f81efbd
--- /dev/null
+++ b/unittests/test_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..387518c9a30aac45facc24d77ea8c8532a4a8b16
--- /dev/null
+++ b/unittests/test_schema.py
@@ -0,0 +1,26 @@
+#!/bin/python
+# Test configuration schema
+# A. Schlemmer, 01/2021
+
+from jsonschema.exceptions import ValidationError
+from pytest import raises
+from glob import glob
+import os
+from caosdb.configuration import config_to_yaml, validate_yaml_schema
+from configparser import ConfigParser
+
+
+def test_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+ c = ConfigParser()
+ c.read(fn)
+ validate_yaml_schema(config_to_yaml(c))
+
+
+def test_broken_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
+ print(fn)
+ with raises(ValidationError):
+ c = ConfigParser()
+ c.read(fn)
+ validate_yaml_schema(config_to_yaml(c))