Merge branch 'f-fix-json-schema' into 'dev'

Revert "Revert "Merge branch 'f-validate-config' into 'dev'""

See merge request !29

README_SETUP.md

@@ -11,6 +11,10 @@ typically be installed automatically):
 - `PyYaml`
 - `PySocks`
 
+Optional packages:
+- `keyring`
+- `jsonschema`
+
 ### How to install ###
 
 #### Linux ####
@@ -30,12 +34,14 @@ packages you will ever need out of the box.  If you prefer, you may also install
 After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
 installation](#generic-installation) section.
 
-#### iOS ####
+#### MacOS ####
 
-If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
-package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/).  After installation
-from python.org, it is recommended to also update the TLS certificates for Python (this requires
-administrator rights for your user):
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
 
 ```sh
 # Replace this with your Python version number:
@@ -45,7 +51,8 @@ cd /Applications/Python\ 3.9/
 sudo ./Install\ Certificates.command
 ```
 
-After these steps, you may continue with the [Generic installation](#generic-installation).
+After these steps, you may continue with the [Generic
+installation](#generic-installation).
 
 #### Generic installation ####
 
@@ -66,6 +73,13 @@ cd caosdb-pylib
 pip3 install --user .
 ```
 
+For installation of optional packages, install with an additional option, e.g. for 
+validating with the caosdb json schema:
+
+```sh
+pip3 install --user .[jsonschema]
+```
+
 ## Configuration ##
 
 The  configuration is done using `ini` configuration files.

setup.py

@@ -159,11 +159,12 @@ def setup_package():
         package_dir={'': 'src'},
         install_requires=['lxml>=3.6.4',
                           'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
-        extras_require={'keyring': ['keyring>=13.0.0']},
+        extras_require={'keyring': ['keyring>=13.0.0'],
+                        'jsonschema': ['jsonschema==4.0.1']},
         setup_requires=["pytest-runner>=2.0,<3dev"],
-        tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
+        tests_require=["pytest", "pytest-cov", "coverage>=4.4.2", "jsonschema==4.0.1"],
         package_data={
-            'caosdb': ['cert/indiscale.ca.crt'],
+            'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
         },
         scripts=["src/caosdb/utils/caosdb_admin.py"]
     )

src/caosdb/configuration.py

@@ -21,6 +21,16 @@
 #
 # ** end header
 #
+
+import os
+import yaml
+import warnings
+try:
+    optional_jsonschema_validate = None
+    from jsonschema import validate as optional_jsonschema_validate
+except ImportError:
+    pass
+
 try:
     # python2
     from ConfigParser import ConfigParser
@@ -47,7 +57,9 @@ def configure(inifile):
         _pycaosdbconf = None
     if _pycaosdbconf is None:
         _reset_config()
-    return _pycaosdbconf.read(inifile)
+    read_config = _pycaosdbconf.read(inifile)
+    validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+    return read_config
 
 
 def get_config():
@@ -55,6 +67,33 @@ def get_config():
     return _pycaosdbconf
 
 
+def config_to_yaml(config):
+    valobj = {}
+    for s in config.sections():
+        valobj[s] = {}
+        for key, value in config[s].items():
+            # TODO: Can the type be inferred from the config object?
+            if key in ["timeout", "debug"]:
+                valobj[s][key] = int(value)
+            elif key in ["ssl_insecure"]:
+                valobj[s][key] = bool(value)
+            else:
+                valobj[s][key] = value
+
+    return valobj
+
+
+def validate_yaml_schema(valobj):
+    if optional_jsonschema_validate:
+        with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
+            schema = yaml.load(f, Loader=yaml.SafeLoader)
+        optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+    else:
+        warnings.warn("""
+                        Warning: The validation could not be performed because `jsonschema` is not installed.
+                    """)
+
+
 def _read_config_files():
     """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
 

src/caosdb/schema-pycaosdb-ini.yml

+schema-pycaosdb-ini:
+  type: object
+  additionalProperties: false
+  properties:
+    Container:
+      additionalProperties: false
+      properties:
+        debug:
+          default: 0
+          type: integer
+          enum: [0, 1, 2]
+    Connection:
+      description: Settings for the connection to the CaosDB server
+      additionalProperties: false
+      properties:
+        url:
+          description: URL of the CaosDB server
+          type: string
+          pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+          examples: [https://demo.indiscale.com/, https://localhost:10443/]
+        username:
+          type: string
+          description: User name used for authentication with the server
+          examples: [admin]
+        password_method:
+          description: The password input method defines how the password is supplied that is used for authentication with the server.
+          type: string
+          default: input
+          enum: [input, plain, pass, keyring]
+        password_identifier:
+          type: string
+        password:
+          type: string
+          examples: [caosdb]
+        auth_token:
+          type: string
+          description: Using an authentication token to connect with the server. This setting is not recommended for users.
+        cacert:
+          type: string
+          description: If the server's SSL certificate cannot be validated by your installed certificates (default or installed by your admins), you may also need to supply the matching key (pem file)
+          examples: [/path/to/caosdb.ca.pem]
+        ssl_insecure:
+          description: If this option is set, the SSL certificate of the server will not be validated. This has the potential of a man-in-the-middle attack. Use with care!
+          type: boolean
+          default: false
+        ssl_version:
+            description: You may define the ssl version to be used. It has to be the name of the corresponding attribute in the Python ssl module.
+            examples: [PROTOCOL_TLS]
+        debug:
+          default: 0
+          type: integer
+          enum: [0, 1, 2]
+          description: The debug key allows control the verbosity. Set it to 1 or 2 in case you  want to see debugging output or if you want to learn more about the internals of the protocol.  0 disables debugging output.
+        socket_proxy:
+          examples: [localhost:12345]
+          type: string
+          description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+        implementation:
+          description: This option is used internally and for testing. Do not override.
+          examples: [_DefaultCaosDBServerConnection]
+        timeout:
+          type: integer
+      allOf:
+        - if:
+            properties:
+              password_method:
+                const: input
+          then:
+            required: [url]
+        - if:
+            properties:
+              password_method:
+                const: plain
+          then:
+            required: [url, username, password]
+        - if:
+            properties:
+              password_method:
+                const: pass
+          then:
+            required: [url, username, password_identifier]
+        - if:
+            properties:
+              password_method:
+                const: keyring
+          then:
+            required: [url, username]

tox.ini

@@ -7,4 +7,5 @@ deps = .
     nose
     pytest
     pytest-cov
+    jsonschema==4.0.1
 commands=py.test --cov=caosdb -vv {posargs}

unittests/broken_configs/pycaosdb1.ini

+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+username=username
+password_method=pass

unittests/broken_configs/pycaosdb2.ini

+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=9

unittests/broken_configs/pycaosdb3.ini

+[connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0

unittests/broken_configs/pycaosdb4.ini

+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+key=bla
\ No newline at end of file

unittests/test_configs/pycaosdb-indiscale-demo.ini

+[Connection]
+url=https://demo.indiscale.com/
+username=admin
+password=caosdb
+password_method=plain
+cacert=/etc/ssl/cert.pem
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
\ No newline at end of file

unittests/test_configs/pycaosdb1.ini

+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+password_identifier=SECTION/caosdb
+username=username
+password_method=pass

unittests/test_configs/pycaosdb2.ini

+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=0

unittests/test_configs/pycaosdb3.ini

+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0

unittests/test_schema.py

+#!/bin/python
+# Test configuration schema
+# A. Schlemmer, 01/2021
+
+from jsonschema.exceptions import ValidationError
+from pytest import raises
+from glob import glob
+import os
+from caosdb.configuration import config_to_yaml, validate_yaml_schema
+from configparser import ConfigParser
+
+
+def test_config_files():
+    for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+        c = ConfigParser()
+        c.read(fn)
+        validate_yaml_schema(config_to_yaml(c))
+
+
+def test_broken_config_files():
+    for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
+        print(fn)
+        with raises(ValidationError):
+            c = ConfigParser()
+            c.read(fn)
+            validate_yaml_schema(config_to_yaml(c))