WIP: http proxy

setup.py

@@ -171,7 +171,12 @@ def setup_package():
         python_requires='>=3.8',
         package_dir={'': 'src'},
         install_requires=['lxml>=4.6.3',
-                          'PyYAML>=5.4.1', 'future', 'PySocks>=1.6.7'],
+                          "requests>=2.28.1",
+                          "python-dateutil>=2.8.2",
+                          'PyYAML>=5.4.1',
+                          'future',
+                          'PySocks>=1.6.7',
+                         ],
         extras_require={'keyring': ['keyring>=13.0.0'],
                         'jsonschema': ['jsonschema>=4.4.0']},
         setup_requires=["pytest-runner>=2.0,<3dev"],

src/caosdb/connection/connection.py

@@ -31,6 +31,9 @@ import sys
 from builtins import str  # pylint: disable=redefined-builtin
 from errno import EPIPE as BrokenPipe
 from socket import error as SocketError
+from urllib.parse import urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
 
 from caosdb.configuration import get_config
 from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -63,6 +66,32 @@ except ImportError:
 _LOGGER = logging.getLogger(__name__)
 
 
+class _WrappedHTTPResponse2(CaosDBHTTPResponse):
+
+    def __init__(self, response):
+        self.response = response
+
+    @property
+    def reason(self):
+        return self.response.reason
+
+    @property
+    def status(self):
+        return self.response.status_code
+
+    def read(self, size=None):
+        return self.response.raw.read(size)
+
+    def getheader(self, name, default=None):
+        return self.response.headers[name] if name in self.response.headers else default
+
+    def getheaders(self):
+        return self.response.headers.items()
+
+    def close(self):
+        self.response.close()
+
+
 class _WrappedHTTPResponse(CaosDBHTTPResponse):
 
     def __init__(self, response):
@@ -101,7 +130,6 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
     def __init__(self):
         self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
             version=version, implementation=type(self).__name__))
-        self._http_con = None
         self._base_path = None
 
     def request(self, method, path, headers=None, body=None, **kwargs):
@@ -133,8 +161,25 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
         if headers is None:
             headers = {}
         headers["User-Agent"] = self._useragent
+
         try:
-            self._http_con = StreamingHTTPSConnection(
+            if self.setup_fields["https_proxy"] is not None:
+                session = HTTPSession()
+                session.proxies = {
+                    "https": self.setup_fields["https_proxy"]
+                }
+                response = session.request(method=method,
+                                           url=self.setup_fields["url_base_path"] + path,
+                                           headers=headers, data=body, stream=True)
+                return _WrappedHTTPResponse2(response)
+        except HTTPConnectionError as conn_err:
+            raise CaosDBConnectionError(
+                "Connection failed. Network or server down? " + str(conn_err)
+            )
+
+
+        try:
+            self._http_con = ProxyConnection(
                 # TODO looks as if configure needs to be done first.
                 # That is however not assured.
                 host=self.setup_fields["host"],
@@ -213,14 +258,24 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
                 "file.")
 
         socket_proxy = None
-
         if "socket_proxy" in config:
             socket_proxy = config["socket_proxy"]
 
+        https_proxy = None
+        if "https_proxy" in config:
+            result = urlparse(config["https_proxy"], scheme="http")
+            if result.scheme != "http":
+                raise ValueError(
+                    "The `https_proxy` parameter or config option must "
+                    "contain a valid http uri or None")
+            https_proxy = result.scheme + "://" + result.netloc
+
         self.setup_fields = {
+            "url_base_path": config["url"] + "/",
             "host": host,
             "timeout": int(config.get("timeout")),
             "context": context,
+            "https_proxy": https_proxy,
             "socket_proxy": socket_proxy}
 
 
@@ -342,6 +397,11 @@ def configure_connection(**kwargs):
         An authentication token which has been issued by the CaosDB Server.
         Implies `password_method="auth_token"` if set.  An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
 
+    https_proxy : str
+        Define a https proxy, e.g. `https://localhost:8888`. Currently,
+        authentication against the proxy and non-TLS connections are not
+        supported.  (Default: None)
+
     implementation : CaosDBServerConnection
         The class which implements the connection. (Default:
         _DefaultCaosDBServerConnection)

src/caosdb/connection/streaminghttp.py

@@ -70,13 +70,22 @@ class StreamingHTTPSConnection(client.HTTPSConnection, object):
     that overrides the `send()` method to support iterable body objects."""
     # pylint: disable=unused-argument, arguments-differ
 
-    def __init__(self, socket_proxy=None, **kwargs):
+    def __init__(self, socket_proxy=None, https_proxy=None, **kwargs):
+        host = kwargs["host"]
+        port = int(kwargs["port"]) if "port" in kwargs else None
         if socket_proxy is not None:
-            host, port = socket_proxy.split(":")
-            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
-                                  int(port))
+            proxy_host, proxy_port = socket_proxy.split(":")
+            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, proxy_host,
+                                  int(proxy_port))
             socket.socket = socks.socksocket
+        if https_proxy is not None:
+            tunnel_host = host
+            tunnel_port = port
+            host, port = https_proxy.split(":")
+            port = int(port)
         super(StreamingHTTPSConnection, self).__init__(**kwargs)
+        if tunnel_host is not None:
+            self.set_tunnel(host=host, port=port)
 
     def _send_output(self, body, **kwargs):
         """Send the currently buffered request and clear the buffer.

src/caosdb/schema-pycaosdb-ini.yml

@@ -55,6 +55,10 @@ schema-pycaosdb-ini:
           examples: ["localhost:12345"]
           type: string
           description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+        https_proxy:
+          examples: ["https://localhost:8888"]
+          type: string
+          description: Define a HTTPS Proxy. Currently, authentication against the proxy and non-TLS HTTP connections are not supported.
         implementation:
           description: This option is used internally and for testing. Do not override.
           examples: [_DefaultCaosDBServerConnection]

tox.ini

@@ -7,7 +7,6 @@ deps = .
     nose
     pytest
     pytest-cov
-    python-dateutil
     jsonschema==4.0.1
 commands=py.test --cov=caosdb -vv {posargs}