From 4c6866773598b9be3a40d925f376099743d5509f Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Wed, 2 Nov 2022 15:34:24 +0100
Subject: [PATCH] WIP: http proxy
---
setup.py | 7 ++-
src/caosdb/connection/connection.py | 66 ++++++++++++++++++++++++--
src/caosdb/connection/streaminghttp.py | 17 +++++--
src/caosdb/schema-pycaosdb-ini.yml | 4 ++
tox.ini | 1 -
5 files changed, 86 insertions(+), 9 deletions(-)
diff --git a/setup.py b/setup.py
index e044d1f5..50c5b89e 100755
--- a/setup.py
+++ b/setup.py
@@ -171,7 +171,12 @@ def setup_package():
python_requires='>=3.8',
package_dir={'': 'src'},
install_requires=['lxml>=4.6.3',
- 'PyYAML>=5.4.1', 'future', 'PySocks>=1.6.7'],
+ "requests>=2.28.1",
+ "python-dateutil>=2.8.2",
+ 'PyYAML>=5.4.1',
+ 'future',
+ 'PySocks>=1.6.7',
+ ],
extras_require={'keyring': ['keyring>=13.0.0'],
'jsonschema': ['jsonschema>=4.4.0']},
setup_requires=["pytest-runner>=2.0,<3dev"],
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 43eb3410..3fff3b7d 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -31,6 +31,9 @@ import sys
from builtins import str # pylint: disable=redefined-builtin
from errno import EPIPE as BrokenPipe
from socket import error as SocketError
+from urllib.parse import urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
from caosdb.configuration import get_config
from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -63,6 +66,32 @@ except ImportError:
_LOGGER = logging.getLogger(__name__)
+class _WrappedHTTPResponse2(CaosDBHTTPResponse):
+
+ def __init__(self, response):
+ self.response = response
+
+ @property
+ def reason(self):
+ return self.response.reason
+
+ @property
+ def status(self):
+ return self.response.status_code
+
+ def read(self, size=None):
+ return self.response.raw.read(size)
+
+ def getheader(self, name, default=None):
+ return self.response.headers[name] if name in self.response.headers else default
+
+ def getheaders(self):
+ return self.response.headers.items()
+
+ def close(self):
+ self.response.close()
+
+
class _WrappedHTTPResponse(CaosDBHTTPResponse):
def __init__(self, response):
@@ -101,7 +130,6 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
def __init__(self):
self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
version=version, implementation=type(self).__name__))
- self._http_con = None
self._base_path = None
def request(self, method, path, headers=None, body=None, **kwargs):
@@ -133,8 +161,25 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
if headers is None:
headers = {}
headers["User-Agent"] = self._useragent
+
try:
- self._http_con = StreamingHTTPSConnection(
+ if self.setup_fields["https_proxy"] is not None:
+ session = HTTPSession()
+ session.proxies = {
+ "https": self.setup_fields["https_proxy"]
+ }
+ response = session.request(method=method,
+ url=self.setup_fields["url_base_path"] + path,
+ headers=headers, data=body, stream=True)
+ return _WrappedHTTPResponse2(response)
+ except HTTPConnectionError as conn_err:
+ raise CaosDBConnectionError(
+ "Connection failed. Network or server down? " + str(conn_err)
+ )
+
+
+ try:
+ self._http_con = ProxyConnection(
# TODO looks as if configure needs to be done first.
# That is however not assured.
host=self.setup_fields["host"],
@@ -213,14 +258,24 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
"file.")
socket_proxy = None
-
if "socket_proxy" in config:
socket_proxy = config["socket_proxy"]
+ https_proxy = None
+ if "https_proxy" in config:
+ result = urlparse(config["https_proxy"], scheme="http")
+ if result.scheme != "http":
+ raise ValueError(
+ "The `https_proxy` parameter or config option must "
+ "contain a valid http uri or None")
+ https_proxy = result.scheme + "://" + result.netloc
+
self.setup_fields = {
+ "url_base_path": config["url"] + "/",
"host": host,
"timeout": int(config.get("timeout")),
"context": context,
+ "https_proxy": https_proxy,
"socket_proxy": socket_proxy}
@@ -342,6 +397,11 @@ def configure_connection(**kwargs):
An authentication token which has been issued by the CaosDB Server.
Implies `password_method="auth_token"` if set. An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
+ https_proxy : str
+ Define a https proxy, e.g. `https://localhost:8888`. Currently,
+ authentication against the proxy and non-TLS connections are not
+ supported. (Default: None)
+
implementation : CaosDBServerConnection
The class which implements the connection. (Default:
_DefaultCaosDBServerConnection)
diff --git a/src/caosdb/connection/streaminghttp.py b/src/caosdb/connection/streaminghttp.py
index 01774301..e3442795 100644
--- a/src/caosdb/connection/streaminghttp.py
+++ b/src/caosdb/connection/streaminghttp.py
@@ -70,13 +70,22 @@ class StreamingHTTPSConnection(client.HTTPSConnection, object):
that overrides the `send()` method to support iterable body objects."""
# pylint: disable=unused-argument, arguments-differ
- def __init__(self, socket_proxy=None, **kwargs):
+ def __init__(self, socket_proxy=None, https_proxy=None, **kwargs):
+ host = kwargs["host"]
+ port = int(kwargs["port"]) if "port" in kwargs else None
if socket_proxy is not None:
- host, port = socket_proxy.split(":")
- socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
- int(port))
+ proxy_host, proxy_port = socket_proxy.split(":")
+ socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, proxy_host,
+ int(proxy_port))
socket.socket = socks.socksocket
+ if https_proxy is not None:
+ tunnel_host = host
+ tunnel_port = port
+ host, port = https_proxy.split(":")
+ port = int(port)
super(StreamingHTTPSConnection, self).__init__(**kwargs)
+ if tunnel_host is not None:
+ self.set_tunnel(host=host, port=port)
def _send_output(self, body, **kwargs):
"""Send the currently buffered request and clear the buffer.
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
index a81bf006..bd795e8a 100644
--- a/src/caosdb/schema-pycaosdb-ini.yml
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -55,6 +55,10 @@ schema-pycaosdb-ini:
examples: ["localhost:12345"]
type: string
description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ https_proxy:
+ examples: ["https://localhost:8888"]
+ type: string
+ description: Define a HTTPS Proxy. Currently, authentication against the proxy and non-TLS HTTP connections are not supported.
implementation:
description: This option is used internally and for testing. Do not override.
examples: [_DefaultCaosDBServerConnection]
diff --git a/tox.ini b/tox.ini
index e3218918..50c22d57 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,6 @@ deps = .
nose
pytest
pytest-cov
- python-dateutil
jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
--
GitLab