Migrate http proxy tests to pproxy (#199)

* Replaced mitmproxy by pproxy for proxy tests

* Rebased onto master branch HEAD.

Removed unused fixtures.

Co-authored-by: Florimond Manca <florimond.manca@gmail.com>

* Apply suggestions from code review

Co-authored-by: Florimond Manca <florimond.manca@gmail.com>

* Apply suggestions from code review

Co-authored-by: Florimond Manca <florimond.manca@gmail.com>

Co-authored-by: Florimond Manca <florimond.manca@gmail.com>

Author: cdeler

requirements.txt

@@ -22,10 +22,9 @@ flake8==3.8.4
 flake8-bugbear==20.1.4
 flake8-pie==0.6.1
 isort==5.5.4
-mitmproxy==5.2
 mypy==0.782
+pproxy==2.3.5
 pytest==6.1.0
 pytest-trio==0.6.0
 pytest-cov==2.10.1
-trustme==0.6.0
 uvicorn==0.12.1

tests/async_tests/test_interfaces.py

@@ -1,5 +1,4 @@
 import platform
-import ssl
 
 import pytest
 
@@ -217,7 +216,6 @@ async def test_http_request_local_address(backend: str, server: Server) -> None:
 @pytest.mark.anyio
 async def test_proxy_https_requests(
     proxy_server: URL,
-    ca_ssl_context: ssl.SSLContext,
     proxy_mode: str,
     http2: bool,
     https_server: Server,
@@ -229,7 +227,6 @@ async def test_proxy_https_requests(
     async with httpcore.AsyncHTTPProxy(
         proxy_server,
         proxy_mode=proxy_mode,
-        ssl_context=ca_ssl_context,
         max_connections=max_connections,
         http2=http2,
     ) as http:

tests/conftest.py

@@ -1,109 +1,27 @@
-import asyncio
 import contextlib
 import os
-import ssl
 import threading
 import time
 import typing
 
 import pytest
-import trustme
 import uvicorn
-from mitmproxy import options, proxy
-from mitmproxy.tools.dump import DumpMaster
 
 from httpcore._types import URL
 
-from .utils import Server
+from .utils import Server, http_proxy_server
 
-PROXY_HOST = "127.0.0.1"
-PROXY_PORT = 8080
 SERVER_HOST = "example.org"
 HTTPS_SERVER_URL = "https://example.org"
 
 
-class RunNotify:
-    """A mitmproxy addon wrapping an event to notify us when the server is running."""
-
-    def __init__(self) -> None:
-        self.started = threading.Event()
-
-    def running(self) -> None:
-        self.started.set()
-
-
-class ProxyWrapper(threading.Thread):
-    """Runs an mitmproxy in a separate thread."""
-
-    def __init__(self, host: str, port: int, **kwargs: typing.Any) -> None:
-        self.host = host
-        self.port = port
-        self.options = kwargs
-        super().__init__()
-        self.notify = RunNotify()
-
-    def run(self) -> None:
-        # mitmproxy uses asyncio internally but the default loop policy
-        # will only create event loops for the main thread, create one
-        # as part of the thread startup
-        asyncio.set_event_loop(asyncio.new_event_loop())
-        opts = options.Options(
-            listen_host=self.host, listen_port=self.port, **self.options
-        )
-        pconf = proxy.config.ProxyConfig(opts)
-
-        self.master = DumpMaster(opts)
-        self.master.server = proxy.server.ProxyServer(pconf)
-        self.master.addons.add(self.notify)
-        self.master.run()
-
-    def join(self, timeout: float = None) -> None:
-        self.master.shutdown()
-        super().join()
-
-
-@pytest.fixture(scope="session")
-def cert_authority() -> trustme.CA:
-    return trustme.CA()
-
-
-@pytest.fixture()
-def ca_ssl_context(cert_authority: trustme.CA) -> ssl.SSLContext:
-    ctx = ssl.create_default_context()
-    cert_authority.configure_trust(ctx)
-    return ctx
-
-
-@pytest.fixture(scope="session")
-def example_org_cert(cert_authority: trustme.CA) -> trustme.LeafCert:
-    return cert_authority.issue_cert("example.org")
-
-
 @pytest.fixture(scope="session")
-def example_org_cert_path(example_org_cert: trustme.LeafCert) -> typing.Iterator[str]:
-    with example_org_cert.private_key_and_cert_chain_pem.tempfile() as tmp:
-        yield tmp
+def proxy_server() -> typing.Iterator[URL]:
+    proxy_host = "127.0.0.1"
+    proxy_port = 8080
 
-
-@pytest.fixture()
-def proxy_server(example_org_cert_path: str) -> typing.Iterator[URL]:
-    """Starts a proxy server on a different thread and yields its origin tuple.
-
-    The server is configured to use a trustme CA and key, this will allow our
-    test client to make HTTPS requests when using the ca_ssl_context fixture
-    above.
-
-    Note this is only required because mitmproxy's main purpose is to analyse
-    traffic. Other proxy servers do not need this but mitmproxy is easier to
-    integrate in our tests.
-    """
-    try:
-        thread = ProxyWrapper(PROXY_HOST, PROXY_PORT, certs=[example_org_cert_path])
-        thread.start()
-        thread.notify.started.wait()
-        yield (b"http", PROXY_HOST.encode(), PROXY_PORT, b"/")
-    finally:
-        thread.join()
+    with http_proxy_server(proxy_host, proxy_port) as proxy_url:
+        yield proxy_url
 
 
 class UvicornServer(uvicorn.Server):

tests/sync_tests/test_interfaces.py

@@ -1,5 +1,4 @@
 import platform
-import ssl
 
 import pytest
 
@@ -217,7 +216,6 @@ def test_http_request_local_address(backend: str, server: Server) -> None:
 
 def test_proxy_https_requests(
     proxy_server: URL,
-    ca_ssl_context: ssl.SSLContext,
     proxy_mode: str,
     http2: bool,
     https_server: Server,
@@ -229,7 +227,6 @@ def test_proxy_https_requests(
     with httpcore.SyncHTTPProxy(
         proxy_server,
         proxy_mode=proxy_mode,
-        ssl_context=ca_ssl_context,
         max_connections=max_connections,
         http2=http2,
     ) as http:

tests/utils.py

@@ -1,3 +1,7 @@
+import contextlib
+import socket
+import subprocess
+import time
 from typing import Tuple
 
 import sniffio
@@ -11,6 +15,17 @@ def lookup_sync_backend():
     return "sync"
 
 
+def _wait_can_connect(host: str, port: int):
+    while True:
+        try:
+            sock = socket.create_connection((host, port))
+        except ConnectionRefusedError:
+            time.sleep(0.25)
+        else:
+            sock.close()
+            break
+
+
 class Server:
     """
     Represents the server we're testing against.
@@ -27,3 +42,19 @@ def netloc(self) -> Tuple[bytes, int]:
     @property
     def host_header(self) -> Tuple[bytes, bytes]:
         return (b"host", self._host.encode("utf-8"))
+
+
+@contextlib.contextmanager
+def http_proxy_server(proxy_host: str, proxy_port: int):
+
+    proc = None
+    try:
+        command = ["pproxy", "-l", f"http://{proxy_host}:{proxy_port}/"]
+        proc = subprocess.Popen(command)
+
+        _wait_can_connect(proxy_host, proxy_port)
+
+        yield b"http", proxy_host.encode(), proxy_port, b"/"
+    finally:
+        if proc is not None:
+            proc.kill()