Skip to content

Stealth browser automation for AI agents over MCP and the Chrome DevTools Protocol: navigation, network hooks, DOM extraction, and pixel-accurate UI cloning.

16.0 KB
Raw Blame
"""Authenticated proxy forwarding for browser launch proxies."""

from __future__ import annotations

import asyncio
import base64
import socket
import ssl
from ssl import SSLContext
from struct import calcsize, error as struct_error, pack, unpack
from typing import Optional
from urllib.parse import unquote, urlparse

from .debug_logger import debug_logger

DROPPED_HEADERS = ("proxy-authorization:",)
HOP_BY_HOP_HEADERS = ("connection:", "proxy-connection:", "keep-alive:")


def _free_port() -> int:
    """Return a free loopback TCP port for the local proxy listener."""
    free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    free_socket.bind(("127.0.0.1", 0))
    free_socket.listen(5)
    port = free_socket.getsockname()[1]
    free_socket.close()
    return port


class AuthenticatedProxyForwarder:
    """Forward local unauthenticated proxy traffic to an authenticated upstream proxy."""

    def __init__(
        self,
        proxy_server: str,
        ssl_context: Optional[SSLContext] = None,
    ) -> None:
        """
        Initialize an authenticated proxy forwarder.

        Args:
            proxy_server (str): Upstream proxy URL with credentials.
            ssl_context (Optional[SSLContext]): SSL context for HTTPS upstream proxies.
        """
        raw_value = proxy_server.strip()
        normalized_value = (
            raw_value if "://" in raw_value else f"http://{raw_value}"
        )
        parsed = urlparse(normalized_value)

        if not parsed.scheme:
            raise ValueError("Proxy URL is missing a scheme")
        if not parsed.hostname:
            raise ValueError("Proxy URL is missing a hostname")
        if parsed.port is None:
            raise ValueError("Proxy URL is missing a port")
        if parsed.username is None or parsed.password is None:
            raise ValueError("Proxy URL must include both username and password")

        self.server: Optional[asyncio.AbstractServer] = None
        self.ssl_context = ssl_context
        self.scheme = parsed.scheme
        self.use_ssl = parsed.scheme == "https"
        self.username = unquote(parsed.username)
        self.password = unquote(parsed.password)
        self.fw_host = parsed.hostname
        self.fw_port = parsed.port
        self.host = "127.0.0.1"
        self.port = _free_port()

        if self.scheme.startswith("http"):
            self._proxy_server = f"http://{self.host}:{self.port}"
        else:
            self._proxy_server = f"{self.scheme}://{self.host}:{self.port}"

    @property
    def proxy_server(self) -> str:
        """Return the local proxy address exposed to the browser."""
        return self._proxy_server

    async def start(self) -> None:
        """Start the local authenticated proxy forwarder."""
        if self.server is not None:
            return

        self.server = await asyncio.start_server(
            self.handle_request,
            host=self.host,
            port=self.port,
        )
        await self.server.start_serving()
        debug_logger.log_info(
            "proxy_forwarder",
            "start",
            f"Started {self.scheme} forwarder on {self.proxy_server}",
            {"upstream_host": self.fw_host, "upstream_port": self.fw_port},
        )

    async def close(self) -> None:
        """Stop the local authenticated proxy forwarder."""
        if self.server is None:
            return

        self.server.close()
        await self.server.wait_closed()
        self.server = None
        debug_logger.log_info(
            "proxy_forwarder",
            "close",
            f"Stopped forwarder on {self.proxy_server}",
        )

    async def handle_request(
        self,
        reader: asyncio.StreamReader,
        writer: asyncio.StreamWriter,
    ) -> None:
        """
        Route an incoming local proxy connection to the scheme-specific handler.

        Args:
            reader (asyncio.StreamReader): Local client reader.
            writer (asyncio.StreamWriter): Local client writer.
        """
        try:
            if self.scheme.startswith("socks"):
                await self._handle_socks_request(reader, writer)
                return

            if self.scheme.startswith("http"):
                await self._handle_http_request(reader, writer)
                return

            raise ValueError(f"Unsupported proxy scheme: {self.scheme}")
        except Exception as error:
            debug_logger.log_error(
                "proxy_forwarder",
                "handle_request",
                error,
                {"scheme": self.scheme},
            )
            await self._close_writer(writer)

    async def _handle_http_request(
        self,
        reader: asyncio.StreamReader,
        writer: asyncio.StreamWriter,
    ) -> None:
        """
        Forward one proxied request through an authenticated upstream HTTP proxy.

        CONNECT requests become a tunnel. Plain HTTP requests are sent with
        Connection: close, because only the first request on a connection
        gets the Proxy-Authorization header and later keep-alive requests
        would reach the upstream proxy without credentials.

        Args:
            reader (asyncio.StreamReader): Local client reader.
            writer (asyncio.StreamWriter): Local client writer.
        """
        max_line_length = 8192
        request_timeout = 5.0
        upstream_connect_timeout = 30.0
        remote_writer: Optional[asyncio.StreamWriter] = None
        pipe_tasks: list[asyncio.Task] = []
        header_lines: list[bytes] = []

        try:
            request_line = await asyncio.wait_for(
                reader.readline(),
                timeout=request_timeout,
            )
            if not request_line:
                await self._close_writer(writer)
                return
            if len(request_line) > max_line_length:
                await self._write_and_close(
                    writer,
                    b"HTTP/1.1 431 Request Header Fields Too Large\r\n\r\n",
                )
                return

            request_line_text = request_line.decode("utf-8", errors="ignore")
            parts = request_line_text.split()
            if len(parts) < 3:
                await self._write_and_close(writer, b"HTTP/1.1 400 Bad Request\r\n\r\n")
                return

            method = parts[0].upper()
            target_host_port = parts[1]
            if method == "CONNECT":
                if ":" not in target_host_port:
                    await self._write_and_close(
                        writer,
                        b"HTTP/1.1 400 Bad Request\r\n\r\n",
                    )
                    return

                host, port_text = target_host_port.rsplit(":", 1)
                try:
                    port = int(port_text)
                except ValueError as error:
                    raise ValueError(f"Invalid target port: {target_host_port}") from error
                if not host or port < 1 or port > 65535:
                    raise ValueError(f"Invalid target endpoint: {target_host_port}")

            while True:
                header = await asyncio.wait_for(
                    reader.readline(),
                    timeout=request_timeout,
                )
                if len(header) > max_line_length:
                    await self._write_and_close(
                        writer,
                        b"HTTP/1.1 431 Request Header Fields Too Large\r\n\r\n",
                    )
                    return
                if not header or header in {b"\r\n", b"\n"}:
                    break
                header_lines.append(header)

            connection_args = {"host": self.fw_host, "port": self.fw_port}
            if self.use_ssl:
                connection_args["ssl"] = self.ssl_context or ssl.create_default_context()

            remote_reader, remote_writer = await asyncio.wait_for(
                asyncio.open_connection(**connection_args),
                timeout=upstream_connect_timeout,
            )

            credentials = f"{self.username}:{self.password}"
            encoded_credentials = base64.b64encode(credentials.encode()).decode("ascii")
            dropped_headers = DROPPED_HEADERS if method == "CONNECT" else DROPPED_HEADERS + HOP_BY_HOP_HEADERS
            remote_writer.write(request_line)
            for header in header_lines:
                header_text = header.decode("utf-8", errors="ignore").lower()
                if header_text.startswith(dropped_headers):
                    continue
                remote_writer.write(header)
            remote_writer.write(
                f"Proxy-Authorization: Basic {encoded_credentials}\r\n".encode()
            )
            if method == "CONNECT":
                remote_writer.write(b"Proxy-Connection: Keep-Alive\r\n")
            else:
                remote_writer.write(b"Connection: close\r\nProxy-Connection: close\r\n")
            remote_writer.write(b"\r\n")
            await remote_writer.drain()

            if method == "CONNECT":
                response_line = await asyncio.wait_for(
                    remote_reader.readline(),
                    timeout=request_timeout,
                )
                if not response_line:
                    await self._write_and_close(
                        writer,
                        b"HTTP/1.1 502 Bad Gateway\r\n\r\n",
                    )
                    return

                while True:
                    header = await asyncio.wait_for(
                        remote_reader.readline(),
                        timeout=request_timeout,
                    )
                    if not header or header in {b"\r\n", b"\n"}:
                        break

                response_parts = response_line.decode("utf-8", errors="ignore").split()
                if len(response_parts) < 2 or response_parts[1] != "200":
                    await self._write_and_close(
                        writer,
                        b"HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\n\r\n"
                        b"Upstream proxy rejected the connection\r\n",
                    )
                    return

                writer.write(b"HTTP/1.1 200 Connection Established\r\n\r\n")
                await writer.drain()

            event = asyncio.Event()
            pipe_tasks = [
                asyncio.create_task(self.pipe(remote_reader, writer, event)),
                asyncio.create_task(self.pipe(reader, remote_writer, event)),
            ]
            await asyncio.gather(*pipe_tasks)
        except asyncio.TimeoutError:
            await self._write_and_close(writer, b"HTTP/1.1 504 Gateway Timeout\r\n\r\n")
        except Exception:
            raise
        finally:
            for task in pipe_tasks:
                if not task.done():
                    task.cancel()
            if pipe_tasks:
                await asyncio.gather(*pipe_tasks, return_exceptions=True)
            if remote_writer is not None:
                await self._close_writer(remote_writer)
            await self._close_writer(writer)

    async def _handle_socks_request(
        self,
        reader: asyncio.StreamReader,
        writer: asyncio.StreamWriter,
    ) -> None:
        """
        Forward a SOCKS5 tunnel through an authenticated upstream SOCKS proxy.

        Args:
            reader (asyncio.StreamReader): Local client reader.
            writer (asyncio.StreamWriter): Local client writer.
        """
        atyp_ipv4 = 0x01
        atyp_dns = 0x03
        atyp_ipv6 = 0x04
        remote_writer: Optional[asyncio.StreamWriter] = None

        async def read_struct(
            stream: asyncio.StreamReader,
            fmt: str,
        ) -> tuple:
            """Read and unpack an exact struct payload from a stream."""
            size = calcsize(fmt)
            data = await stream.readexactly(size)
            try:
                return unpack(fmt, data)
            except struct_error as error:
                raise ValueError(f"Invalid SOCKS packet for format {fmt}") from error

        try:
            version, num_methods = await read_struct(reader, "!BB")
            await reader.readexactly(num_methods)
            writer.write(pack("!BB", version, 0))
            await writer.drain()

            version, cmd, reserved, atyp = await read_struct(reader, "!BBBB")

            if atyp == atyp_ipv4:
                address_payload = await reader.readexactly(4)
            elif atyp == atyp_ipv6:
                address_payload = await reader.readexactly(16)
            elif atyp == atyp_dns:
                hostname_length = (await read_struct(reader, "!B"))[0]
                hostname = await reader.readexactly(hostname_length)
                address_payload = pack("!B", hostname_length) + hostname
            else:
                raise ValueError(f"Unsupported SOCKS address type: {atyp}")

            port_payload = await reader.readexactly(2)

            remote_reader, remote_writer = await asyncio.open_connection(
                host=self.fw_host,
                port=self.fw_port,
            )

            remote_writer.write(pack("!BBB", version, 1, 2))
            await remote_writer.drain()
            _, auth_method = await read_struct(remote_reader, "!BB")

            if auth_method == 2:
                auth_ticket = pack(
                    f"!BB{len(self.username)}sB{len(self.password)}s",
                    1,
                    len(self.username),
                    self.username.encode(),
                    len(self.password),
                    self.password.encode(),
                )
                remote_writer.write(auth_ticket)
                await remote_writer.drain()
                _, auth_result = await read_struct(remote_reader, "!BB")
                if auth_result != 0:
                    raise ValueError(
                        f"SOCKS upstream authentication failed: {auth_result}"
                    )

            remote_writer.write(pack("!BBBB", version, cmd, reserved, atyp))
            remote_writer.write(address_payload)
            remote_writer.write(port_payload)
            await remote_writer.drain()

            event = asyncio.Event()
            await asyncio.gather(
                self.pipe(remote_reader, writer, event),
                self.pipe(reader, remote_writer, event),
            )
        finally:
            if remote_writer is not None:
                await self._close_writer(remote_writer)
            await self._close_writer(writer)

    @staticmethod
    async def pipe(
        reader: asyncio.StreamReader,
        writer: asyncio.StreamWriter,
        event: asyncio.Event,
    ) -> None:
        """
        Relay bytes between two streams until either side closes.

        Args:
            reader (asyncio.StreamReader): Source stream.
            writer (asyncio.StreamWriter): Destination stream.
            event (asyncio.Event): Shared completion signal.
        """
        while not event.is_set():
            try:
                data = await asyncio.wait_for(reader.read(2**16), 1)
                if not data:
                    break
                writer.write(data)
                await writer.drain()
            except asyncio.TimeoutError:
                continue
            except Exception:
                break
        event.set()

    @staticmethod
    async def _write_and_close(
        writer: asyncio.StreamWriter,
        payload: bytes,
    ) -> None:
        """
        Write a terminal response and close a stream.

        Args:
            writer (asyncio.StreamWriter): Stream writer to close.
            payload (bytes): Response payload to write before closing.
        """
        writer.write(payload)
        await writer.drain()
        await AuthenticatedProxyForwarder._close_writer(writer)

    @staticmethod
    async def _close_writer(writer: asyncio.StreamWriter) -> None:
        """
        Close a stream writer if it is still open.

        Args:
            writer (asyncio.StreamWriter): Stream writer to close.
        """
        if writer.is_closing():
            return
        writer.close()
        try:
            await writer.wait_closed()
        except Exception:
            pass