Skip to content

Force Proxy to any Windows/MacOS/Linux Application TCP and UDP traffic

20.5 KB
Raw Blame
#!/usr/bin/env python3
"""Client side of the full end-to-end suite (tests/e2e_full.sh). The client never knows whether ProxyBridge is running: the very
same scenarios run with the engine off and on, and the suite compares the results.

usage: full_client.py <scenario> [<scenario> ...]
  tcp     http get from two origins, 200 parallel and 400 sequential connections, request latency, a big download and a big upload
          with checksums
  udp     one socket to several destinations, connected socket, datagram sizes 1..59904 bytes, bound source port, 300 concurrent
          sockets, a 3000 datagrams/s stream (loss), round-trip time
  dns     real DNS packets (A query, NXDOMAIN, 200 parallel queries, queries over TCP, queries per second)
  quic    QUIC long-header packets (RFC 9000 Version Negotiation): the datagram shape of HTTP/3, 1200 bytes, connection ids mirrored
  load    500 parallel TCP connections, 1000 concurrent UDP sockets (LOAD_UDP), 20000 small UDP datagrams back to back
  churn   what a resolver does: a new UDP socket per lookup, CHURN_N (700) times in a row
  newflow latency of the first datagram of brand-new UDP flows (echo and DNS port)
  udpblock  one datagram that a BLOCK rule must stop: no reply may arrive

Output: "RESULT ok|BAD <name> | <detail>", "METRIC <name> <value>", "INFO <text>". Environment: DL_MB (default 100), UL_MB (50),
STREAM_N (6000), the origin addresses are fixed (see tests/nexus_origins.py and tests/full_quic_server.py).
"""
import concurrent.futures as cf
import hashlib, os, random, socket, struct, sys, threading, time

O1, O2, DNSIP = "10.77.1.1", "10.77.1.2", "10.77.1.53"
BLOCK = bytes((i * 7 + 13) & 255 for i in range(1 << 20))
DL_MB = int(os.environ.get("DL_MB", "100"))
UL_MB = int(os.environ.get("UL_MB", "50"))
STREAM_N = int(os.environ.get("STREAM_N", "6000"))
# Every proxied UDP socket holds one TCP control connection to the proxy until its session expires. Stock Nexus Proxy resets
# connections beyond 500 per source address (tests/e2e_full.sh raises that in its private copy); ProxyBridge itself allows 2048
# UDP sessions per process.
LOAD_UDP = int(os.environ.get("LOAD_UDP", "1000"))


def res(name, ok, detail=""):
    print("RESULT", "ok" if ok else "BAD", name, "|", detail, flush=True)      # the suite compares names, not details


def metric(name, value):
    print("METRIC", name, value, flush=True)


def info(text):
    print("INFO", text, flush=True)


def pattern_sha(mb):
    h = hashlib.sha256()
    for _ in range(mb):
        h.update(BLOCK)
    return h.hexdigest()


def pct(values, p):
    values = sorted(values)
    return values[min(len(values) - 1, int(len(values) * p))]


# ------------------------------------------------------------------ TCP
def http_get(ip, port, path="/", timeout=15):
    s = socket.create_connection((ip, port), timeout=timeout)
    try:
        s.sendall(("GET %s HTTP/1.0\r\nHost: x\r\n\r\n" % path).encode())
        data = b""
        while True:
            d = s.recv(65536)
            if not d:
                break
            data += d
        return data.split(b"\r\n\r\n", 1)[-1]
    finally:
        s.close()


def tcp():
    res("tcp: origin 1 answers", http_get(O1, 18700) == b"origin-1\n")
    res("tcp: origin 2 answers", http_get(O2, 18700) == b"origin-2\n")

    def one(_):
        try:
            return http_get(O1, 18700) == b"origin-1\n"
        except OSError:
            return False
    with cf.ThreadPoolExecutor(64) as ex:
        okc = sum(ex.map(one, range(200)))
    res("tcp: 200 parallel connections", okc == 200, "%d/200" % okc)
    t0 = time.perf_counter()
    okc = sum(one(i) for i in range(400))
    res("tcp: 400 sequential connect/request/close cycles", okc == 400, "%d/400" % okc)
    metric("tcp_cycles_per_s", round(400 / (time.perf_counter() - t0)))
    lat = []
    for _ in range(200):
        t = time.perf_counter()
        http_get(O1, 18700)
        lat.append((time.perf_counter() - t) * 1000)
    metric("tcp_request_p50_ms", round(pct(lat, .5), 2))
    metric("tcp_request_p99_ms", round(pct(lat, .99), 2))

    # big download with a checksum of the received stream
    s = socket.create_connection((O1, 18700), timeout=60)
    s.sendall(("GET /big?mb=%d HTTP/1.0\r\n\r\n" % DL_MB).encode())
    t0 = time.perf_counter()
    buf = b""
    while b"\r\n\r\n" not in buf:
        buf += s.recv(65536)
    head, _, rest = buf.partition(b"\r\n\r\n")
    h = hashlib.sha256(rest)
    got = len(rest)
    while True:
        d = s.recv(1 << 20)
        if not d:
            break
        h.update(d)
        got += len(d)
    dt = time.perf_counter() - t0
    s.close()
    res("tcp: %d MB download, checksum identical" % DL_MB, got == DL_MB << 20 and h.hexdigest() == pattern_sha(DL_MB), "%d bytes" % got)
    metric("tcp_download_MBps", round(DL_MB / dt))

    # big upload; the origin answers with the sha256 of what it received
    s = socket.create_connection((O2, 18700), timeout=60)
    s.sendall(("POST /upload HTTP/1.0\r\nContent-Length: %d\r\n\r\n" % (UL_MB << 20)).encode())
    t0 = time.perf_counter()
    for _ in range(UL_MB):
        s.sendall(BLOCK)
    dt = time.perf_counter() - t0
    ans = b""
    while True:
        d = s.recv(4096)
        if not d:
            break
        ans += d
    s.close()
    res("tcp: %d MB upload, origin received identical bytes" % UL_MB, ans.split(b"\r\n\r\n", 1)[-1].decode() == pattern_sha(UL_MB))
    metric("tcp_upload_MBps", round(UL_MB / dt))


# ------------------------------------------------------------------ UDP
def usock(t=4.0):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.settimeout(t)
    return s


def udp():
    s = usock()
    for dst, tag in (((O1, 18710), b"U1:"), ((O2, 18710), b"U2:"), ((O1, 18710), b"U1:")):
        try:
            s.sendto(b"hello", dst)
            d, a = s.recvfrom(100)
            res("udp: one socket, destination %s:%d" % dst, d == tag + b"hello" and a == dst, "from %s:%d" % a)
        except OSError as e:
            res("udp: one socket, destination %s:%d" % dst, False, str(e))
    c = usock()
    c.connect((O1, 18711))
    try:
        c.send(b"conn")
        d = c.recv(100)
        res("udp: connected socket, peer address restored", d == b"U3:conn" and c.getpeername() == (O1, 18711), "peer %s:%d" % c.getpeername())
    except OSError as e:
        res("udp: connected socket, peer address restored", False, str(e))
    for n in (1, 100, 1200, 1472, 8000, 30000, 59904):
        b = usock(8)
        payload = bytes((i * 31 + n) & 255 for i in range(n))
        try:
            b.sendto(payload, (O1, 18710))
            d, a = b.recvfrom(70000)
            res("udp: datagram of %d bytes" % n, d == b"U1:" + payload)
        except OSError as e:
            res("udp: datagram of %d bytes" % n, False, str(e))
    port = random.randint(41000, 49000)
    b = usock()
    b.bind(("0.0.0.0", port))
    try:
        b.sendto(b"bound", (O1, 18710))
        d, a = b.recvfrom(100)
        res("udp: socket bound to a chosen source port gets the reply", d == b"U1:bound" and b.getsockname()[1] == port, "port %d" % port)
    except OSError as e:
        res("udp: socket bound to a chosen source port", False, str(e))

    def one(i):
        k = usock(8)
        dst = (("10.77.1.%d" % (1 + i % 2)), 18710)
        try:
            k.sendto(b"n%d" % i, dst)
            d, a = k.recvfrom(100)
            return d.endswith(b"n%d" % i) and a == dst
        except OSError:
            return False
        finally:
            k.close()
    with cf.ThreadPoolExecutor(32) as ex:
        okc = sum(ex.map(one, range(300)))
    res("udp: 300 concurrent sockets across two destinations (99% or better)", okc >= 297, "%d/300" % okc)   # UDP is lossy by nature

    # sustained stream, replies counted while still sending
    q = usock(1)
    q.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4 << 20)
    got, stop = set(), threading.Event()

    def rx():
        q.settimeout(0.2)
        while not stop.is_set():
            try:
                d, _ = q.recvfrom(2000)
                got.add(int.from_bytes(d[3:7], "big"))
            except socket.timeout:
                pass
    th = threading.Thread(target=rx, daemon=True)
    th.start()
    t0 = time.perf_counter()
    for i in range(STREAM_N):
        q.sendto(i.to_bytes(4, "big") + b"x" * 996, (O1, 18710))
        while time.perf_counter() < t0 + (i + 1) / 3000.0:
            time.sleep(0)
    sent_for = time.perf_counter() - t0
    time.sleep(2)
    stop.set()
    th.join()
    loss = 100.0 * (STREAM_N - len(got)) / STREAM_N
    res("udp: stream of %d datagrams at 3000/s, loss <= 1%%" % STREAM_N, loss <= 1.0, "%.2f%% lost" % loss)
    metric("udp_stream_loss_pct", round(loss, 2))
    metric("udp_stream_send_pps", round(STREAM_N / sent_for))

    # fire and forget: one datagram per socket, the socket is closed at once. The engine looks at the sender only after the datagram
    # arrived, by then the socket is gone: the datagram must still be delivered.
    tag = os.urandom(4).hex()
    for i in range(50):
        k = usock()
        k.sendto(("FF:%s:%d" % (tag, i)).encode(), (O1, 4434))
        k.close()
    time.sleep(1.5)
    q2 = usock(3)
    try:
        q2.sendto(("COUNT:%s" % tag).encode(), (O1, 4434))
        got_ff = int(q2.recvfrom(100)[0])
    except (OSError, ValueError):
        got_ff = -1
    res("udp: 50 fire-and-forget datagrams (socket closed right after sending) all arrive", got_ff == 50, "%d/50" % got_ff)

    rtt = []
    r = usock(2)
    for _ in range(1000):
        t = time.perf_counter()
        r.sendto(b"p", (O1, 18710))
        r.recvfrom(100)
        rtt.append((time.perf_counter() - t) * 1e6)
    metric("udp_rtt_p50_us", round(pct(rtt, .5)))
    metric("udp_rtt_p99_us", round(pct(rtt, .99)))
    res("udp: 1000 round trips", True)


# ------------------------------------------------------------------ DNS
def dns_query(name, qtype=1, txid=None):
    txid = random.randint(0, 65535) if txid is None else txid
    q = struct.pack("!HHHHHH", txid, 0x0100, 1, 0, 0, 0)
    for label in name.split("."):
        q += bytes([len(label)]) + label.encode()
    return txid, q + b"\0" + struct.pack("!HH", qtype, 1)


def skip_name(m, off):
    while True:
        l = m[off]
        if l == 0:
            return off + 1
        if l & 0xC0 == 0xC0:
            return off + 2
        off += 1 + l


def dns_answer(m, txid):
    """(rcode, [ipv4 strings]) of a response, or None when it is not the answer to txid."""
    if len(m) < 12 or struct.unpack("!H", m[:2])[0] != txid:
        return None
    flags, qd, an = struct.unpack("!HHH", m[2:8])
    off = 12
    for _ in range(qd):
        off = skip_name(m, off) + 4
    ips = []
    for _ in range(an):
        off = skip_name(m, off)
        t, _, _, rl = struct.unpack("!HHIH", m[off:off + 10])
        off += 10
        if t == 1 and rl == 4:
            ips.append(socket.inet_ntoa(m[off:off + 4]))
        off += rl
    return flags & 0xF, ips


def dns_udp(name, timeout=4):
    txid, q = dns_query(name)
    s = usock(timeout)
    try:
        s.sendto(q, (DNSIP, 53))
        m, _ = s.recvfrom(4096)
        return dns_answer(m, txid)
    except OSError:
        return None
    finally:
        s.close()


def dns_tcp(name):
    txid, q = dns_query(name)
    s = socket.create_connection((DNSIP, 53), timeout=5)
    try:
        s.sendall(struct.pack("!H", len(q)) + q)
        n = struct.unpack("!H", s.recv(2))[0]
        m = b""
        while len(m) < n:
            m += s.recv(n - len(m))
        return dns_answer(m, txid)
    except OSError:
        return None
    finally:
        s.close()


def dns():
    r = dns_udp("app.example.test")
    res("dns: UDP A query app.example.test", r == (0, ["10.77.1.5"]), str(r))
    r = dns_udp("svc.corp.test")
    res("dns: UDP A query svc.corp.test", r == (0, ["10.77.1.6"]), str(r))
    r = dns_udp("does-not-exist.test")
    res("dns: UDP NXDOMAIN", r is not None and r[0] == 3, str(r))
    with cf.ThreadPoolExecutor(32) as ex:
        okc = sum(1 for x in ex.map(lambda i: dns_udp("app.example.test"), range(200)) if x == (0, ["10.77.1.5"]))
    res("dns: 200 parallel UDP queries (99% or better: a resolver retries)", okc >= 198, "%d/200" % okc)
    okc = sum(1 for _ in range(20) if dns_tcp("svc.corp.test") == (0, ["10.77.1.6"]))
    res("dns: 20 queries over TCP", okc == 20, "%d/20" % okc)
    lat, lost = [], 0
    for _ in range(500):
        t = time.perf_counter()
        if dns_udp("app.example.test", timeout=2) != (0, ["10.77.1.5"]):
            lost += 1
        lat.append((time.perf_counter() - t) * 1000)
    res("dns: 500 sequential UDP queries, none lost", lost == 0, "%d lost" % lost)
    metric("dns_udp_qps", round(500 / (sum(lat) / 1000)))
    metric("dns_udp_p99_ms", round(pct(lat, .99), 2))


# ------------------------------------------------------------------ QUIC-shaped UDP
def quic_initial(version=0x1a2a3a4a, size=1200):
    dcid, scid = os.urandom(8), os.urandom(8)
    pkt = bytes([0xC3]) + struct.pack("!I", version) + bytes([len(dcid)]) + dcid + bytes([len(scid)]) + scid + b"\x00"
    body = os.urandom(size - len(pkt) - 2)
    pkt += struct.pack("!H", 0x4000 | len(body)) + body
    return dcid, scid, pkt


def vn_ok(reply, dcid, scid):
    if len(reply) < 7 or not reply[0] & 0x80 or struct.unpack("!I", reply[1:5])[0] != 0:
        return False
    dl = reply[5]
    if reply[6:6 + dl] != scid:                      # the reply's DCID is the client's SCID
        return False
    sl = reply[6 + dl]
    if reply[7 + dl:7 + dl + sl] != dcid:            # ... and its SCID is the client's DCID
        return False
    tail = reply[7 + dl + sl:]
    return len(tail) >= 4 and len(tail) % 4 == 0 and struct.pack("!I", 1) in [tail[i:i + 4] for i in range(0, len(tail), 4)]


def quic_once(dst, timeout=5):
    dcid, scid, pkt = quic_initial()
    s = usock(timeout)
    try:
        s.sendto(pkt, dst)
        d, a = s.recvfrom(2000)
        return a == dst and vn_ok(d, dcid, scid)
    except OSError:
        return False
    finally:
        s.close()


def quic():
    res("quic: 1200-byte Initial with an unknown version -> Version Negotiation", quic_once((O1, 4433)))
    res("quic: same to a second server", quic_once((O2, 4433)))
    with cf.ThreadPoolExecutor(32) as ex:
        okc = sum(ex.map(lambda i: quic_once((("10.77.1.%d" % (1 + i % 2)), 4433)), range(100)))
    res("quic: 100 parallel handshakes (Version Negotiation each, 99% or better)", okc >= 99, "%d/100" % okc)
    s = usock()
    okc = 0
    for i in range(20):                                # one socket, many packets, both servers: 5-tuple stays stable
        dcid, scid, pkt = quic_initial()
        s.sendto(pkt, (("10.77.1.%d" % (1 + i % 2)), 4433))
        try:
            d, _ = s.recvfrom(2000)
            okc += vn_ok(d, dcid, scid)
        except OSError:
            pass
    res("quic: one socket, 20 packets to two servers", okc == 20, "%d/20" % okc)
    s.sendto(b"not quic at all", (O1, 4433))
    try:
        d, _ = s.recvfrom(100)
        res("quic: a non-QUIC datagram reaches the server unchanged", d == b"NOTQUIC" + b"not quic", d[:20].decode("latin1"))
    except OSError as e:
        res("quic: a non-QUIC datagram reaches the server unchanged", False, str(e))
    dcid, scid, pkt = quic_initial(version=1)
    s.sendto(pkt, (O2, 4433))
    try:
        d, _ = s.recvfrom(100)
        res("quic: a version 1 Initial reaches the server unchanged", d.startswith(b"NOTQUIC") and d[7:] == pkt[:8])
    except OSError as e:
        res("quic: a version 1 Initial reaches the server unchanged", False, str(e))


# ------------------------------------------------------------------ load
def load():
    def one(_):
        try:
            return http_get(O1, 18700, timeout=30) == b"origin-1\n"
        except OSError:
            return False
    t0 = time.perf_counter()
    with cf.ThreadPoolExecutor(128) as ex:
        okc = sum(ex.map(one, range(500)))
    res("load: 500 TCP connections, 128 at a time", okc == 500, "%d/500 in %.1fs" % (okc, time.perf_counter() - t0))
    socks = []
    got = 0
    try:
        for i in range(LOAD_UDP):
            k = usock(10)
            socks.append(k)
            k.sendto(b"u%d" % i, ((("10.77.1.%d" % (1 + i % 2)), 18710)))
        for i, k in enumerate(socks):
            try:
                d, _ = k.recvfrom(100)
                got += d.endswith(b"u%d" % i)
            except OSError:
                pass
    finally:
        for k in socks:
            k.close()
    res("load: %d UDP sockets open at once, each answered" % LOAD_UDP, got >= 0.99 * LOAD_UDP, "%d/%d" % (got, LOAD_UDP))
    # one socket sends small datagrams at a fixed rate; the answers are counted while it is still sending
    def burst(n, pps):
        q = usock(1)
        q.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20)
        seen, stop = set(), threading.Event()

        def rx():
            q.settimeout(0.2)
            while not stop.is_set():
                try:
                    d, _ = q.recvfrom(2000)
                    seen.add(int.from_bytes(d[3:7], "big"))
                except socket.timeout:
                    pass
        th = threading.Thread(target=rx, daemon=True)
        th.start()
        t0 = time.perf_counter()
        for i in range(n):
            q.sendto(i.to_bytes(4, "big") + b"y" * 60, (O1, 18710))
            while time.perf_counter() < t0 + (i + 1) / pps:
                time.sleep(0)
        time.sleep(2)
        stop.set()
        th.join()
        q.close()
        return 100.0 * (n - len(seen)) / n
    for pps in (3000, 6000, 12000):
        loss = burst(10000, pps)
        metric("udp_burst_%d_pps_loss_pct" % pps, round(loss, 2))
        if pps == 3000:
            res("load: 10000 small UDP datagrams at 3000/s, loss <= 2%", loss <= 2.0, "%.2f%% lost" % loss)
        else:
            info("%d datagrams/s: %.2f%% lost (reported, not a pass/fail line: a single-threaded Node proxy saturates)" % (pps, loss))


def newflow():
    """Latency of the FIRST datagram of a brand-new UDP flow (new socket each time): for a proxied flow this includes the SOCKS5
    UDP ASSOCIATE. Echo port and DNS port are measured separately because port 53 takes its own path in the engine."""
    for label, dst, payload in (("echo", (O1, 18710), b"x"), ("dns", (DNSIP, 53), None)):
        lat, fails = [], 0
        for _ in range(60):
            s = usock(3)
            try:
                if payload is None:
                    txid, q = dns_query("app.example.test")
                    msg = q
                else:
                    msg = payload
                t = time.perf_counter()
                s.sendto(msg, dst)
                s.recvfrom(2000)
                lat.append((time.perf_counter() - t) * 1000)
            except OSError:
                fails += 1
            finally:
                s.close()
        if lat:
            metric("newflow_%s_p50_ms" % label, round(pct(lat, .5), 2))
            metric("newflow_%s_p99_ms" % label, round(pct(lat, .99), 2))
        res("newflow: first datagram of 60 new %s flows answered" % label, fails == 0, "%d lost" % fails)


def churn():
    """What a resolver does: a new UDP socket per lookup, one query, close. Every proxied UDP socket keeps a session (and a TCP
    control connection to the proxy) until it has been idle for the UDP idle timeout (60 s by default), so this piles up sessions."""
    n = int(os.environ.get("CHURN_N", "700"))
    lost, first_lost, t0 = 0, None, time.perf_counter()
    for i in range(n):
        if dns_udp("app.example.test", timeout=2) != (0, ["10.77.1.5"]):
            lost += 1
            first_lost = i if first_lost is None else first_lost
    dt = time.perf_counter() - t0
    res("churn: %d lookups, each on a new UDP socket" % n, lost == 0, "%d lost%s" % (lost, "" if first_lost is None else ", first at #%d" % first_lost))
    metric("churn_lookups_per_s", round(n / dt))


def udpblock():
    s = usock(2.5)
    try:
        s.sendto(b"should-not-arrive", (O1, 18710))
        s.recvfrom(100)
        res("udp blocked: nothing may come back", False, "a reply arrived")
    except (socket.timeout, OSError):
        res("udp blocked: nothing may come back", True)


SCENARIOS = {"tcp": tcp, "udp": udp, "dns": dns, "quic": quic, "load": load, "udpblock": udpblock, "newflow": newflow, "churn": churn}

if __name__ == "__main__":
    names = sys.argv[1:] or ["tcp", "udp", "dns", "quic"]
    for n in names:
        if n not in SCENARIOS:
            print("unknown scenario", n, file=sys.stderr)
            sys.exit(2)
    for n in names:
        try:
            SCENARIOS[n]()
        except Exception as e:                               # a crashed scenario is a failure, not a silent skip
            res("%s: scenario crashed" % n, False, "%s: %s" % (type(e).__name__, e))