The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
"""Tests for the gateway's model-API path filter (gateway.py, api_paths.json): default deny, per provider.
python3 scripts/usage_gateway/tests/test_paths.py # stdlib only; the end-to-end part starts gateway.py on loopback
"""
from __future__ import annotations
import http.client
import json
import pathlib
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
HERE = pathlib.Path(__file__).resolve().parents[1]
sys.path.insert(0, str(HERE))
import gateway # noqa: E402
gateway.PROVIDERS[:] = gateway.load_paths()
def ok(host: str, method: str, path: str, plain: bool = False) -> bool:
return gateway.path_allowed(gateway.provider_of(host, plain), method, path)
class Rules(unittest.TestCase):
def test_model_calls_of_recorded_runs(self):
"""Every (host, method, path) the gateway recorded in official runs up to 2026-10-10."""
for host, method, path in [
("chatgpt.com", "POST", "/backend-api/codex/responses"),
("chatgpt.com", "GET", "/backend-api/codex/models?client_version=0.160.1"),
("api.openai.com", "POST", "/v1/responses"),
("res-1.cognitiveservices.azure.com", "POST", "/openai/v1/responses"),
("res-1.openai.azure.com", "POST", "/openai/v1/responses?api-version=preview"),
("res-1.services.ai.azure.com", "POST", "/openai/v1/responses"),
("api.anthropic.com", "POST", "/v1/messages?beta=true"),
("api.anthropic.com", "POST", "/v1/messages/count_tokens?beta=true")]:
self.assertTrue(ok(host, method, path), (host, method, path))
for path in ("/v1/messages", "/v1/messages/count_tokens", "/v1/chat/completions"):
self.assertTrue(ok("10.0.0.5:8080", "POST", path, plain=True), path)
def test_chat_completions_only_on_a_self_hosted_upstream(self):
"""At the providers that API also serves models that search without a tool declaration: not forwarded there."""
for host, path in (("api.openai.com", "/v1/chat/completions"), ("openrouter.ai", "/api/v1/chat/completions"),
("res-1.openai.azure.com", "/openai/deployments/gpt-6/chat/completions"),
("res-1.openai.azure.com", "/openai/v1/chat/completions")):
self.assertFalse(ok(host, "POST", path), (host, path))
self.assertTrue(ok("10.0.0.5:8080", "POST", "/v1/chat/completions", plain=True))
def test_a_star_is_one_segment(self):
with tempfile.TemporaryDirectory() as td:
f = pathlib.Path(td) / "p.json"
f.write_text(json.dumps({"client_tool_types": ["function", "custom", "namespace"], "provider_side_fields": ["mcp_servers"],
"providers": [{"name": "x", "hosts": ["h.example"], "allow": [["POST", "/d/*/chat"]]}]}))
saved = (list(gateway.PROVIDERS), set(gateway.CLIENT_TOOLS), set(gateway.PROVIDER_FIELDS))
try:
gateway.PROVIDERS[:] = gateway.load_paths(f)
self.assertTrue(ok("h.example", "POST", "/d/gpt-6/chat"))
for path in ("/d/a/b/chat", "/d/../chat", "/d//chat", "/d/chat", "/d/*/chat/x"):
self.assertFalse(ok("h.example", "POST", path), path)
finally:
gateway.PROVIDERS[:] = saved[0]
gateway.CLIENT_TOOLS.clear(); gateway.CLIENT_TOOLS.update(saved[1])
gateway.PROVIDER_FIELDS.clear(); gateway.PROVIDER_FIELDS.update(saved[2])
def test_everything_else_is_refused(self):
for host, method, path in [
("chatgpt.com", "POST", "/backend-api/wham/apps"), # account connectors (apps)
("chatgpt.com", "GET", "/backend-api/wham/usage"),
("chatgpt.com", "GET", "/backend-api/wham/settings/user"),
("chatgpt.com", "GET", "/backend-api/wham/accounts/check"),
("chatgpt.com", "POST", "/backend-api/codex/analytics-events/events"),
("chatgpt.com", "POST", "/backend-api/wham/tasks"),
("chatgpt.com", "GET", "/backend-api/plugins/list"),
("chatgpt.com", "POST", "/backend-api/codex/alpha/search"), # Codex's own web search
("chatgpt.com", "POST", "/backend-api/conversation"),
("chatgpt.com", "POST", "/backend-api/files"),
("chatgpt.com", "GET", "/backend-api/codex/responses"), # a listed path, another method
("chatgpt.com", "POST", "/backend-api/codex/models"),
("api.openai.com", "POST", "/v1/files"),
("api.openai.com", "POST", "/v1/images/generations"),
("api.openai.com", "GET", "/v1/responses/resp_123"),
("api.openai.com", "POST", "/v1/conversations"),
("api.openai.com", "POST", "/v1/vector_stores"),
("api.anthropic.com", "POST", "/v1/files"),
("api.anthropic.com", "POST", "/v1/messages/batches"),
("api.anthropic.com", "GET", "/api/oauth/profile"),
("api.anthropic.com", "POST", "/v1/mcp_servers"),
("auth.openai.com", "POST", "/oauth/token"), # an allowlisted host without a list
("ab.chatgpt.com", "POST", "/backend-api/codex/responses"),
("mcp.anthropic.com", "POST", "/v1/messages"),
("unknown.example.com", "POST", "/v1/responses")]:
self.assertFalse(ok(host, method, path), (host, method, path))
def test_no_way_around_a_listed_path(self):
for path in ("/backend-api/codex/responses/../../wham/apps", "/backend-api/codex/responses/%2e%2e/apps",
"/backend-api/codex//responses", "/backend-api/codex/responses/", "/backend-api/codex/./responses",
"/backend-api/codex/responses;x=1", "/backend-api/codex/responses\\..\\apps",
"/BACKEND-API/codex/responses", "/backend-api/codex/responses/compact/more", "", "/",
"/backend-api/codex/responses%3Fx", "/backend-api/codex/*"):
self.assertFalse(ok("chatgpt.com", "POST", path), path)
self.assertFalse(ok("res-1.openai.azure.com", "POST", "/openai/v1/../responses"))
self.assertFalse(ok("10.0.0.5:8080", "POST", "/v1/messages")) # an HTTPS host without a provider
self.assertFalse(ok("chatgpt.com", "POST", "/backend-api/codex/responses", plain=True)) # http: its own list only
def test_bad_rule_files_do_not_load(self):
with tempfile.TemporaryDirectory() as td:
f = pathlib.Path(td) / "p.json"
base = {"client_tool_types": ["function"], "provider_side_fields": ["mcp_servers"]}
for allow in ([["PUT", "/v1/x"]], [["POST", "v1/x"]], [["POST", "/v1/x?y"]], [["POST", "/v1//x"]], [["POST"]]):
f.write_text(json.dumps({**base, "providers": [{"name": "x", "hosts": ["h"], "allow": allow}]}))
with self.assertRaises((ValueError, TypeError)):
gateway.load_paths(f)
for doc in ({}, {"providers": []}, {**base, "client_tool_types": [1], "providers": []}):
f.write_text(json.dumps(doc))
with self.assertRaises((KeyError, ValueError)):
gateway.load_paths(f)
for probe in (["POST", "/v1/files"], ["GET", "/backend-api/wham/apps"], ["HEAD", "/a?b"]): # a probe carries nothing
f.write_text(json.dumps({**base, "harness_probes": [probe], "providers": []}))
with self.assertRaises(ValueError):
gateway.load_paths(f)
self.assertEqual(gateway.CLIENT_TOOLS, {"function", "custom", "namespace"}) # a bad file changes nothing
def test_server_tools(self):
st = gateway.server_tools
# what the harnesses declare: Claude Code's tools carry no type; Codex 0.160 declares its tools inside `input`
self.assertEqual(st(json.dumps({"tools": [{"name": "Bash", "input_schema": {}}, {"type": "function", "name": "x"},
{"type": "custom", "name": "apply_patch"}]})), [])
codex = {"model": "m", "input": [{"type": "message", "content": [{"type": "input_text", "text": "the word \"tools\""}]},
{"type": "additional_tools", "tools": [{"type": "namespace", "name": "functions", "tools": [
{"type": "function", "name": "exec_command"}, {"type": "custom", "name": "apply_patch"}]}]}]}
self.assertEqual(st(json.dumps(codex)), [])
codex["input"][1]["tools"][0]["tools"].append({"type": "web_search"})
self.assertEqual(st(json.dumps(codex)), ["web_search"])
self.assertEqual(st(json.dumps({"tools": [{"type": "web_search"}, {"type": "function"}, {"type": "mcp"},
{"type": "web_search_20250305", "name": "web_search"},
{"type": "shell", "environment": {"type": "container_auto"}}]})),
["mcp", "shell", "web_search", "web_search_20250305"])
self.assertEqual(st('{"\\u0074ools": [{"type": "image_generation"}]}'), ["image_generation"]) # an escaped key
self.assertEqual(st(json.dumps({"tools": [], "mcp_servers": [{"url": "https://x"}]})), ["mcp_servers"])
self.assertEqual(st(json.dumps({"model": "gpt-search", "messages": [], "web_search_options": {}})), ["web_search_options"])
self.assertEqual(st(json.dumps({"model": "openai/gpt-x:online", "messages": [], "plugins": [{"id": "web"}]})),
["model:online", "plugins"])
self.assertEqual(st(json.dumps({"model": "m", "mcp_servers": [], "plugins": None})), [])
self.assertEqual(st(json.dumps({"model": "a", "models": ["b", "c:ONLINE "]})), ["model:online"])
self.assertEqual(st(json.dumps({"model": "m", "input": "hi", "prompt": {"id": "pmpt_1"}})), ["prompt"])
self.assertEqual(st(json.dumps({"tools": [{"type": ["web_search"]}, {"type": {"a": 1}}]})), ["['web_search']", "{'a': 1}"])
self.assertEqual(st(json.dumps({"input": [{"type": "tool_search_output", "tools": [{"type": "mcp", "server_url": "https://x"}]}]})), ["mcp"])
# data is not a declaration: a tool call's arguments, a tool's schema, a message that talks about tools
data = {"model": "m", "tools": [{"name": "Agent", "input_schema": {"type": "object", "properties": {
"tools": {"type": "array", "items": {"type": "string"}}}}},
{"type": "function", "name": "f", "parameters": {"properties": {"tools": {"type": "web_search"}}}}],
"messages": [{"role": "assistant", "content": [
{"type": "tool_use", "id": "t1", "name": "Agent", "input": {"tools": ["Bash", "Read"], "x": {"tools": "all"}}},
{"type": "tool_use", "id": "t2", "name": "Plan", "input": {"tools": [{"type": "hammer"}], "mcp_servers": [1],
"source": {"type": "url", "url": "https://x"}}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [
{"type": "text", "text": "{\"tools\": [{\"type\": \"web_search\"}]}"},
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}]}]}],
"input": [{"type": "function_call", "name": "x", "arguments": "{\"tools\": [{\"type\": \"mcp\"}]}"},
{"type": "message", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AAAA"}]}]}
self.assertEqual(st(json.dumps(data)), [])
# content the provider would fetch by address
for block in ({"type": "image", "source": {"type": "url", "url": "https://x/a.png"}},
{"type": "document", "source": {"type": "url", "url": "https://x/a.pdf"}},
{"type": "tool_result", "tool_use_id": "t", "content": [{"type": "image", "source": {"type": "url", "url": "http://x"}}]},
{"type": "image_url", "image_url": {"url": " HTTPS://x/a.png"}},
{"type": "input_image", "image_url": "https://x/a.png"}, {"type": "input_file", "file_url": "https://x/a.pdf"}):
for key in ("messages", "input"):
self.assertEqual(st(json.dumps({"model": "m", key: [{"role": "user", "content": [block]}]})), ["url_source"], (key, block))
for item in ({"type": "function_call_output", "call_id": "c", "output": [{"type": "input_file", "file_url": "https://x/a.pdf"}]},
{"type": "custom_tool_call_output", "call_id": "c", "output": [{"type": "input_image", "image_url": "https://x/a.png"}]}):
self.assertEqual(st(json.dumps({"model": "m", "input": [item]})), ["url_source"], item)
nested = {"type": "document", "source": {"type": "content", "content": [{"type": "image", "source": {"type": "url", "url": "https://x"}}]}}
self.assertEqual(st(json.dumps({"model": "m", "messages": [{"role": "user", "content": [nested]}]})), ["url_source"])
ok_out = {"type": "function_call_output", "call_id": "c", "output": [{"type": "input_image", "image_url": "data:image/png;base64,AA"},
{"type": "input_text", "text": "https://x is a string"}]}
self.assertEqual(st(json.dumps({"model": "m", "input": [ok_out, {"type": "function_call_output", "call_id": "d", "output": "https://x"}]})), [])
for body in ("", "not json", "[1]", '"x"', '{"tools": 5}', '{"tools": [1]}', '{"tools": [], "tools": [{"type": "mcp"}]}',
"\ufeff{}", "[" * 100000):
with self.assertRaises(gateway.Unreadable, msg=body[:40]):
st(body)
def test_decode_request(self):
import gzip
dec = gateway.decode_request
self.assertEqual(dec(b'{"a": 1}', ""), ('{"a": 1}', "text"))
self.assertEqual(dec(gzip.compress(b'{"a": 1}'), "gzip"), ('{"a": 1}', "text"))
self.assertIsNone(dec(gzip.compress(b"{}") + b"trailing", "gzip")[0]) # more than one gzip member's worth
self.assertIsNone(dec(b'{"a": 1}', "gzip")[0]) # the header lies
for enc in ("br", "deflate", "x-gzip", "gzip, identity"):
self.assertIsNone(dec(b"{}", enc)[0], enc)
self.assertIsNone(dec(b"\xff\xfe{\x00}\x00", "")[0]) # not UTF-8
try:
import zstandard
except ModuleNotFoundError:
return
c = zstandard.ZstdCompressor()
two = c.compress(b'{"tools": [') + c.compress(b'{"type": "mcp"}]}') # two frames: all of them are read
self.assertEqual(dec(two, "zstd"), ('{"tools": [{"type": "mcp"}]}', "text"))
import shutil
if shutil.which("zstd"): # without the module: the zstd program
saved = sys.modules.get("zstandard")
sys.modules["zstandard"] = None
try:
self.assertEqual(dec(two, "zstd"), ('{"tools": [{"type": "mcp"}]}', "text"))
finally:
sys.modules["zstandard"] = saved
class Stub(BaseHTTPRequestHandler):
seen: list[tuple[str, str]] = []
def log_message(self, *a):
pass
def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length") or 0))
Stub.seen.append((self.command, self.path))
body = json.dumps({"type": "message", "model": "stub", "usage": {"input_tokens": 3, "output_tokens": 2,
"cache_read_input_tokens": 1}}).encode()
self.send_response(200); self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
do_GET = do_POST
class EndToEnd(unittest.TestCase):
"""gateway.py as the runner starts it, in front of a stub model server on loopback (an --http-host upstream)."""
@classmethod
def setUpClass(cls):
cls.td = tempfile.TemporaryDirectory()
cls.jobs = pathlib.Path(cls.td.name)
cls.stub = ThreadingHTTPServer(("127.0.0.1", 0), Stub)
threading.Thread(target=cls.stub.serve_forever, daemon=True).start()
cls.up = f"127.0.0.1:{cls.stub.server_address[1]}"
ready = cls.jobs / "ready"
cls.proc = subprocess.Popen([sys.executable, str(HERE / "gateway.py"), "--jobs", str(cls.jobs), "--bind", "127.0.0.1",
"--ready-file", str(ready), "--http-host", cls.up],
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
for _ in range(100):
if ready.exists() and ready.read_text():
break
if cls.proc.poll() is not None:
raise RuntimeError(cls.proc.stderr.read().decode()[-2000:])
time.sleep(0.1)
cls.port = int(ready.read_text())
@classmethod
def tearDownClass(cls):
cls.proc.terminate(); cls.proc.wait(10); cls.proc.stderr.close()
cls.stub.shutdown(); cls.stub.server_close(); cls.td.cleanup()
def call(self, method: str, path: str, body: dict | None = None):
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
data = json.dumps(body).encode() if body is not None else None
c.request(method, path, body=data, headers={"Content-Type": "application/json", "Authorization": "Bearer SECRET-KEY"})
r = c.getresponse(); out = r.read(); c.close()
return r.status, out
def records(self) -> list[dict]:
f = self.jobs / ".gateway" / "unattributed.jsonl" # no trial owns loopback
return [json.loads(x) for x in f.read_text().splitlines()] if f.exists() else []
def test_forwards_model_calls_and_refuses_the_rest(self):
Stub.seen.clear()
st, out = self.call("POST", f"/h/{self.up}/v1/messages?beta=true",
{"model": "m", "messages": [], "tools": [{"name": "Bash", "input_schema": {}},
{"type": "custom", "name": "Edit"}]})
self.assertEqual(st, 200, out)
self.assertEqual(Stub.seen, [("POST", "/v1/messages?beta=true")])
# a tool the provider would run for the agent: refused in the provider's error format, nothing forwarded
for tools in ([{"type": "web_search_20250305", "name": "web_search"}], [{"type": "function", "name": "f"}, {"type": "mcp"}],
[{"type": "image_generation"}], [{"type": "web_search"}]):
st, out = self.call("POST", f"/h/{self.up}/v1/messages", {"model": "m", "messages": [], "tools": tools})
self.assertEqual(st, 403, out)
self.assertIn("asks the model provider to run", json.loads(out)["error"]["message"])
# a body the gateway cannot read is not forwarded either (one keep-alive connection for all of them)
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
for raw, enc in ((b"", None), (b"not json", None), (b'{"tools": [], "tools": [{"type": "mcp"}]}', None),
(b'{"model": "m"}', "br"), (b'{"model": "m"}', "gzip")):
hdr = {"Content-Type": "application/json", **({"Content-Encoding": enc} if enc else {})}
c.request("POST", f"/h/{self.up}/v1/messages", body=raw, headers=hdr)
r = c.getresponse(); out = r.read()
self.assertEqual(r.status, 403, (raw, enc, out))
self.assertIn("not one JSON object", json.loads(out)["error"]["message"])
c.request("POST", f"/h/{self.up}/v1/messages", body=b"{}", headers={"Content-Length": "abc"}) # answered, not dropped
self.assertEqual(c.getresponse().status, 400)
c.close()
# two Content-Encoding headers (which one would the upstream believe?), a chunked body, a zstd label on other bytes
import socket
for head, body in ((b"Content-Encoding: identity\r\nContent-Encoding: gzip\r\nContent-Length: 2\r\n", b"{}"),
(b"Transfer-Encoding: chunked\r\n", b"2\r\n{}\r\n0\r\n\r\n"),
(b"Content-Encoding: zstd\r\nContent-Length: 14\r\n", b'{"model": "m"}')):
with socket.create_connection(("127.0.0.1", self.port), timeout=30) as sk:
sk.sendall(b"POST /h/" + self.up.encode() + b"/v1/messages HTTP/1.1\r\nHost: x\r\n" + head + b"\r\n" + body)
self.assertTrue(sk.recv(65536).startswith(b"HTTP/1.1 403"), head)
before = len(self.records()) # Claude Code's own start-up probe: refused, but no record of a refusal
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
c.request("HEAD", f"/h/{self.up}/api/hello")
r = c.getresponse(); r.read(); c.close()
self.assertEqual((r.status, len(self.records())), (403, before))
self.assertEqual(self.call("GET", f"/h/{self.up}/api/hello")[0], 403) # only that method: this one is recorded
self.assertEqual(len(self.records()), before + 1)
for method in ("PUT", "DELETE", "PATCH"):
self.assertEqual(self.call(method, f"/h/{self.up}/v1/messages", {} if method != "DELETE" else None)[0], 403)
self.assertEqual(len(Stub.seen), 1, Stub.seen)
st, _ = self.call("POST", f"/h/{self.up}/v1/chat/completions", {"model": "m", "messages": []}) # the self-hosted upstream
self.assertEqual((st, Stub.seen[-1]), (200, ("POST", "/v1/chat/completions")))
Stub.seen.pop()
try: # a zstd body, as Codex sends with a ChatGPT login: read, checked, forwarded as sent
import zstandard
z = zstandard.ZstdCompressor()
for payload, want in (({"model": "m", "messages": []}, 200), ({"model": "m", "messages": [], "tools": [{"type": "mcp"}]}, 403)):
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
c.request("POST", f"/h/{self.up}/v1/messages", body=z.compress(json.dumps(payload).encode()),
headers={"Content-Type": "application/json", "Content-Encoding": "zstd"})
r = c.getresponse(); r.read(); c.close()
self.assertEqual(r.status, want, payload)
Stub.seen.pop()
except ModuleNotFoundError:
pass
self.assertEqual(len(Stub.seen), 1, Stub.seen)
for method, path in (("POST", "/v1/files"), ("GET", "/v1/messages"), ("POST", "/v1/messages/../files"),
("POST", "/backend-api/wham/apps"), ("POST", "/v1/messages/batches")):
st, out = self.call(method, f"/h/{self.up}{path}", {"secret": "BODY-NOT-LOGGED"} if method == "POST" else None)
self.assertEqual(st, 403, (method, path, out))
self.assertIn("not a model API call", json.loads(out)["error"]["message"])
self.assertEqual(len(Stub.seen), 1, Stub.seen) # nothing refused reached the upstream
st, _ = self.call("POST", "/u/chatgpt.com/backend-api/wham/apps", {}) # refused before any connection
self.assertEqual(st, 403)
st, _ = self.call("POST", "/u/github.com/v1/responses", {}) # not a model API host at all
self.assertEqual(st, 403)
recs = self.records()
refused = [r for r in recs if r.get("refused") == "path_not_allowed"]
self.assertEqual(sorted((r["method"], r["path"]) for r in refused if r["method"] in ("GET", "POST")),
sorted([("POST", "/v1/files"), ("GET", "/v1/messages"), ("POST", "/v1/messages/../files"),
("POST", "/backend-api/wham/apps"), ("POST", "/v1/messages/batches"), ("GET", "/api/hello"),
("POST", "/backend-api/wham/apps")]))
self.assertFalse([r for r in recs if r["method"] == "HEAD"])
self.assertTrue(all(r["status"] == 403 for r in refused))
self.assertEqual([(r["host"], r["path"]) for r in recs if r.get("refused") == "host_not_allowed"], [("github.com", "/v1/responses")])
tool = [r for r in recs if r.get("refused") == "server_tool"]
self.assertEqual([r["server_tools"] for r in tool][:4], [["web_search_20250305"], ["mcp"], ["image_generation"], ["web_search"]])
self.assertEqual(len([r for r in recs if r.get("refused") == "body_unreadable"]), 8)
self.assertEqual(sorted(r["method"] for r in refused if r["method"] not in ("GET", "POST")), ["DELETE", "PATCH", "PUT"])
fwd = [r for r in recs if not r.get("refused")]
self.assertTrue(all("server_tools" not in r and "body_unchecked" not in r for r in fwd), fwd)
fwd = [r for r in fwd if r["path"] == "/v1/messages" and r["request_bytes"] > 60]
self.assertEqual(len(fwd), 1)
self.assertEqual(fwd[0]["usage"], {"input": 4, "cached": 1, "output": 2})
everything = "".join(p.read_text() for p in (self.jobs / ".gateway").iterdir() if p.is_file())
self.assertNotIn("SECRET-KEY", everything)
self.assertNotIn("BODY-NOT-LOGGED", everything)
class Broken(BaseHTTPRequestHandler):
"""An upstream that fails: /v1/messages answers 503; /v1/responses starts a stream and breaks it off mid-way."""
def log_message(self, *a):
pass
def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length") or 0))
if self.path.startswith("/v1/messages"):
body = b'{"type": "error", "error": {"type": "overloaded_error"}}'
self.send_response(503); self.send_header("Content-Length", str(len(body))); self.end_headers()
self.wfile.write(body)
return
self.send_response(200); self.send_header("Content-Type", "text/event-stream")
self.send_header("Content-Length", "100000"); self.end_headers()
self.wfile.write(b'data: {"type": "response.created"}\n\n'); self.wfile.flush()
self.close_connection = True # far short of the announced length: the stream is broken off
class UpstreamFailures(unittest.TestCase):
"""A model call that fails on the way (the upstream unreachable: 502; a stream broken off; a provider
5xx) is recorded as an upstream_failure, and totals.json counts the failures in a row up to the latest call."""
def test_tally(self):
gateway.ARGS = type("A", (), {"budget_input": 0, "budget_output": 0})()
tot = {"requests": 0, "input": 0, "cached": 0, "output": 0, "no_usage": 0}
ok = {"ts": 1, "usage": {"input": 5, "cached": 0, "output": 1}}
for rec in (ok, {"ts": 2, "forwarded": False, "upstream_failure": "unreachable: ConnectionRefusedError"},
{"ts": 3, "usage": {"input": 2, "cached": 0, "output": 0}, "upstream_failure": "stream broken off: X"},
{"ts": 4, "refused": "path_not_allowed"}):
gateway.tally(tot, rec)
self.assertEqual((tot["requests"], tot["input"], tot["upstream_failures"], tot["failures_in_a_row"]), (2, 7, 2, 2))
self.assertEqual(tot["last_upstream_failure"], "stream broken off: X")
gateway.tally(tot, {**ok, "ts": 5})
self.assertEqual((tot["requests"], tot["upstream_failures"], tot["failures_in_a_row"]), (3, 2, 0))
def test_end_to_end(self):
import socket
with tempfile.TemporaryDirectory() as td:
jobs = pathlib.Path(td)
stub = ThreadingHTTPServer(("127.0.0.1", 0), Broken)
threading.Thread(target=stub.serve_forever, daemon=True).start()
up = f"127.0.0.1:{stub.server_address[1]}"
with socket.socket() as sk: # a port nothing listens on
sk.bind(("127.0.0.1", 0)); dead = f"127.0.0.1:{sk.getsockname()[1]}"
ready = jobs / "ready"
proc = subprocess.Popen([sys.executable, str(HERE / "gateway.py"), "--jobs", str(jobs), "--bind", "127.0.0.1",
"--ready-file", str(ready), "--http-host", up, "--http-host", dead],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
for _ in range(100):
if ready.exists() and ready.read_text():
break
time.sleep(0.1)
port = int(ready.read_text())
def call(path):
c = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
c.request("POST", path, body=b'{"model": "m", "input": []}', headers={"Content-Type": "application/json"})
r = c.getresponse()
try:
r.read()
whole = True
except http.client.HTTPException:
whole = False
c.close()
return r.status, whole
self.assertEqual(call(f"/h/{dead}/v1/responses")[0], 502)
self.assertEqual(call(f"/h/{up}/v1/messages"), (503, True))
self.assertEqual(call(f"/h/{up}/v1/responses"), (200, False)) # the agent sees the stream end broken
recs = [json.loads(x) for x in (jobs / ".gateway" / "unattributed.jsonl").read_text().splitlines()]
fails = [r.get("upstream_failure", "") for r in recs]
self.assertEqual(len(recs), 3, recs)
self.assertTrue(fails[0].startswith("unreachable: ") and recs[0]["forwarded"] is False, recs[0])
self.assertEqual(fails[1], "HTTP 503")
self.assertTrue(fails[2].startswith("stream broken off: "), recs[2])
self.assertEqual(recs[2]["status"], 200)
finally:
proc.terminate(); proc.wait(10)
stub.shutdown(); stub.server_close()
if __name__ == "__main__":
unittest.main()