Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): give the upstream write phase its own timeout (#3327)

Tejas Chopraa507249b9270
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index e802bd0d6..a8758a929 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -514,6 +514,19 @@ def dashboard(port: int, no_open: bool) -> None:
"Env: HEADROOM_CONNECT_TIMEOUT_SECONDS."
),
)
+ "--write-timeout-seconds",
+ type=click.IntRange(min=1),
+ default=None,
+ envvar="HEADROOM_WRITE_TIMEOUT_SECONDS",
+ help=(
+ "Seconds the upstream send may take before it is abandoned (default: 150). "
+ "On HTTP/1.1 this bounds the whole request body, so raise it if you push "
+ "large bodies over a slow link. Lower it to fail over a dead pooled "
+ "connection faster; --connect-timeout-seconds only guards a fresh connect. "
+ "Env: HEADROOM_WRITE_TIMEOUT_SECONDS."
+ ),
+)
@click.option(
"--anthropic-buffered-request-timeout-seconds",
type=click.IntRange(min=1),
@@ -1042,6 +1055,7 @@ def proxy(
retry_max_delay_ms: int | None,
request_timeout_seconds: int | None,
connect_timeout_seconds: int | None,
+ write_timeout_seconds: int | None,
anthropic_buffered_request_timeout_seconds: int | None,
anthropic_pre_upstream_concurrency: int | None,
anthropic_pre_upstream_acquire_timeout_seconds: float | None,
@@ -1376,6 +1390,7 @@ def proxy(
connect_timeout_seconds=connect_timeout_seconds
if connect_timeout_seconds is not None
else 10,
+ write_timeout_seconds=write_timeout_seconds if write_timeout_seconds is not None else 150,
anthropic_buffered_request_timeout_seconds=(
anthropic_buffered_request_timeout_seconds
if anthropic_buffered_request_timeout_seconds is not None
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 5f5e4a510..652ce47fe 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -469,7 +469,10 @@ class AnthropicHandlerMixin:
return httpx.Timeout(
connect=self.config.connect_timeout_seconds,
read=self.config.anthropic_buffered_request_timeout_seconds,
- write=self.config.request_timeout_seconds,
+ # A buffered turn waits longer for the *answer*, but sending the
+ # request is the same operation it always was — it gets the same
+ # write bound as every other path (#3259).
+ write=self.config.write_timeout_seconds,
pool=self.config.connect_timeout_seconds,
)
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 6b85689c3..fd93a9c72 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -368,6 +368,26 @@ class ProxyConfig:
# Timeouts
request_timeout_seconds: int = 300
connect_timeout_seconds: int = 10
+ # Sending the request is a different operation from waiting for the model to
+ # answer, but `write` used to inherit `request_timeout_seconds`, so pushing
+ # bytes got the same 300s budget as a model thinking. That left the write
+ # phase effectively unbounded in practice: when an upstream stops draining —
+ # a pooled socket whose peer went away over a laptop sleep, a network change,
+ # a NAT timeout — the send blocks until the OS gives up retransmitting
+ # (~180-220s on macOS), which is *under* 300s, so no timeout ever fired and
+ # the proxy hung, retried, and hung again (#3259).
+ #
+ # This bounds the WHOLE body send, not one chunk of it. httpx hands a bytes
+ # body to the transport as a single write, so on HTTP/1.1 the entire upload
+ # runs inside one timer -- measured: a 16MB body against a peer draining
+ # steadily at ~1MB/s raises WriteTimeout at exactly the configured bound,
+ # healthy peer or not. On HTTP/2 the body is split by flow control and the
+ # waiting-for-window part is charged to `read`, so only real socket writes
+ # count against it. The default has to clear the slower of those two while
+ # staying under the OS retransmit ceiling that made the inherited 300s
+ # unreachable: 150s carries a 15MB body -- the largest #3259 reports -- over
+ # a ~1 Mbps uplink, and still fires well before the OS gives up at ~180s.
+ write_timeout_seconds: int = 150
# Anthropic buffered reads can legitimately run longer than the generic
# proxy request cap. Keep the generic timeout unchanged elsewhere.
anthropic_buffered_request_timeout_seconds: int = 600
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index e4c94d70c..a063fb0c8 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -654,7 +654,8 @@ def _provider_httpx_client_options(
"timeout": httpx.Timeout(
connect=config.connect_timeout_seconds,
read=config.request_timeout_seconds,
- write=config.request_timeout_seconds,
+ # Not request_timeout_seconds: see ProxyConfig.write_timeout_seconds.
+ write=config.write_timeout_seconds,
pool=config.connect_timeout_seconds,
),
"limits": httpx.Limits(
@@ -5443,6 +5444,11 @@ def _proxy_config_from_env() -> ProxyConfig:
600,
min_value=1,
),
+ write_timeout_seconds=_get_env_int(
+ "HEADROOM_WRITE_TIMEOUT_SECONDS",
+ 150,
+ min_value=1,
+ ),
buffered_ccr_grace_seconds=_get_env_float(
"HEADROOM_BUFFERED_CCR_GRACE_SECONDS",
DEFAULT_BUFFERED_CCR_GRACE_SECONDS,
@@ -6193,6 +6199,13 @@ if __name__ == "__main__":
args.anthropic_buffered_request_timeout_seconds,
min_value=1,
),
+ # Env-only here, like connect/request: this parser exposes no timeout
+ # flag but the buffered one.
+ write_timeout_seconds=_get_env_int(
+ "HEADROOM_WRITE_TIMEOUT_SECONDS",
+ 150,
+ min_value=1,
+ ),
vertex_api_url=_get_env_str("VERTEX_TARGET_API_URL", args.vertex_api_url),
# Backend settings
backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]
diff --git a/headroom/settings_store.py b/headroom/settings_store.py
index ad134ebc0..4e36fc73b 100644
--- a/headroom/settings_store.py
+++ b/headroom/settings_store.py
@@ -561,6 +561,22 @@ SETTINGS: tuple[SettingField, ...] = (
help="Upstream connection timeout in seconds. Default: 10.",
tier="advanced",
),
+ SettingField(
+ "HEADROOM_WRITE_TIMEOUT_SECONDS",
+ "write_timeout_seconds",
+ "Write timeout (s)",
+ "Timeouts",
+ "int",
+ default=None,
+ minimum=1,
+ help=(
+ "Seconds the upstream send may take before it is abandoned. Default: 150. "
+ "On HTTP/1.1 this bounds the whole request body, so raise it if you push "
+ "large bodies over a slow link. Lower it to fail over a dead pooled "
+ "connection faster; the connect timeout only guards a fresh connect."
+ ),
+ tier="advanced",
+ ),
SettingField(
"HEADROOM_ANTHROPIC_BUFFERED_REQUEST_TIMEOUT_SECONDS",
"anthropic_buffered_request_timeout_seconds",
diff --git a/tests/test_proxy/test_anthropic_buffered_timeout.py b/tests/test_proxy/test_anthropic_buffered_timeout.py
index 6e0c753bb..b9a379b94 100644
--- a/tests/test_proxy/test_anthropic_buffered_timeout.py
+++ b/tests/test_proxy/test_anthropic_buffered_timeout.py
@@ -56,7 +56,10 @@ def _assert_buffered_timeout(timeout: httpx.Timeout | None) -> None:
assert isinstance(timeout, httpx.Timeout)
assert timeout.connect == 3.0
assert timeout.read == 19.0
- assert timeout.write == 7.0
+ # `write` is no longer request_timeout_seconds (7): sending the request is a
+ # separate operation from waiting for the answer, so it has its own bound --
+ # the fixture's write_timeout_seconds, not the read budget (#3259).
+ assert timeout.write == 11.0
assert timeout.pool == 3.0
@@ -73,6 +76,7 @@ def _make_config() -> ProxyConfig:
image_optimize=False,
connect_timeout_seconds=3,
request_timeout_seconds=7,
+ write_timeout_seconds=11,
anthropic_buffered_request_timeout_seconds=19,
)
@@ -461,5 +465,9 @@ def test_generic_proxy_timeout_defaults_stay_unchanged():
assert timeout.connect == 10.0
assert timeout.read == 300.0
- assert timeout.write == 300.0
assert timeout.pool == 10.0
+ # `write` deliberately no longer tracks read. Inheriting 300s put the send
+ # bound above the ~180-220s point where the OS abandons retransmission, so
+ # it could never fire against a dead peer (#3259).
+ assert timeout.write == 150.0
+ assert timeout.write < timeout.read
diff --git a/tests/test_upstream_write_timeout.py b/tests/test_upstream_write_timeout.py
new file mode 100644
index 000000000..f3bf18610
--- /dev/null
+++ b/tests/test_upstream_write_timeout.py
@@ -0,0 +1,109 @@
+"""The write timeout must be independent of the read timeout.
+
+`write` used to inherit `request_timeout_seconds`, so pushing request bytes got
+the same budget as waiting for a model to answer. That left the write phase
+effectively unbounded against a dead peer: when an upstream stops draining, the
+send blocks until the OS abandons retransmission (~180-220s on macOS), which is
+*under* the 300s it inherited — so no timeout fired, the request hung, the retry
+hung again, and a single incident blocked the client for minutes (#3259).
+
+Separating them is what lets the existing retry logic fail over to a fresh
+connection instead of stalling behind a socket whose peer is gone.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.proxy.models import ProxyConfig
+from headroom.proxy.server import _provider_httpx_client_options
+
+
+def _timeout(config: ProxyConfig):
+ _http2, kwargs = _provider_httpx_client_options(config, verify=True)
+ return kwargs["timeout"]
+
+
+def test_write_does_not_inherit_the_read_budget() -> None:
+ """The regression itself: a long read budget must not extend the send."""
+ timeout = _timeout(ProxyConfig(request_timeout_seconds=300))
+
+ assert timeout.read == 300
+ assert timeout.write != 300
+ assert timeout.write == ProxyConfig().write_timeout_seconds
+
+
+def test_write_timeout_defaults_below_the_os_retransmit_ceiling() -> None:
+ """A default above ~180s would never fire before the OS gave up anyway.
+
+ The whole failure mode is that the inherited 300s sat above the point where
+ macOS abandons retransmission, so the timeout was unreachable in practice.
+ """
+ assert ProxyConfig().write_timeout_seconds < 180
+
+
+def test_write_timeout_default_can_carry_a_large_body() -> None:
+ """The bound covers the whole upload, so it has to fit a real request.
+
+ httpx hands a bytes body to the transport as one write, so on HTTP/1.1 the
+ entire body is sent inside a single timer -- this is NOT a per-chunk budget.
+ Measured against a peer draining a 16MB body at ~1MB/s, WriteTimeout fires
+ at exactly the configured bound even though the peer is healthy. #3259's
+ reporter sends 7-15MB bodies, so a default that cannot carry 15MB over a
+ modest uplink would turn this fix into an outage for them.
+ """
+ budget = ProxyConfig().write_timeout_seconds
+ largest_reported_body_bytes = 15 * 1024 * 1024
+ slow_uplink_bytes_per_second = 125 * 1024 # ~1 Mbps
+
+ assert largest_reported_body_bytes / slow_uplink_bytes_per_second <= budget
+
+
+def test_write_timeout_is_configurable() -> None:
+ timeout = _timeout(ProxyConfig(write_timeout_seconds=15))
+
+ assert timeout.write == 15
+
+
+def test_other_phases_are_unchanged() -> None:
+ """Only `write` moves; connect/read/pool keep the values they always had."""
+ config = ProxyConfig(
+ request_timeout_seconds=300,
+ connect_timeout_seconds=10,
+ )
+
+ timeout = _timeout(config)
+
+ assert timeout.connect == 10
+ assert timeout.read == 300
+ assert timeout.pool == 10
+
+
[email protected]("buffered_read", [600, 900])
+def test_buffered_anthropic_turn_keeps_its_long_read_but_bounded_write(
+ buffered_read: int,
+) -> None:
+ """A buffered turn waits longer for the answer, not longer to send.
+
+ This path sets its own timeout, so it needs the split applied too —
+ otherwise the one path most likely to carry a large body keeps the
+ unbounded write.
+ """
+ from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
+
+ class _Handler(AnthropicHandlerMixin):
+ def __init__(self, config: ProxyConfig) -> None:
+ self.config = config
+
+ handler = _Handler(
+ ProxyConfig(
+ anthropic_buffered_request_timeout_seconds=buffered_read,
+ request_timeout_seconds=300,
+ write_timeout_seconds=60,
+ )
+ )
+
+ timeout = handler._anthropic_buffered_request_timeout()
+
+ assert timeout.read == buffered_read
+ assert timeout.write == 60