Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): give the upstream write phase its own timeout (#3327)
Tejas Chopraa507249b9270
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex e802bd0d6..a8758a929 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -514,6 +514,19 @@ def dashboard(port: int, no_open: bool) -> None:"Env: HEADROOM_CONNECT_TIMEOUT_SECONDS."),)+ "--write-timeout-seconds",+ type=click.IntRange(min=1),+ default=None,+ envvar="HEADROOM_WRITE_TIMEOUT_SECONDS",+ help=(+ "Seconds the upstream send may take before it is abandoned (default: 150). "+ "On HTTP/1.1 this bounds the whole request body, so raise it if you push "+ "large bodies over a slow link. Lower it to fail over a dead pooled "+ "connection faster; --connect-timeout-seconds only guards a fresh connect. "+ "Env: HEADROOM_WRITE_TIMEOUT_SECONDS."+ ),+)@click.option("--anthropic-buffered-request-timeout-seconds",type=click.IntRange(min=1),@@ -1042,6 +1055,7 @@ def proxy(retry_max_delay_ms: int | None,request_timeout_seconds: int | None,connect_timeout_seconds: int | None,+ write_timeout_seconds: int | None,anthropic_buffered_request_timeout_seconds: int | None,anthropic_pre_upstream_concurrency: int | None,anthropic_pre_upstream_acquire_timeout_seconds: float | None,@@ -1376,6 +1390,7 @@ def proxy(connect_timeout_seconds=connect_timeout_secondsif connect_timeout_seconds is not Noneelse 10,+ write_timeout_seconds=write_timeout_seconds if write_timeout_seconds is not None else 150,anthropic_buffered_request_timeout_seconds=(anthropic_buffered_request_timeout_secondsif anthropic_buffered_request_timeout_seconds is not Nonediff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 5f5e4a510..652ce47fe 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -469,7 +469,10 @@ class AnthropicHandlerMixin:return httpx.Timeout(connect=self.config.connect_timeout_seconds,read=self.config.anthropic_buffered_request_timeout_seconds,- write=self.config.request_timeout_seconds,+ # A buffered turn waits longer for the *answer*, but sending the+ # request is the same operation it always was — it gets the same+ # write bound as every other path (#3259).+ write=self.config.write_timeout_seconds,pool=self.config.connect_timeout_seconds,)diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 6b85689c3..fd93a9c72 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -368,6 +368,26 @@ class ProxyConfig:# Timeoutsrequest_timeout_seconds: int = 300connect_timeout_seconds: int = 10+ # Sending the request is a different operation from waiting for the model to+ # answer, but `write` used to inherit `request_timeout_seconds`, so pushing+ # bytes got the same 300s budget as a model thinking. That left the write+ # phase effectively unbounded in practice: when an upstream stops draining —+ # a pooled socket whose peer went away over a laptop sleep, a network change,+ # a NAT timeout — the send blocks until the OS gives up retransmitting+ # (~180-220s on macOS), which is *under* 300s, so no timeout ever fired and+ # the proxy hung, retried, and hung again (#3259).+ #+ # This bounds the WHOLE body send, not one chunk of it. httpx hands a bytes+ # body to the transport as a single write, so on HTTP/1.1 the entire upload+ # runs inside one timer -- measured: a 16MB body against a peer draining+ # steadily at ~1MB/s raises WriteTimeout at exactly the configured bound,+ # healthy peer or not. On HTTP/2 the body is split by flow control and the+ # waiting-for-window part is charged to `read`, so only real socket writes+ # count against it. The default has to clear the slower of those two while+ # staying under the OS retransmit ceiling that made the inherited 300s+ # unreachable: 150s carries a 15MB body -- the largest #3259 reports -- over+ # a ~1 Mbps uplink, and still fires well before the OS gives up at ~180s.+ write_timeout_seconds: int = 150# Anthropic buffered reads can legitimately run longer than the generic# proxy request cap. Keep the generic timeout unchanged elsewhere.anthropic_buffered_request_timeout_seconds: int = 600diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex e4c94d70c..a063fb0c8 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -654,7 +654,8 @@ def _provider_httpx_client_options("timeout": httpx.Timeout(connect=config.connect_timeout_seconds,read=config.request_timeout_seconds,- write=config.request_timeout_seconds,+ # Not request_timeout_seconds: see ProxyConfig.write_timeout_seconds.+ write=config.write_timeout_seconds,pool=config.connect_timeout_seconds,),"limits": httpx.Limits(@@ -5443,6 +5444,11 @@ def _proxy_config_from_env() -> ProxyConfig:600,min_value=1,),+ write_timeout_seconds=_get_env_int(+ "HEADROOM_WRITE_TIMEOUT_SECONDS",+ 150,+ min_value=1,+ ),buffered_ccr_grace_seconds=_get_env_float("HEADROOM_BUFFERED_CCR_GRACE_SECONDS",DEFAULT_BUFFERED_CCR_GRACE_SECONDS,@@ -6193,6 +6199,13 @@ if __name__ == "__main__":args.anthropic_buffered_request_timeout_seconds,min_value=1,),+ # Env-only here, like connect/request: this parser exposes no timeout+ # flag but the buffered one.+ write_timeout_seconds=_get_env_int(+ "HEADROOM_WRITE_TIMEOUT_SECONDS",+ 150,+ min_value=1,+ ),vertex_api_url=_get_env_str("VERTEX_TARGET_API_URL", args.vertex_api_url),# Backend settingsbackend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]diff --git a/headroom/settings_store.py b/headroom/settings_store.pyindex ad134ebc0..4e36fc73b 100644--- a/headroom/settings_store.py+++ b/headroom/settings_store.py@@ -561,6 +561,22 @@ SETTINGS: tuple[SettingField, ...] = (help="Upstream connection timeout in seconds. Default: 10.",tier="advanced",),+ SettingField(+ "HEADROOM_WRITE_TIMEOUT_SECONDS",+ "write_timeout_seconds",+ "Write timeout (s)",+ "Timeouts",+ "int",+ default=None,+ minimum=1,+ help=(+ "Seconds the upstream send may take before it is abandoned. Default: 150. "+ "On HTTP/1.1 this bounds the whole request body, so raise it if you push "+ "large bodies over a slow link. Lower it to fail over a dead pooled "+ "connection faster; the connect timeout only guards a fresh connect."+ ),+ tier="advanced",+ ),SettingField("HEADROOM_ANTHROPIC_BUFFERED_REQUEST_TIMEOUT_SECONDS","anthropic_buffered_request_timeout_seconds",diff --git a/tests/test_proxy/test_anthropic_buffered_timeout.py b/tests/test_proxy/test_anthropic_buffered_timeout.pyindex 6e0c753bb..b9a379b94 100644--- a/tests/test_proxy/test_anthropic_buffered_timeout.py+++ b/tests/test_proxy/test_anthropic_buffered_timeout.py@@ -56,7 +56,10 @@ def _assert_buffered_timeout(timeout: httpx.Timeout | None) -> None:assert isinstance(timeout, httpx.Timeout)assert timeout.connect == 3.0assert timeout.read == 19.0- assert timeout.write == 7.0+ # `write` is no longer request_timeout_seconds (7): sending the request is a+ # separate operation from waiting for the answer, so it has its own bound --+ # the fixture's write_timeout_seconds, not the read budget (#3259).+ assert timeout.write == 11.0assert timeout.pool == 3.0@@ -73,6 +76,7 @@ def _make_config() -> ProxyConfig:image_optimize=False,connect_timeout_seconds=3,request_timeout_seconds=7,+ write_timeout_seconds=11,anthropic_buffered_request_timeout_seconds=19,)@@ -461,5 +465,9 @@ def test_generic_proxy_timeout_defaults_stay_unchanged():assert timeout.connect == 10.0assert timeout.read == 300.0- assert timeout.write == 300.0assert timeout.pool == 10.0+ # `write` deliberately no longer tracks read. Inheriting 300s put the send+ # bound above the ~180-220s point where the OS abandons retransmission, so+ # it could never fire against a dead peer (#3259).+ assert timeout.write == 150.0+ assert timeout.write < timeout.readdiff --git a/tests/test_upstream_write_timeout.py b/tests/test_upstream_write_timeout.pynew file mode 100644index 000000000..f3bf18610--- /dev/null+++ b/tests/test_upstream_write_timeout.py@@ -0,0 +1,109 @@+"""The write timeout must be independent of the read timeout.++`write` used to inherit `request_timeout_seconds`, so pushing request bytes got+the same budget as waiting for a model to answer. That left the write phase+effectively unbounded against a dead peer: when an upstream stops draining, the+send blocks until the OS abandons retransmission (~180-220s on macOS), which is+*under* the 300s it inherited — so no timeout fired, the request hung, the retry+hung again, and a single incident blocked the client for minutes (#3259).++Separating them is what lets the existing retry logic fail over to a fresh+connection instead of stalling behind a socket whose peer is gone.+"""++from __future__ import annotations++import pytest++from headroom.proxy.models import ProxyConfig+from headroom.proxy.server import _provider_httpx_client_options+++def _timeout(config: ProxyConfig):+ _http2, kwargs = _provider_httpx_client_options(config, verify=True)+ return kwargs["timeout"]+++def test_write_does_not_inherit_the_read_budget() -> None:+ """The regression itself: a long read budget must not extend the send."""+ timeout = _timeout(ProxyConfig(request_timeout_seconds=300))++ assert timeout.read == 300+ assert timeout.write != 300+ assert timeout.write == ProxyConfig().write_timeout_seconds+++def test_write_timeout_defaults_below_the_os_retransmit_ceiling() -> None:+ """A default above ~180s would never fire before the OS gave up anyway.++ The whole failure mode is that the inherited 300s sat above the point where+ macOS abandons retransmission, so the timeout was unreachable in practice.+ """+ assert ProxyConfig().write_timeout_seconds < 180+++def test_write_timeout_default_can_carry_a_large_body() -> None:+ """The bound covers the whole upload, so it has to fit a real request.++ httpx hands a bytes body to the transport as one write, so on HTTP/1.1 the+ entire body is sent inside a single timer -- this is NOT a per-chunk budget.+ Measured against a peer draining a 16MB body at ~1MB/s, WriteTimeout fires+ at exactly the configured bound even though the peer is healthy. #3259's+ reporter sends 7-15MB bodies, so a default that cannot carry 15MB over a+ modest uplink would turn this fix into an outage for them.+ """+ budget = ProxyConfig().write_timeout_seconds+ largest_reported_body_bytes = 15 * 1024 * 1024+ slow_uplink_bytes_per_second = 125 * 1024 # ~1 Mbps++ assert largest_reported_body_bytes / slow_uplink_bytes_per_second <= budget+++def test_write_timeout_is_configurable() -> None:+ timeout = _timeout(ProxyConfig(write_timeout_seconds=15))++ assert timeout.write == 15+++def test_other_phases_are_unchanged() -> None:+ """Only `write` moves; connect/read/pool keep the values they always had."""+ config = ProxyConfig(+ request_timeout_seconds=300,+ connect_timeout_seconds=10,+ )++ timeout = _timeout(config)++ assert timeout.connect == 10+ assert timeout.read == 300+ assert timeout.pool == 10++[email protected]("buffered_read", [600, 900])+def test_buffered_anthropic_turn_keeps_its_long_read_but_bounded_write(+ buffered_read: int,+) -> None:+ """A buffered turn waits longer for the answer, not longer to send.++ This path sets its own timeout, so it needs the split applied too —+ otherwise the one path most likely to carry a large body keeps the+ unbounded write.+ """+ from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin++ class _Handler(AnthropicHandlerMixin):+ def __init__(self, config: ProxyConfig) -> None:+ self.config = config++ handler = _Handler(+ ProxyConfig(+ anthropic_buffered_request_timeout_seconds=buffered_read,+ request_timeout_seconds=300,+ write_timeout_seconds=60,+ )+ )++ timeout = handler._anthropic_buffered_request_timeout()++ assert timeout.read == buffered_read+ assert timeout.write == 60