Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): count a ceiling-stopped stream exactly instead of estimating its text (#3771)
Tejas Choprab36e59a89370
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 03a698fe7..bc74fbb42 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -969,13 +969,20 @@ class StreamingMixin:output_tokens, output_tokens_source = estimate_output_tokens(sse_text=full_sse_data,total_bytes=stream_state["total_bytes"],+ # The provider body as sent, so a turn stopped by its output+ # ceiling can be counted exactly instead of estimated. Matters+ # most for a tool call truncated mid-arguments, whose dropped+ # JSON leaves almost no text to count.+ body=body,)# Name the actual basis. The old message always said "from N bytes"# even though that is now only true for the fallback rung, and an# operator reading it needs to know which estimate they are looking# at before trusting the number.basis = (- "counted from stream text"+ "exact: turn hit its output-token ceiling"+ if output_tokens_source == "exact_ceiling"+ else "counted from stream text"if output_tokens_source == "estimated_text"else f"estimated from {stream_state['total_bytes']} raw SSE bytes")diff --git a/headroom/proxy/stream_output_tokens.py b/headroom/proxy/stream_output_tokens.pyindex 96e410581..965f7b7ea 100644--- a/headroom/proxy/stream_output_tokens.py+++ b/headroom/proxy/stream_output_tokens.py@@ -123,15 +123,88 @@ def extract_stream_text(sse: str) -> str:return "".join(parts)-def estimate_output_tokens(*, sse_text: str, total_bytes: int) -> tuple[int, str]:+# Terminal stop reasons that mean "the output-token ceiling cut this turn off".+# Anthropic messages say ``max_tokens``; OpenAI chat completions say ``length``.+_CEILING_STOP_REASONS = frozenset({"max_tokens", "length", "max_output_tokens"})++# Request fields that carry that ceiling, per surface. All three are denominated+# in OUTPUT tokens (reasoning included, where a model emits it), which is what+# makes the ceiling an exact count rather than a lower bound.+_CEILING_FIELDS = ("max_tokens", "max_completion_tokens", "max_output_tokens")+++def _hit_output_ceiling(sse_text: str) -> bool:+ """Whether the stream ended because it hit the request's output ceiling."""+ for obj in _iter_sse_payloads(sse_text):+ if not isinstance(obj, dict):+ continue++ # OpenAI chat completions — choices[].finish_reason+ choices = obj.get("choices")+ if isinstance(choices, list):+ for choice in choices:+ if isinstance(choice, dict) and choice.get("finish_reason") in (+ _CEILING_STOP_REASONS+ ):+ return True+ continue++ # Anthropic messages — message_delta.delta.stop_reason+ delta = obj.get("delta")+ if isinstance(delta, dict) and delta.get("stop_reason") in _CEILING_STOP_REASONS:+ return True++ # OpenAI responses — response.incomplete_details.reason+ response = obj.get("response")+ if isinstance(response, dict):+ details = response.get("incomplete_details")+ if isinstance(details, dict) and details.get("reason") in _CEILING_STOP_REASONS:+ return True+ return False+++def _requested_ceiling(body: Any) -> int | None:+ """The output-token ceiling the request asked for, or ``None``."""+ if not isinstance(body, dict):+ return None+ for field in _CEILING_FIELDS:+ value = body.get(field)+ # bool is an int subclass and would sail through as 0/1.+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:+ return value+ return None+++def estimate_output_tokens(*, sse_text: str, total_bytes: int, body: Any = None) -> tuple[int, str]:"""Return ``(tokens, source)`` for a stream with no provider usage chunk.``source`` names which rung of the ladder produced the number so the callercan log it honestly rather than implying the provider reported it:+ * ``exact_ceiling`` — the turn hit its output-token ceiling (exact)* ``estimated_text`` — counted from the generated text (good)* ``estimated_bytes`` — the raw-wire fallback (poor, last resort)++ The top rung exists because counting the stream's text is catastrophically+ wrong for exactly one shape of turn: a TOOL CALL cut off by ``max_tokens``.+ The arguments stream as ``input_json_delta`` fragments, and an upstream that+ stops mid-object drops the incomplete remainder rather than emit unparseable+ JSON — while still billing every token it generated. Measured against+ Anthropic: a turn billed 1500 output tokens left ~45 characters of+ recoverable text, which the text rung reads as ~11 tokens. A 100x+ under-count, landing on the most expensive turns there are.++ Nothing needs estimating there. The ceiling is denominated in output tokens,+ so a turn that stopped *because* it hit the ceiling produced exactly that+ many. ``body`` is the provider request as sent, so the value read here is+ the ceiling the upstream actually enforced — not whatever the client+ originally asked for before the proxy's own levers touched it."""+ if _hit_output_ceiling(sse_text):+ ceiling = _requested_ceiling(body)+ if ceiling is not None:+ return ceiling, "exact_ceiling"+text = extract_stream_text(sse_text)if text:# At least one token for any non-empty answer: integer division woulddiff --git a/tests/test_stream_output_tokens.py b/tests/test_stream_output_tokens.pyindex 0c4dbafe0..cc3760906 100644--- a/tests/test_stream_output_tokens.py+++ b/tests/test_stream_output_tokens.py@@ -169,3 +169,116 @@ def test_falls_back_to_bytes_when_no_text_is_recoverable() -> None:def test_negative_or_zero_bytes_are_safe() -> None:assert estimate_output_tokens(sse_text="", total_bytes=0) == (0, "estimated_bytes")assert estimate_output_tokens(sse_text="", total_bytes=-5) == (0, "estimated_bytes")+++# --------------------------------------------------------------------------- #+# A turn stopped by its output ceiling is counted exactly, never estimated.+#+# Measured against Anthropic while probing echo ratios: a Write tool call cut+# off by ``max_tokens`` billed 1500 output tokens and left ~45 characters of+# recoverable stream text. Tool arguments stream as ``input_json_delta``+# fragments, and an upstream that stops mid-object drops the incomplete+# remainder rather than emit unparseable JSON. The text rung reads that as ~11+# tokens: a 100x under-count, on the most expensive turns there are. Repeated+# at two ceilings, both exact:+#+# max_tokens=1500 -> usage.output_tokens=1500+# max_tokens=4096 -> usage.output_tokens=4096+#+# Which is the fix: the ceiling is denominated in output tokens, so a turn that+# stopped because it hit the ceiling produced exactly that many.+# --------------------------------------------------------------------------- #+++def _truncated_anthropic_tool_call() -> str:+ """An Anthropic tool call whose arguments were cut off mid-JSON."""+ return _sse(+ {+ "type": "content_block_start",+ "index": 0,+ "content_block": {"type": "tool_use", "id": "toolu_1", "name": "Write", "input": {}},+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "input_json_delta", "partial_json": '{"path"'},+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "input_json_delta", "partial_json": ': "a/b.py"'},+ },+ {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}},+ {"type": "message_stop"},+ done=False,+ )+++def test_truncated_tool_call_is_counted_from_the_ceiling_not_the_text() -> None:+ sse = _truncated_anthropic_tool_call()+ tokens, source = estimate_output_tokens(+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_tokens": 1500}+ )+ assert source == "exact_ceiling"+ assert tokens == 1500++ # What the text rung would have said, and why it cannot be trusted here.+ text_tokens, text_source = estimate_output_tokens(sse_text=sse, total_bytes=len(sse.encode()))+ assert text_source == "estimated_text"+ assert text_tokens < 50+++def test_openai_chat_length_finish_reason_uses_max_completion_tokens() -> None:+ sse = _sse(+ _chat_delta("partial answer that ran out of room"),+ {"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}]},+ )+ tokens, source = estimate_output_tokens(+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_completion_tokens": 900}+ )+ assert (tokens, source) == (900, "exact_ceiling")+++def test_openai_responses_incomplete_uses_max_output_tokens() -> None:+ sse = _sse(+ {"type": "response.output_text.delta", "delta": "cut off"},+ {+ "type": "response.incomplete",+ "response": {"incomplete_details": {"reason": "max_output_tokens"}},+ },+ )+ tokens, source = estimate_output_tokens(+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_output_tokens": 256}+ )+ assert (tokens, source) == (256, "exact_ceiling")+++def test_a_turn_that_finished_normally_still_counts_its_text() -> None:+ """The new rung must not swallow the common case."""+ sse = _sse(+ _chat_delta("a complete answer "),+ _chat_delta("that stopped on its own"),+ {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},+ )+ tokens, source = estimate_output_tokens(+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_tokens": 4096}+ )+ assert source == "estimated_text"+ assert tokens == len(extract_stream_text(sse)) // TEXT_CHARS_PER_TOKEN+++def test_ceiling_rung_falls_through_when_the_body_does_not_carry_one() -> None:+ """A truncated turn with no usable ceiling must degrade, not guess."""+ sse = _truncated_anthropic_tool_call()+ for body in (None, {}, {"max_tokens": 0}, {"max_tokens": "1500"}):+ _, source = estimate_output_tokens(sse_text=sse, total_bytes=len(sse.encode()), body=body)+ assert source == "estimated_text", body+++def test_boolean_ceiling_is_not_mistaken_for_a_token_count() -> None:+ """``bool`` is an ``int`` subclass; ``max_tokens: True`` is not 1 token."""+ sse = _truncated_anthropic_tool_call()+ _, source = estimate_output_tokens(+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_tokens": True}+ )+ assert source == "estimated_text"