Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): count a ceiling-stopped stream exactly instead of estimating its text (#3771)

Tejas Choprab36e59a89370
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 03a698fe7..bc74fbb42 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -969,13 +969,20 @@ class StreamingMixin:
output_tokens, output_tokens_source = estimate_output_tokens(
sse_text=full_sse_data,
total_bytes=stream_state["total_bytes"],
+ # The provider body as sent, so a turn stopped by its output
+ # ceiling can be counted exactly instead of estimated. Matters
+ # most for a tool call truncated mid-arguments, whose dropped
+ # JSON leaves almost no text to count.
+ body=body,
)
# Name the actual basis. The old message always said "from N bytes"
# even though that is now only true for the fallback rung, and an
# operator reading it needs to know which estimate they are looking
# at before trusting the number.
basis = (
- "counted from stream text"
+ "exact: turn hit its output-token ceiling"
+ if output_tokens_source == "exact_ceiling"
+ else "counted from stream text"
if output_tokens_source == "estimated_text"
else f"estimated from {stream_state['total_bytes']} raw SSE bytes"
)
diff --git a/headroom/proxy/stream_output_tokens.py b/headroom/proxy/stream_output_tokens.py
index 96e410581..965f7b7ea 100644
--- a/headroom/proxy/stream_output_tokens.py
+++ b/headroom/proxy/stream_output_tokens.py
@@ -123,15 +123,88 @@ def extract_stream_text(sse: str) -> str:
return "".join(parts)
-def estimate_output_tokens(*, sse_text: str, total_bytes: int) -> tuple[int, str]:
+# Terminal stop reasons that mean "the output-token ceiling cut this turn off".
+# Anthropic messages say ``max_tokens``; OpenAI chat completions say ``length``.
+_CEILING_STOP_REASONS = frozenset({"max_tokens", "length", "max_output_tokens"})
+
+# Request fields that carry that ceiling, per surface. All three are denominated
+# in OUTPUT tokens (reasoning included, where a model emits it), which is what
+# makes the ceiling an exact count rather than a lower bound.
+_CEILING_FIELDS = ("max_tokens", "max_completion_tokens", "max_output_tokens")
+
+
+def _hit_output_ceiling(sse_text: str) -> bool:
+ """Whether the stream ended because it hit the request's output ceiling."""
+ for obj in _iter_sse_payloads(sse_text):
+ if not isinstance(obj, dict):
+ continue
+
+ # OpenAI chat completions — choices[].finish_reason
+ choices = obj.get("choices")
+ if isinstance(choices, list):
+ for choice in choices:
+ if isinstance(choice, dict) and choice.get("finish_reason") in (
+ _CEILING_STOP_REASONS
+ ):
+ return True
+ continue
+
+ # Anthropic messages — message_delta.delta.stop_reason
+ delta = obj.get("delta")
+ if isinstance(delta, dict) and delta.get("stop_reason") in _CEILING_STOP_REASONS:
+ return True
+
+ # OpenAI responses — response.incomplete_details.reason
+ response = obj.get("response")
+ if isinstance(response, dict):
+ details = response.get("incomplete_details")
+ if isinstance(details, dict) and details.get("reason") in _CEILING_STOP_REASONS:
+ return True
+ return False
+
+
+def _requested_ceiling(body: Any) -> int | None:
+ """The output-token ceiling the request asked for, or ``None``."""
+ if not isinstance(body, dict):
+ return None
+ for field in _CEILING_FIELDS:
+ value = body.get(field)
+ # bool is an int subclass and would sail through as 0/1.
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ return value
+ return None
+
+
+def estimate_output_tokens(*, sse_text: str, total_bytes: int, body: Any = None) -> tuple[int, str]:
"""Return ``(tokens, source)`` for a stream with no provider usage chunk.
``source`` names which rung of the ladder produced the number so the caller
can log it honestly rather than implying the provider reported it:
+ * ``exact_ceiling`` — the turn hit its output-token ceiling (exact)
* ``estimated_text`` — counted from the generated text (good)
* ``estimated_bytes`` — the raw-wire fallback (poor, last resort)
+
+ The top rung exists because counting the stream's text is catastrophically
+ wrong for exactly one shape of turn: a TOOL CALL cut off by ``max_tokens``.
+ The arguments stream as ``input_json_delta`` fragments, and an upstream that
+ stops mid-object drops the incomplete remainder rather than emit unparseable
+ JSON — while still billing every token it generated. Measured against
+ Anthropic: a turn billed 1500 output tokens left ~45 characters of
+ recoverable text, which the text rung reads as ~11 tokens. A 100x
+ under-count, landing on the most expensive turns there are.
+
+ Nothing needs estimating there. The ceiling is denominated in output tokens,
+ so a turn that stopped *because* it hit the ceiling produced exactly that
+ many. ``body`` is the provider request as sent, so the value read here is
+ the ceiling the upstream actually enforced — not whatever the client
+ originally asked for before the proxy's own levers touched it.
"""
+ if _hit_output_ceiling(sse_text):
+ ceiling = _requested_ceiling(body)
+ if ceiling is not None:
+ return ceiling, "exact_ceiling"
+
text = extract_stream_text(sse_text)
if text:
# At least one token for any non-empty answer: integer division would
diff --git a/tests/test_stream_output_tokens.py b/tests/test_stream_output_tokens.py
index 0c4dbafe0..cc3760906 100644
--- a/tests/test_stream_output_tokens.py
+++ b/tests/test_stream_output_tokens.py
@@ -169,3 +169,116 @@ def test_falls_back_to_bytes_when_no_text_is_recoverable() -> None:
def test_negative_or_zero_bytes_are_safe() -> None:
assert estimate_output_tokens(sse_text="", total_bytes=0) == (0, "estimated_bytes")
assert estimate_output_tokens(sse_text="", total_bytes=-5) == (0, "estimated_bytes")
+
+
+# --------------------------------------------------------------------------- #
+# A turn stopped by its output ceiling is counted exactly, never estimated.
+#
+# Measured against Anthropic while probing echo ratios: a Write tool call cut
+# off by ``max_tokens`` billed 1500 output tokens and left ~45 characters of
+# recoverable stream text. Tool arguments stream as ``input_json_delta``
+# fragments, and an upstream that stops mid-object drops the incomplete
+# remainder rather than emit unparseable JSON. The text rung reads that as ~11
+# tokens: a 100x under-count, on the most expensive turns there are. Repeated
+# at two ceilings, both exact:
+#
+# max_tokens=1500 -> usage.output_tokens=1500
+# max_tokens=4096 -> usage.output_tokens=4096
+#
+# Which is the fix: the ceiling is denominated in output tokens, so a turn that
+# stopped because it hit the ceiling produced exactly that many.
+# --------------------------------------------------------------------------- #
+
+
+def _truncated_anthropic_tool_call() -> str:
+ """An Anthropic tool call whose arguments were cut off mid-JSON."""
+ return _sse(
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {"type": "tool_use", "id": "toolu_1", "name": "Write", "input": {}},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "input_json_delta", "partial_json": '{"path"'},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "input_json_delta", "partial_json": ': "a/b.py"'},
+ },
+ {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}},
+ {"type": "message_stop"},
+ done=False,
+ )
+
+
+def test_truncated_tool_call_is_counted_from_the_ceiling_not_the_text() -> None:
+ sse = _truncated_anthropic_tool_call()
+ tokens, source = estimate_output_tokens(
+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_tokens": 1500}
+ )
+ assert source == "exact_ceiling"
+ assert tokens == 1500
+
+ # What the text rung would have said, and why it cannot be trusted here.
+ text_tokens, text_source = estimate_output_tokens(sse_text=sse, total_bytes=len(sse.encode()))
+ assert text_source == "estimated_text"
+ assert text_tokens < 50
+
+
+def test_openai_chat_length_finish_reason_uses_max_completion_tokens() -> None:
+ sse = _sse(
+ _chat_delta("partial answer that ran out of room"),
+ {"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}]},
+ )
+ tokens, source = estimate_output_tokens(
+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_completion_tokens": 900}
+ )
+ assert (tokens, source) == (900, "exact_ceiling")
+
+
+def test_openai_responses_incomplete_uses_max_output_tokens() -> None:
+ sse = _sse(
+ {"type": "response.output_text.delta", "delta": "cut off"},
+ {
+ "type": "response.incomplete",
+ "response": {"incomplete_details": {"reason": "max_output_tokens"}},
+ },
+ )
+ tokens, source = estimate_output_tokens(
+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_output_tokens": 256}
+ )
+ assert (tokens, source) == (256, "exact_ceiling")
+
+
+def test_a_turn_that_finished_normally_still_counts_its_text() -> None:
+ """The new rung must not swallow the common case."""
+ sse = _sse(
+ _chat_delta("a complete answer "),
+ _chat_delta("that stopped on its own"),
+ {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
+ )
+ tokens, source = estimate_output_tokens(
+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_tokens": 4096}
+ )
+ assert source == "estimated_text"
+ assert tokens == len(extract_stream_text(sse)) // TEXT_CHARS_PER_TOKEN
+
+
+def test_ceiling_rung_falls_through_when_the_body_does_not_carry_one() -> None:
+ """A truncated turn with no usable ceiling must degrade, not guess."""
+ sse = _truncated_anthropic_tool_call()
+ for body in (None, {}, {"max_tokens": 0}, {"max_tokens": "1500"}):
+ _, source = estimate_output_tokens(sse_text=sse, total_bytes=len(sse.encode()), body=body)
+ assert source == "estimated_text", body
+
+
+def test_boolean_ceiling_is_not_mistaken_for_a_token_count() -> None:
+ """``bool`` is an ``int`` subclass; ``max_tokens: True`` is not 1 token."""
+ sse = _truncated_anthropic_tool_call()
+ _, source = estimate_output_tokens(
+ sse_text=sse, total_bytes=len(sse.encode()), body={"max_tokens": True}
+ )
+ assert source == "estimated_text"