Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(pipeline): put per-request saved tokens on OutcomeSnapshot (#4231)

Tejas Chopra8cedf7be4dec
diff --git a/headroom/pipeline.py b/headroom/pipeline.py
index 491dd5ef9..55186496f 100644
--- a/headroom/pipeline.py
+++ b/headroom/pipeline.py
@@ -77,6 +77,15 @@ class OutcomeSnapshot:
cache_write_tokens: int = 0
turn_index: int = 0
transforms_applied: tuple[str, ...] = ()
+ #: Input tokens Headroom removed from THIS request's payload (the wire
+ #: figure PERF logs as ``tok_saved``), message compression only.
+ tokens_saved: int = 0
+ #: The share of ``tokens_saved`` new to this conversation, which is what
+ #: running totals book (PERF ``tok_novel``). The proxy funnel always sets
+ #: both; they differ only on ``/v1/responses``, where the whole transcript
+ #: is re-sent and recompressed every turn, and ``tokens_saved -
+ #: novel_tokens_saved`` is the re-sent history removed again.
+ novel_tokens_saved: int = 0
@property
def truncated(self) -> bool:
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index 9b53b69b6..3503d710f 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -571,44 +571,6 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
await handler.metrics.record_failed(provider=outcome.provider)
return
- # Outcome stage: hand the meter's reading to any pipeline extension that
- # adapts to it — a ceiling that was hit, an effort setting that did or did
- # not pay off. The snapshot is frozen, so an extension can learn from the
- # measurement without being able to rewrite it: extensions declare what
- # they did, the core records what happened, and attribution is the core's
- # arithmetic over both.
- #
- # Best-effort and last in line: a handler without a pipeline manager, or an
- # extension that raises, must not affect the response or the stats below.
- _pipeline = getattr(handler, "pipeline_extensions", None)
- if _pipeline is not None and getattr(_pipeline, "enabled", False):
- try:
- from headroom.pipeline import OutcomeSnapshot, PipelineStage
-
- _pipeline.emit(
- PipelineStage.OUTCOME_OBSERVED,
- operation="proxy.outcome",
- request_id=outcome.request_id,
- provider=outcome.provider,
- model=outcome.model,
- outcome=OutcomeSnapshot(
- request_id=outcome.request_id,
- provider=outcome.provider,
- model=outcome.model,
- output_tokens=outcome.output_tokens,
- thinking_tokens=outcome.thinking_tokens,
- thinking_inferred=outcome.thinking_inferred,
- stop_reason=outcome.stop_reason,
- input_tokens=outcome.provider_input_tokens or outcome.optimized_tokens,
- cache_read_tokens=outcome.cache_read_tokens,
- cache_write_tokens=outcome.cache_write_tokens,
- turn_index=outcome.turn_index,
- transforms_applied=tuple(str(t) for t in (outcome.transforms_applied or ())),
- ),
- )
- except Exception: # noqa: BLE001 - telemetry must never break a response
- logger.debug("OUTCOME_OBSERVED emit failed", exc_info=True)
-
# Output-shaping savings ledger (counterfactual estimator). The shaper
# tags each request's (arm, stratum) onto ``transforms_applied``; feed the
# observed output tokens to the recorder so it can produce an honest
@@ -653,6 +615,49 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
if novel_tokens_saved is None:
novel_tokens_saved = outcome.tokens_saved
+ # Outcome stage: hand the meter's reading to any pipeline extension that
+ # adapts to it — a ceiling that was hit, an effort setting that did or did
+ # not pay off. The snapshot is frozen, so an extension can learn from the
+ # measurement without being able to rewrite it: extensions declare what
+ # they did, the core records what happened, and attribution is the core's
+ # arithmetic over both.
+ #
+ # Emitted after the novel share is known, so an extension reads the same
+ # per-request savings the running totals book instead of parsing PERF lines.
+ #
+ # Best-effort: a handler without a pipeline manager, or an extension that
+ # raises, must not affect the response or the stats below.
+ _pipeline = getattr(handler, "pipeline_extensions", None)
+ if _pipeline is not None and getattr(_pipeline, "enabled", False):
+ try:
+ from headroom.pipeline import OutcomeSnapshot, PipelineStage
+
+ _pipeline.emit(
+ PipelineStage.OUTCOME_OBSERVED,
+ operation="proxy.outcome",
+ request_id=outcome.request_id,
+ provider=outcome.provider,
+ model=outcome.model,
+ outcome=OutcomeSnapshot(
+ request_id=outcome.request_id,
+ provider=outcome.provider,
+ model=outcome.model,
+ output_tokens=outcome.output_tokens,
+ thinking_tokens=outcome.thinking_tokens,
+ thinking_inferred=outcome.thinking_inferred,
+ stop_reason=outcome.stop_reason,
+ input_tokens=outcome.provider_input_tokens or outcome.optimized_tokens,
+ cache_read_tokens=outcome.cache_read_tokens,
+ cache_write_tokens=outcome.cache_write_tokens,
+ turn_index=outcome.turn_index,
+ transforms_applied=tuple(str(t) for t in (outcome.transforms_applied or ())),
+ tokens_saved=outcome.tokens_saved,
+ novel_tokens_saved=novel_tokens_saved,
+ ),
+ )
+ except Exception: # noqa: BLE001 - telemetry must never break a response
+ logger.debug("OUTCOME_OBSERVED emit failed", exc_info=True)
+
# Tool-schema savings (deferral + turn-hook tool shrink) live in per-request
# tags and never move tok_before/after; aggregate them into Metrics so the
# session summary / cost summary / all-layers total can surface the layer.
diff --git a/tests/test_proxy/test_conversation_savings.py b/tests/test_proxy/test_conversation_savings.py
index 0d75e4323..52a826c04 100644
--- a/tests/test_proxy/test_conversation_savings.py
+++ b/tests/test_proxy/test_conversation_savings.py
@@ -337,3 +337,29 @@ def test_server_side_state_means_incremental_input() -> None:
savings_conversation_key(_responses_body("more", conversation={"id": "conv_abc"})) is None
)
assert savings_conversation_key({"model": "gpt-6-astra", "messages": []}) is None
+
+
+async def test_pipeline_extensions_read_both_figures_from_the_snapshot() -> None:
+ """An extension sees the wire figure and the novel share the totals book,
+ so it never has to parse PERF lines to tell re-sent history from new."""
+ from headroom.pipeline import PipelineExtensionManager, PipelineStage
+
+ seen: list[Any] = []
+
+ class _Recorder:
+ def on_pipeline_event(self, event: Any) -> None:
+ if event.stage is PipelineStage.OUTCOME_OBSERVED:
+ seen.append(event.outcome)
+
+ handler = _Handler()
+ handler.pipeline_extensions = PipelineExtensionManager(extensions=[_Recorder()], discover=False)
+ await emit_request_outcome(
+ handler, _responses_outcome(tokens_saved=62_806, conversation_tokens_saved=62_806)
+ )
+ await emit_request_outcome(handler, _responses_outcome())
+
+ assert [(s.tokens_saved, s.novel_tokens_saved) for s in seen] == [
+ (62_806, 62_806),
+ (76_020, 13_214),
+ ]