Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(pipeline): put per-request saved tokens on OutcomeSnapshot (#4231)
Tejas Chopra8cedf7be4dec
diff --git a/headroom/pipeline.py b/headroom/pipeline.pyindex 491dd5ef9..55186496f 100644--- a/headroom/pipeline.py+++ b/headroom/pipeline.py@@ -77,6 +77,15 @@ class OutcomeSnapshot:cache_write_tokens: int = 0turn_index: int = 0transforms_applied: tuple[str, ...] = ()+ #: Input tokens Headroom removed from THIS request's payload (the wire+ #: figure PERF logs as ``tok_saved``), message compression only.+ tokens_saved: int = 0+ #: The share of ``tokens_saved`` new to this conversation, which is what+ #: running totals book (PERF ``tok_novel``). The proxy funnel always sets+ #: both; they differ only on ``/v1/responses``, where the whole transcript+ #: is re-sent and recompressed every turn, and ``tokens_saved -+ #: novel_tokens_saved`` is the re-sent history removed again.+ novel_tokens_saved: int = 0@propertydef truncated(self) -> bool:diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 9b53b69b6..3503d710f 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -571,44 +571,6 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:await handler.metrics.record_failed(provider=outcome.provider)return- # Outcome stage: hand the meter's reading to any pipeline extension that- # adapts to it — a ceiling that was hit, an effort setting that did or did- # not pay off. The snapshot is frozen, so an extension can learn from the- # measurement without being able to rewrite it: extensions declare what- # they did, the core records what happened, and attribution is the core's- # arithmetic over both.- #- # Best-effort and last in line: a handler without a pipeline manager, or an- # extension that raises, must not affect the response or the stats below.- _pipeline = getattr(handler, "pipeline_extensions", None)- if _pipeline is not None and getattr(_pipeline, "enabled", False):- try:- from headroom.pipeline import OutcomeSnapshot, PipelineStage-- _pipeline.emit(- PipelineStage.OUTCOME_OBSERVED,- operation="proxy.outcome",- request_id=outcome.request_id,- provider=outcome.provider,- model=outcome.model,- outcome=OutcomeSnapshot(- request_id=outcome.request_id,- provider=outcome.provider,- model=outcome.model,- output_tokens=outcome.output_tokens,- thinking_tokens=outcome.thinking_tokens,- thinking_inferred=outcome.thinking_inferred,- stop_reason=outcome.stop_reason,- input_tokens=outcome.provider_input_tokens or outcome.optimized_tokens,- cache_read_tokens=outcome.cache_read_tokens,- cache_write_tokens=outcome.cache_write_tokens,- turn_index=outcome.turn_index,- transforms_applied=tuple(str(t) for t in (outcome.transforms_applied or ())),- ),- )- except Exception: # noqa: BLE001 - telemetry must never break a response- logger.debug("OUTCOME_OBSERVED emit failed", exc_info=True)-# Output-shaping savings ledger (counterfactual estimator). The shaper# tags each request's (arm, stratum) onto ``transforms_applied``; feed the# observed output tokens to the recorder so it can produce an honest@@ -653,6 +615,49 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:if novel_tokens_saved is None:novel_tokens_saved = outcome.tokens_saved+ # Outcome stage: hand the meter's reading to any pipeline extension that+ # adapts to it — a ceiling that was hit, an effort setting that did or did+ # not pay off. The snapshot is frozen, so an extension can learn from the+ # measurement without being able to rewrite it: extensions declare what+ # they did, the core records what happened, and attribution is the core's+ # arithmetic over both.+ #+ # Emitted after the novel share is known, so an extension reads the same+ # per-request savings the running totals book instead of parsing PERF lines.+ #+ # Best-effort: a handler without a pipeline manager, or an extension that+ # raises, must not affect the response or the stats below.+ _pipeline = getattr(handler, "pipeline_extensions", None)+ if _pipeline is not None and getattr(_pipeline, "enabled", False):+ try:+ from headroom.pipeline import OutcomeSnapshot, PipelineStage++ _pipeline.emit(+ PipelineStage.OUTCOME_OBSERVED,+ operation="proxy.outcome",+ request_id=outcome.request_id,+ provider=outcome.provider,+ model=outcome.model,+ outcome=OutcomeSnapshot(+ request_id=outcome.request_id,+ provider=outcome.provider,+ model=outcome.model,+ output_tokens=outcome.output_tokens,+ thinking_tokens=outcome.thinking_tokens,+ thinking_inferred=outcome.thinking_inferred,+ stop_reason=outcome.stop_reason,+ input_tokens=outcome.provider_input_tokens or outcome.optimized_tokens,+ cache_read_tokens=outcome.cache_read_tokens,+ cache_write_tokens=outcome.cache_write_tokens,+ turn_index=outcome.turn_index,+ transforms_applied=tuple(str(t) for t in (outcome.transforms_applied or ())),+ tokens_saved=outcome.tokens_saved,+ novel_tokens_saved=novel_tokens_saved,+ ),+ )+ except Exception: # noqa: BLE001 - telemetry must never break a response+ logger.debug("OUTCOME_OBSERVED emit failed", exc_info=True)+# Tool-schema savings (deferral + turn-hook tool shrink) live in per-request# tags and never move tok_before/after; aggregate them into Metrics so the# session summary / cost summary / all-layers total can surface the layer.diff --git a/tests/test_proxy/test_conversation_savings.py b/tests/test_proxy/test_conversation_savings.pyindex 0d75e4323..52a826c04 100644--- a/tests/test_proxy/test_conversation_savings.py+++ b/tests/test_proxy/test_conversation_savings.py@@ -337,3 +337,29 @@ def test_server_side_state_means_incremental_input() -> None:savings_conversation_key(_responses_body("more", conversation={"id": "conv_abc"})) is None)assert savings_conversation_key({"model": "gpt-6-astra", "messages": []}) is None+++async def test_pipeline_extensions_read_both_figures_from_the_snapshot() -> None:+ """An extension sees the wire figure and the novel share the totals book,+ so it never has to parse PERF lines to tell re-sent history from new."""+ from headroom.pipeline import PipelineExtensionManager, PipelineStage++ seen: list[Any] = []++ class _Recorder:+ def on_pipeline_event(self, event: Any) -> None:+ if event.stage is PipelineStage.OUTCOME_OBSERVED:+ seen.append(event.outcome)++ handler = _Handler()+ handler.pipeline_extensions = PipelineExtensionManager(extensions=[_Recorder()], discover=False)+ await emit_request_outcome(+ handler, _responses_outcome(tokens_saved=62_806, conversation_tokens_saved=62_806)+ )+ await emit_request_outcome(handler, _responses_outcome())++ assert [(s.tokens_saved, s.novel_tokens_saved) for s in seen] == [+ (62_806, 62_806),+ (76_020, 13_214),+ ]