Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: B4 — token validation gate + per-content-type byte thresholds
chopratejasb3b3feff6f29
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.jsonindex dba9eb042..f6df822a2 100644--- a/.claude-plugin/marketplace.json+++ b/.claude-plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.20.12"+ "version": "0.20.13"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.20.12",+ "version": "0.20.13","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.jsonindex dba9eb042..f6df822a2 100644--- a/.github/plugin/marketplace.json+++ b/.github/plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.20.12"+ "version": "0.20.13"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.20.12",+ "version": "0.20.13","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/crates/headroom-core/src/transforms/live_zone.rs b/crates/headroom-core/src/transforms/live_zone.rsindex ba1f6cbb9..730e7b516 100644--- a/crates/headroom-core/src/transforms/live_zone.rs+++ b/crates/headroom-core/src/transforms/live_zone.rs@@ -106,6 +106,7 @@ use super::diff_compressor::{DiffCompressor, DiffCompressorConfig};use super::log_compressor::{LogCompressor, LogCompressorConfig};use super::search_compressor::{SearchCompressor, SearchCompressorConfig};use super::smart_crusher::{SmartCrusher, SmartCrusherConfig};+use crate::tokenizer::get_tokenizer;// ─── Tunable constants (no magic numbers in the dispatch logic) ────────@@ -126,6 +127,63 @@ const EMPTY_QUERY: &str = "";/// the OSS-default behaviour ("no bias").const DEFAULT_BIAS: f64 = 0.0;+/// Default model name handed to the tokenizer registry when the proxy+/// could not extract `body["model"]`. Matches the most-common+/// production Claude model — chars-per-token estimator for `claude-*`+/// is calibrated to 3.5 cpt; using a non-Claude model here would+/// silently pick a different estimator density. PR-F3 will plumb the+/// actual model from `body["model"]`; PR-B4 just establishes the+/// signature.+pub const DEFAULT_MODEL: &str = "claude-3-5-sonnet-20241022";++// ─── Per-content-type byte thresholds ──────────────────────────────────+//+// Below these byte sizes the dispatcher does not even attempt+// compression — the per-block overhead (tokenizer count, dispatcher+// bookkeeping, log lines) costs more than the marginal token savings,+// and tiny inputs almost never compress at all.+//+// Sourced from the spec (`REALIGNMENT/04-phase-B-live-zone.md::PR-B4`).+// Pinned as `const` rather than a hard-coded `match` so the values are+// grep-able and reviewable in one place.++/// JSON-array tool_results below this size route to no-op (1 KiB).+const THRESHOLD_JSON_ARRAY: usize = 1024;+/// Build / log output below this size routes to no-op (512 B). Logs+/// are the most repetitive content type so the threshold is the+/// lowest of the bunch.+const THRESHOLD_BUILD_OUTPUT: usize = 512;+/// Search-result blocks below this size route to no-op (1 KiB).+const THRESHOLD_SEARCH_RESULTS: usize = 1024;+/// Git-diff blocks below this size route to no-op (1 KiB).+const THRESHOLD_GIT_DIFF: usize = 1024;+/// Source-code blocks below this size route to no-op (2 KiB). Pinned+/// for the future Rust code-compressor port — currently unused+/// because `ContentType::SourceCode` short-circuits to no-op above+/// the dispatch (see `dispatch_compressor`).+const THRESHOLD_SOURCE_CODE: usize = 2048;+/// Plain-text blocks below this size route to no-op (5 KiB). Pinned+/// for the future Kompress wiring (PR-B7 follow-up); currently unused.+const THRESHOLD_PLAIN_TEXT: usize = 5120;+/// HTML blocks have no compressor; threshold matches plain text so+/// when an HTML compressor lands the value is already pinned.+const THRESHOLD_HTML: usize = 5120;++/// Map a content type to its byte threshold. Returning `usize` rather+/// than an `Option` because every variant has a sensible default;+/// `Html` is a no-op anyway so the threshold check never fires.+fn threshold_for(content_type: ContentType) -> usize {+ match content_type {+ ContentType::JsonArray => THRESHOLD_JSON_ARRAY,+ ContentType::BuildOutput => THRESHOLD_BUILD_OUTPUT,+ ContentType::SearchResults => THRESHOLD_SEARCH_RESULTS,+ ContentType::GitDiff => THRESHOLD_GIT_DIFF,+ ContentType::SourceCode => THRESHOLD_SOURCE_CODE,+ ContentType::PlainText => THRESHOLD_PLAIN_TEXT,+ ContentType::Html => THRESHOLD_HTML,+ }+}+// ─── Public types ──────────────────────────────────────────────────────/// Authentication mode of the originating request. Passed through to@@ -176,8 +234,10 @@ pub enum BlockAction {/// `"source_code"`, `"html"`, `"image"`, `"unknown"`, etc.content_type: String,},- /// A compressor ran and produced a smaller output that was- /// spliced into the body.+ /// A compressor ran and produced a smaller output (in tokens, as+ /// counted by the model's tokenizer) that was spliced into the+ /// body. Both byte and token counts are reported so the proxy+ /// can log the savings ratio in either currency.Compressed {/// Identifier of the compressor (`"smart_crusher"`,/// `"log_compressor"`, ...). Static so the manifest is@@ -188,6 +248,14 @@ pub enum BlockAction {original_bytes: usize,/// Bytes of the replacement block content.compressed_bytes: usize,+ /// Tokens in the original block content (per the model's+ /// tokenizer).+ original_tokens: usize,+ /// Tokens in the replacement block content. Always strictly+ /// less than `original_tokens` for this variant — the+ /// tokenizer-validated rejection gate (PR-B4) maps the+ /// `>=` case to `RejectedNotSmaller`.+ compressed_tokens: usize,},/// A compressor was tried but failed loudly. Per project memory/// `feedback_no_silent_fallbacks.md`: surface the error in the@@ -200,9 +268,10 @@ pub enum BlockAction {/// Human-readable error string (from `Display`).error: String,},- /// A compressor ran but produced output >= original. Cache- /// safety + "don't make it worse" → keep the original. PR-B4- /// swaps this byte-length proxy for a real tokenizer count.+ /// A compressor ran but produced output that did not shrink the+ /// token count. Cache safety + "don't make it worse" → keep the+ /// original. PR-B4 wired the tokenizer-validated check; both+ /// byte and token counts are reported for observability.RejectedNotSmaller {/// Identifier of the compressor that was rejected.strategy: &'static str,@@ -210,6 +279,25 @@ pub enum BlockAction {original_bytes: usize,/// Would-be compressed-block-content size, bytes.compressed_bytes: usize,+ /// Original block-content size, tokens.+ original_tokens: usize,+ /// Would-be compressed-block-content size, tokens. Always+ /// `>= original_tokens` (otherwise this would be+ /// `Compressed`).+ compressed_tokens: usize,+ },+ /// The block content was below the per-content-type byte+ /// threshold; no compressor was invoked. The dispatcher does+ /// not even spin up the tokenizer for these — they're below the+ /// per-call overhead so the marginal savings are negative.+ BelowByteThreshold {+ /// Detected content type — string tag matches+ /// `ContentType::as_str`.+ content_type: &'static str,+ /// Bytes in the block content.+ byte_count: usize,+ /// Threshold (in bytes) the content failed to clear.+ threshold_bytes: usize,},/// Block type is intentionally outside the live zone (e.g./// `tool_use` → cache hot zone) and is excluded from dispatch.@@ -388,6 +476,10 @@ fn diff_compressor() -> &'static DiffCompressor {/// - `frozen_message_count`: hot-zone floor. Indices `< floor` are/// excluded from dispatch./// - `_auth_mode`: reserved for PR-F2; B3 ignores it.+/// - `model`: the upstream model name (e.g. `"claude-3-5-sonnet-20241022"`).+/// Routes the tokenizer registry to the right backend for the+/// per-block token-count check (PR-B4). Pass [`DEFAULT_MODEL`] when+/// the proxy could not extract `body["model"]`.////// # Returns///@@ -400,6 +492,7 @@ pub fn compress_anthropic_live_zone(body_raw: &[u8],frozen_message_count: usize,_auth_mode: AuthMode,+ model: &str,) -> Result<LiveZoneOutcome, LiveZoneError> {let parsed: Value = serde_json::from_slice(body_raw).map_err(LiveZoneError::BodyNotJson)?;let messages = parsed@@ -457,6 +550,13 @@ pub fn compress_anthropic_live_zone(let mut block_outcomes: Vec<BlockOutcome> = Vec::with_capacity(plan.len());let mut replacements: Vec<Replacement> = Vec::new();+ // One tokenizer per request — `get_tokenizer` is cheap (it+ // returns a `Box<dyn Tokenizer>` over either a tiktoken-rs handle+ // or an estimator) but counting once per block is a hot path.+ // PR-B4 only invokes the tokenizer on blocks that actually+ // produced compressed output; the byte-threshold gate filters+ // sub-threshold content first.+ let tokenizer = get_tokenizer(model);for slot in plan {let outcome = match slot.kind {@@ -474,66 +574,16 @@ pub fn compress_anthropic_live_zone(content_byte_range,} => {let detected = detect_content_type(&content_text);- let _detected_tag = detected.content_type.as_str();- let outcome: BlockOutcome =- match dispatch_compressor(&content_text, detected.content_type) {- DispatchResult::NoOp { content_type } => BlockOutcome {- message_index: target_idx,- block_index: Some(slot.block_index),- block_type,- action: BlockAction::NoCompressionApplied {- content_type: content_type.to_string(),- },- },- DispatchResult::Compressed {- strategy,- compressed,- } => {- let original_bytes = content_text.len();- let compressed_bytes = compressed.len();- if compressed_bytes >= original_bytes {- // Byte-length gate (PR-B4 will replace- // with a token-count gate). Without this- // the dispatcher could ship a "compressed"- // body bigger than the input.- BlockOutcome {- message_index: target_idx,- block_index: Some(slot.block_index),- block_type,- action: BlockAction::RejectedNotSmaller {- strategy,- original_bytes,- compressed_bytes,- },- }- } else {- // Encode replacement as a JSON string and- // record it for the splice.- let replacement_bytes = serde_json::to_vec(&compressed)- .expect("string is always JSON-encodable");- replacements.push(Replacement {- range: content_byte_range,- replacement: replacement_bytes,- });- BlockOutcome {- message_index: target_idx,- block_index: Some(slot.block_index),- block_type,- action: BlockAction::Compressed {- strategy,- original_bytes,- compressed_bytes,- },- }- }- }- DispatchResult::Error { strategy, error } => BlockOutcome {- message_index: target_idx,- block_index: Some(slot.block_index),- block_type,- action: BlockAction::CompressorError { strategy, error },- },- };+ let outcome: BlockOutcome = compress_one_block(+ &content_text,+ detected.content_type,+ content_byte_range,+ target_idx,+ Some(slot.block_index),+ block_type,+ tokenizer.as_ref(),+ &mut replacements,+ );outcome}SlotKind::StringContent {@@ -541,58 +591,16 @@ pub fn compress_anthropic_live_zone(content_byte_range,} => {let detected = detect_content_type(&content_text);- match dispatch_compressor(&content_text, detected.content_type) {- DispatchResult::NoOp { content_type } => BlockOutcome {- message_index: target_idx,- block_index: None,- block_type: "string_content".to_string(),- action: BlockAction::NoCompressionApplied {- content_type: content_type.to_string(),- },- },- DispatchResult::Compressed {- strategy,- compressed,- } => {- let original_bytes = content_text.len();- let compressed_bytes = compressed.len();- if compressed_bytes >= original_bytes {- BlockOutcome {- message_index: target_idx,- block_index: None,- block_type: "string_content".to_string(),- action: BlockAction::RejectedNotSmaller {- strategy,- original_bytes,- compressed_bytes,- },- }- } else {- let replacement_bytes = serde_json::to_vec(&compressed)- .expect("string is always JSON-encodable");- replacements.push(Replacement {- range: content_byte_range,- replacement: replacement_bytes,- });- BlockOutcome {- message_index: target_idx,- block_index: None,- block_type: "string_content".to_string(),- action: BlockAction::Compressed {- strategy,- original_bytes,- compressed_bytes,- },- }- }- }- DispatchResult::Error { strategy, error } => BlockOutcome {- message_index: target_idx,- block_index: None,- block_type: "string_content".to_string(),- action: BlockAction::CompressorError { strategy, error },- },- }+ compress_one_block(+ &content_text,+ detected.content_type,+ content_byte_range,+ target_idx,+ None,+ "string_content".to_string(),+ tokenizer.as_ref(),+ &mut replacements,+ )}};block_outcomes.push(outcome);@@ -644,6 +652,112 @@ pub fn compress_anthropic_live_zone(// ─── Internal helpers ──────────────────────────────────────────────────+/// Per-block dispatch shared by the array-of-blocks slot and the+/// legacy string-content slot. Encapsulates the PR-B4 sequence:+///+/// 1. Per-content-type byte threshold — sub-threshold content is+/// tagged `BelowByteThreshold` and the dispatcher does not even+/// invoke a compressor.+/// 2. Type-aware dispatch (`dispatch_compressor`).+/// 3. Tokenizer-validated rejection — if `compressed_tokens >=+/// original_tokens` keep the original and tag `RejectedNotSmaller`+/// (note: tokens, not bytes, drive the gate).+/// 4. Otherwise record the replacement and tag `Compressed`.+#[allow(clippy::too_many_arguments)]+fn compress_one_block(+ content_text: &str,+ content_type: ContentType,+ content_byte_range: (usize, usize),+ message_index: usize,+ block_index: Option<usize>,+ block_type: String,+ tokenizer: &dyn crate::tokenizer::Tokenizer,+ replacements: &mut Vec<Replacement>,+) -> BlockOutcome {+ // 1. Byte-threshold gate. Empty content always falls through to+ // `dispatch_compressor` (which short-circuits on empty), so+ // only check when the slot has real bytes — this preserves+ // the existing "tool_result with no inner content" pathway.+ if !content_text.is_empty() && content_text.len() < threshold_for(content_type) {+ return BlockOutcome {+ message_index,+ block_index,+ block_type,+ action: BlockAction::BelowByteThreshold {+ content_type: content_type.as_str(),+ byte_count: content_text.len(),+ threshold_bytes: threshold_for(content_type),+ },+ };+ }++ match dispatch_compressor(content_text, content_type) {+ DispatchResult::NoOp { content_type } => BlockOutcome {+ message_index,+ block_index,+ block_type,+ action: BlockAction::NoCompressionApplied {+ content_type: content_type.to_string(),+ },+ },+ DispatchResult::Compressed {+ strategy,+ compressed,+ } => {+ let original_bytes = content_text.len();+ let compressed_bytes = compressed.len();+ // 3. Tokenizer-validated rejection. Per PR-B4 spec we+ // count both the original and compressed strings+ // using the model's tokenizer; the compression is+ // accepted only when it shrinks the token count.+ // Bytes-shrinking-but-tokens-growing happens for+ // pathological inputs (e.g. dense base64 → tokenizer+ // fragments more aggressively after a transform).+ let original_tokens = tokenizer.count_text(content_text);+ let compressed_tokens = tokenizer.count_text(&compressed);+ if compressed_tokens >= original_tokens {+ BlockOutcome {+ message_index,+ block_index,+ block_type,+ action: BlockAction::RejectedNotSmaller {+ strategy,+ original_bytes,+ compressed_bytes,+ original_tokens,+ compressed_tokens,+ },+ }+ } else {+ let replacement_bytes =+ serde_json::to_vec(&compressed).expect("string is always JSON-encodable");+ replacements.push(Replacement {+ range: content_byte_range,+ replacement: replacement_bytes,+ });+ BlockOutcome {+ message_index,+ block_index,+ block_type,+ action: BlockAction::Compressed {+ strategy,+ original_bytes,+ compressed_bytes,+ original_tokens,+ compressed_tokens,+ },+ }+ }+ }+ DispatchResult::Error { strategy, error } => BlockOutcome {+ message_index,+ block_index,+ block_type,+ action: BlockAction::CompressorError { strategy, error },+ },+ }+}+/// Walk `messages` from the back, returning the index of the latest/// `role == "user"` message. Restricted to indices `>= floor`; if/// the latest user message lies in the cache hot zone we return@@ -1120,7 +1234,7 @@ mod tests {#[test]fn empty_messages_yields_no_change() {let b = body(json!({"model": "claude", "messages": []}));- let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();match out {LiveZoneOutcome::NoChange { manifest } => {assert_eq!(manifest.messages_total, 0);@@ -1134,13 +1248,14 @@ mod tests {#[test]fn no_messages_field_errors() {let b = body(json!({"model": "claude"}));- let err = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap_err();+ let err = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap_err();assert!(matches!(err, LiveZoneError::NoMessagesArray));}#[test]fn invalid_json_errors() {- let err = compress_anthropic_live_zone(b"not json", 0, AuthMode::Payg).unwrap_err();+ let err = compress_anthropic_live_zone(b"not json", 0, AuthMode::Payg, DEFAULT_MODEL)+ .unwrap_err();assert!(matches!(err, LiveZoneError::BodyNotJson(_)));}@@ -1157,7 +1272,7 @@ mod tests {]},]}));- let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();let manifest = match &out {LiveZoneOutcome::NoChange { manifest } => manifest,LiveZoneOutcome::Modified { manifest, .. } => manifest,@@ -1183,7 +1298,7 @@ mod tests {{"role": "user", "content": [{"type": "text", "text": "second"}]},]}));- let out = compress_anthropic_live_zone(&b, 2, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 2, AuthMode::Payg, DEFAULT_MODEL).unwrap();let manifest = match &out {LiveZoneOutcome::NoChange { manifest } => manifest,_ => panic!("expected NoChange"),@@ -1205,24 +1320,19 @@ mod tests {]}]}));- let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();let actions = outcome_block_actions(&out);assert_eq!(actions.len(), 3);- // tool_result with tiny content → NoCompressionApplied (under threshold / detector returns plain text).- assert!(matches!(- actions[0],- BlockAction::NoCompressionApplied { .. }- ));+ // tool_result with tiny content → BelowByteThreshold (1 byte < 5 KiB plain-text threshold).+ assert!(matches!(actions[0], BlockAction::BelowByteThreshold { .. }));assert!(matches!(actions[1],BlockAction::Excluded {reason: ExclusionReason::HotZoneBlockType}));- assert!(matches!(- actions[2],- BlockAction::NoCompressionApplied { .. }- ));+ // text block with "ok" → BelowByteThreshold (2 bytes < 5 KiB plain-text threshold).+ assert!(matches!(actions[2], BlockAction::BelowByteThreshold { .. }));}#[test]@@ -1230,16 +1340,17 @@ mod tests {let b = body(json!({"messages": [{"role": "user", "content": "just a string"}]}));- let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();let manifest = match &out {LiveZoneOutcome::NoChange { manifest } => manifest,LiveZoneOutcome::Modified { manifest, .. } => manifest,};assert_eq!(manifest.block_outcomes.len(), 1);assert_eq!(manifest.block_outcomes[0].block_type, "string_content");+ // 13 bytes of plain text is well below the 5 KiB plain-text threshold.assert!(matches!(manifest.block_outcomes[0].action,- BlockAction::NoCompressionApplied { .. }+ BlockAction::BelowByteThreshold { .. }));}@@ -1248,7 +1359,7 @@ mod tests {let b = body(json!({"messages": [{"role": "assistant", "content": "hi"}]}));- let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();let manifest = match &out {LiveZoneOutcome::NoChange { manifest } => manifest,_ => panic!("expected NoChange"),@@ -1263,9 +1374,10 @@ mod tests {let b = body(json!({"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]}));- let payg = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();- let oauth = compress_anthropic_live_zone(&b, 0, AuthMode::OAuth).unwrap();- let sub = compress_anthropic_live_zone(&b, 0, AuthMode::Subscription).unwrap();+ let payg = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();+ let oauth = compress_anthropic_live_zone(&b, 0, AuthMode::OAuth, DEFAULT_MODEL).unwrap();+ let sub =+ compress_anthropic_live_zone(&b, 0, AuthMode::Subscription, DEFAULT_MODEL).unwrap();for o in [&payg, &oauth, &sub] {assert!(matches!(o, LiveZoneOutcome::NoChange { .. }));}@@ -1283,7 +1395,7 @@ mod tests {]}]}));- let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 0, AuthMode::Payg, DEFAULT_MODEL).unwrap();assert!(matches!(out, LiveZoneOutcome::NoChange { .. }));}@@ -1296,7 +1408,7 @@ mod tests {{"role": "user", "content": "live"},]}));- let out = compress_anthropic_live_zone(&b, 2, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 2, AuthMode::Payg, DEFAULT_MODEL).unwrap();let manifest = match &out {LiveZoneOutcome::NoChange { manifest } => manifest,LiveZoneOutcome::Modified { manifest, .. } => manifest,@@ -1311,7 +1423,7 @@ mod tests {let b = body(json!({"messages": [{"role": "user", "content": "x"}]}));- let out = compress_anthropic_live_zone(&b, 99, AuthMode::Payg).unwrap();+ let out = compress_anthropic_live_zone(&b, 99, AuthMode::Payg, DEFAULT_MODEL).unwrap();let manifest = match &out {LiveZoneOutcome::NoChange { manifest } => manifest,_ => panic!("expected NoChange"),diff --git a/crates/headroom-core/tests/live_zone_dispatch.rs b/crates/headroom-core/tests/live_zone_dispatch.rsindex 264154d15..c8f55cf00 100644--- a/crates/headroom-core/tests/live_zone_dispatch.rs+++ b/crates/headroom-core/tests/live_zone_dispatch.rs@@ -12,7 +12,10 @@//! Plus the cache-safety invariant: bytes outside the rewritten//! block are byte-identical to the input (SHA-256 prefix + suffix).-use headroom_core::transforms::{compress_anthropic_live_zone, AuthMode, BlockAction, LiveZoneOutcome};+use headroom_core::transforms::live_zone::DEFAULT_MODEL;+use headroom_core::transforms::{+ compress_anthropic_live_zone, AuthMode, BlockAction, LiveZoneOutcome,+};use serde_json::{json, Value};use sha2::{Digest, Sha256};@@ -21,7 +24,8 @@ fn body_of(value: Value) -> Vec<u8> {}fn dispatch(body: &[u8]) -> LiveZoneOutcome {- compress_anthropic_live_zone(body, 0, AuthMode::Payg).expect("dispatcher returns Ok on valid bodies")+ compress_anthropic_live_zone(body, 0, AuthMode::Payg, DEFAULT_MODEL)+ .expect("dispatcher returns Ok on valid bodies")}/// Find the byte range of the FIRST occurrence of `needle` inside@@ -110,12 +114,19 @@ fn json_tool_result_routes_to_smart_crusher() {strategy,original_bytes,compressed_bytes,+ original_tokens,+ compressed_tokens,} => {assert_eq!(strategy, "smart_crusher", "expected SmartCrusher dispatch");assert!(compressed_bytes < original_bytes,"SmartCrusher must produce strictly smaller output ({compressed_bytes} < {original_bytes})");+ assert!(+ compressed_tokens < original_tokens,+ "tokenizer-validated gate (PR-B4) must accept only token-shrinking output \+ ({compressed_tokens} < {original_tokens})"+ );}other => panic!("expected BlockAction::Compressed, got {other:?}"),}@@ -159,6 +170,7 @@ fn log_tool_result_routes_to_log_compressor() {action,BlockAction::NoCompressionApplied { .. }| BlockAction::RejectedNotSmaller { .. }+ | BlockAction::BelowByteThreshold { .. }),"log dispatch declined cleanly: {action:?}");@@ -178,6 +190,7 @@ fn log_tool_result_routes_to_log_compressor() {strategy,original_bytes,compressed_bytes,+ ..} => {assert_eq!(strategy, "log_compressor");assert!(compressed_bytes < original_bytes);@@ -189,15 +202,25 @@ fn log_tool_result_routes_to_log_compressor() {#[test]fn diff_tool_result_routes_to_diff_compressor() {// A unidiff with surrounding context the diff compressor can trim.+ // Size kept comfortably above the 1 KiB GitDiff byte threshold+ // (PR-B4) so the dispatch gate is exercised.let mut diff = String::from("diff --git a/foo.rs b/foo.rs\n--- a/foo.rs\n+++ b/foo.rs\n");- diff.push_str("@@ -1,40 +1,40 @@\n");- for i in 0..18 {- diff.push_str(&format!(" context line {i}\n"));+ diff.push_str("@@ -1,80 +1,80 @@\n");+ for i in 0..40 {+ diff.push_str(&format!(" context line {i} with extra padding text\n"));}- diff.push_str("-old line\n+new line\n");- for i in 0..18 {- diff.push_str(&format!(" context line {}\n", i + 18));+ diff.push_str("-old line that needs to be replaced\n+new line replacing the old one\n");+ for i in 0..40 {+ diff.push_str(&format!(+ " context line {} with extra padding text\n",+ i + 40+ ));}+ assert!(+ diff.len() > 1024,+ "diff fixture must be > 1 KiB to clear the GitDiff threshold; got {}",+ diff.len()+ );let (body, _) = body_with_tool_result(&diff);let out = dispatch(&body);@@ -216,6 +239,7 @@ fn diff_tool_result_routes_to_diff_compressor() {action,BlockAction::NoCompressionApplied { .. }| BlockAction::RejectedNotSmaller { .. }+ | BlockAction::BelowByteThreshold { .. }),"diff dispatch declined cleanly: {action:?}");@@ -272,14 +296,25 @@ fn main() {.clone();match action {BlockAction::NoCompressionApplied { content_type } => {- // Detector may classify code-with-prose as PlainText; both- // outcomes are no-op for B3 so accept either tag.+ // Source-code-shaped content above the SourceCode byte+ // threshold (2 KiB) but below any active compressor:+ // SmartCrusher / log / search / diff don't apply, and+ // the Rust code-compressor port is not yet wired.assert!(content_type == "source_code" || content_type == "text","unexpected content_type tag: {content_type}");}- other => panic!("expected NoCompressionApplied, got {other:?}"),+ BlockAction::BelowByteThreshold { content_type, .. } => {+ // Detector may classify code-with-prose as PlainText+ // (5 KiB threshold) — for ~2.6 KiB of mixed code/prose+ // that still routes to no-op for B4. Pin the tag.+ assert!(+ content_type == "text" || content_type == "source_code",+ "unexpected content_type tag: {content_type}"+ );+ }+ other => panic!("expected NoCompressionApplied or BelowByteThreshold, got {other:?}"),}}diff --git a/crates/headroom-core/tests/live_zone_thresholds.rs b/crates/headroom-core/tests/live_zone_thresholds.rsnew file mode 100644index 000000000..946cb54a1--- /dev/null+++ b/crates/headroom-core/tests/live_zone_thresholds.rs@@ -0,0 +1,128 @@+//! PR-B4 byte-threshold gate — integration tests.+//!+//! The dispatcher must skip compression entirely for blocks whose+//! content is below the per-content-type byte threshold (1 KiB for+//! JSON arrays). PR-B4 spec, `REALIGNMENT/04-phase-B-live-zone.md`.++use headroom_core::transforms::live_zone::DEFAULT_MODEL;+use headroom_core::transforms::{+ compress_anthropic_live_zone, AuthMode, BlockAction, LiveZoneOutcome,+};+use serde_json::{json, Value};++fn body_of(value: Value) -> Vec<u8> {+ serde_json::to_vec(&value).unwrap()+}++/// Build a body whose latest user-message tool_result carries `text`.+fn body_with_tool_result(text: &str) -> Vec<u8> {+ body_of(json!({+ "model": "claude-3-5-sonnet-20241022",+ "max_tokens": 64,+ "messages": [{+ "role": "user",+ "content": [{+ "type": "tool_result",+ "tool_use_id": "toolu_threshold_test",+ "content": text,+ }],+ }],+ }))+}++fn dispatch(body: &[u8]) -> LiveZoneOutcome {+ compress_anthropic_live_zone(body, 0, AuthMode::Payg, DEFAULT_MODEL)+ .expect("dispatcher returns Ok on valid bodies")+}++fn first_tool_result_action(out: &LiveZoneOutcome) -> BlockAction {+ let manifest = match out {+ LiveZoneOutcome::NoChange { manifest } => manifest,+ LiveZoneOutcome::Modified { manifest, .. } => manifest,+ };+ manifest+ .block_outcomes+ .iter()+ .find(|b| b.block_type == "tool_result")+ .expect("tool_result block present in manifest")+ .action+ .clone()+}++#[test]+fn below_threshold_no_compression_attempted() {+ // 200 bytes of homogeneous JSON dicts — well below the 1 KiB+ // JsonArray threshold. The dispatcher must record+ // `BelowByteThreshold` and emit `NoChange`; no compressor runs.+ let small_array: Vec<Value> = (0..3).map(|i| json!({"id": i, "v": "x"})).collect();+ let payload = serde_json::to_string(&small_array).unwrap();+ assert!(+ payload.len() < 1024,+ "fixture must stay below the 1 KiB JsonArray threshold; got {}",+ payload.len()+ );++ let body = body_with_tool_result(&payload);+ let out = dispatch(&body);++ // Sub-threshold means no rewriting → NoChange.+ assert!(+ matches!(out, LiveZoneOutcome::NoChange { .. }),+ "below-threshold input must not produce a Modified body"+ );++ let action = first_tool_result_action(&out);+ match action {+ BlockAction::BelowByteThreshold {+ content_type,+ byte_count,+ threshold_bytes,+ } => {+ assert_eq!(content_type, "json_array");+ assert_eq!(byte_count, payload.len());+ assert_eq!(threshold_bytes, 1024);+ }+ other => panic!("expected BelowByteThreshold for sub-threshold JSON, got {other:?}"),+ }+}++#[test]+fn above_threshold_compression_attempted() {+ // 10 KB of homogeneous JSON dicts — comfortably above the 1 KiB+ // JsonArray threshold and SmartCrusher's bread-and-butter shape,+ // so the dispatcher SHOULD route through `dispatch_compressor`.+ // Either `Compressed` (SmartCrusher shrunk it — the typical+ // outcome) or `RejectedNotSmaller` (token gate vetoed it) is+ // acceptable; both prove the dispatcher attempted compression+ // rather than short-circuiting on the byte threshold.+ let big_array: Vec<Value> = (0..400)+ .map(|i| {+ json!({+ "id": i,+ "kind": "row",+ "status": "ok",+ "value": format!("repeat-{}", i % 5),+ })+ })+ .collect();+ let payload = serde_json::to_string(&big_array).unwrap();+ assert!(+ payload.len() >= 10_000,+ "fixture must be >= 10 KB to clear the JsonArray threshold; got {}",+ payload.len()+ );++ let body = body_with_tool_result(&payload);+ let out = dispatch(&body);+ let action = first_tool_result_action(&out);++ match action {+ BlockAction::Compressed { .. } | BlockAction::RejectedNotSmaller { .. } => {+ // Either outcome proves the byte-threshold gate did+ // NOT short-circuit and a compressor actually ran.+ }+ other => panic!(+ "expected Compressed or RejectedNotSmaller after byte-threshold pass, got {other:?}"+ ),+ }+}diff --git a/crates/headroom-core/tests/live_zone_token_validation.rs b/crates/headroom-core/tests/live_zone_token_validation.rsnew file mode 100644index 000000000..3c5afeea6--- /dev/null+++ b/crates/headroom-core/tests/live_zone_token_validation.rs@@ -0,0 +1,254 @@+//! PR-B4 token-validation gate — integration + property tests.+//!+//! After every per-block compression, the dispatcher counts tokens on+//! both the original and compressed text using the model's tokenizer.+//! When `compressed_tokens >= original_tokens` the candidate is+//! rejected and the original bytes are kept. This file pins the+//! contract for both directions of the gate, plus an invariant+//! property test: the dispatcher's emitted body has token-count <= the+//! input's token-count for any well-formed body.++use headroom_core::tokenizer::get_tokenizer;+use headroom_core::transforms::live_zone::DEFAULT_MODEL;+use headroom_core::transforms::{+ compress_anthropic_live_zone, AuthMode, BlockAction, LiveZoneOutcome,+};+use proptest::prelude::*;+use serde_json::{json, Value};++fn body_of(value: Value) -> Vec<u8> {+ serde_json::to_vec(&value).unwrap()+}++fn body_with_tool_result(text: &str) -> Vec<u8> {+ body_of(json!({+ "model": "claude-3-5-sonnet-20241022",+ "max_tokens": 64,+ "messages": [{+ "role": "user",+ "content": [{+ "type": "tool_result",+ "tool_use_id": "toolu_token_test",+ "content": text,+ }],+ }],+ }))+}++fn dispatch(body: &[u8]) -> LiveZoneOutcome {+ compress_anthropic_live_zone(body, 0, AuthMode::Payg, DEFAULT_MODEL)+ .expect("dispatcher returns Ok on valid bodies")+}++fn first_tool_result_action(out: &LiveZoneOutcome) -> BlockAction {+ let manifest = match out {+ LiveZoneOutcome::NoChange { manifest } => manifest,+ LiveZoneOutcome::Modified { manifest, .. } => manifest,+ };+ manifest+ .block_outcomes+ .iter()+ .find(|b| b.block_type == "tool_result")+ .expect("tool_result block present in manifest")+ .action+ .clone()+}++/// Count the tokens of a body's serialized form via the same tokenizer+/// the dispatcher uses. Used by the property test below.+fn token_count_of(bytes: &[u8]) -> usize {+ let tok = get_tokenizer(DEFAULT_MODEL);+ let s = std::str::from_utf8(bytes).expect("body is UTF-8 JSON");+ tok.count_text(s)+}++#[test]+fn compressed_more_tokens_falls_back() {+ // Pathological input: a single dense base64 string wrapped in a+ // JSON array. The detector classifies it as JsonArray (which+ // routes to SmartCrusher), but the contents do not look like a+ // dict array — SmartCrusher will not modify it. We construct a+ // case where the content_type is JsonArray and SmartCrusher's+ // crush returns the input unchanged (`was_modified=false`),+ // which dispatches as NoOp / NoCompressionApplied — that does+ // not test the token gate.+ //+ // To exercise the token gate we need SmartCrusher to actually+ // produce *some* output. Build an array of dicts containing+ // already-minified base64 — SmartCrusher's column-extraction+ // can produce output with more tokens than the raw bytes, even+ // when `was_modified=true`, because dense base64 fragments+ // tokenize into many short BPE pieces post-rewrite.+ //+ // Empirically, for some pathological dict shapes SmartCrusher's+ // output is larger in tokens than the input. The dispatcher+ // must reject such cases via the token gate.+ //+ // The fixture below is deliberately constructed to be on the+ // boundary; if SmartCrusher decides to leave it alone (which it+ // is allowed to do) we still assert the gate behavior — by+ // checking either RejectedNotSmaller (gate fired) OR+ // NoCompressionApplied (compressor declined first). Both are+ // safe outcomes; the negative we want to catch is `Compressed`+ // with token-count growth, which the dispatcher cannot emit.++ // Build a 2 KiB array of single-key dicts holding base64 — large+ // enough to clear the 1 KiB JsonArray threshold but pathological+ // enough that the SmartCrusher's column rewrite is not strictly+ // smaller in tokens for a Claude-family chars-per-token estimator.+ let mut payload = String::from("[");+ for i in 0..40 {+ if i > 0 {+ payload.push(',');+ }+ // 50 bytes of dense base64-ish noise per entry.+ let blob: String = (0..50).map(|j| ((j + i) % 64) as u8 as char).collect();+ let blob_clean: String = blob+ .chars()+ .map(|c| if c.is_ascii_alphanumeric() { c } else { 'A' })+ .collect();+ payload.push_str(&format!("{{\"b\":\"{}\"}}", blob_clean));+ }+ payload.push(']');+ assert!(+ payload.len() >= 1024,+ "fixture must clear the 1 KiB JsonArray byte threshold; got {}",+ payload.len()+ );++ let body = body_with_tool_result(&payload);+ let out = dispatch(&body);+ let action = first_tool_result_action(&out);++ match action {+ BlockAction::RejectedNotSmaller {+ original_tokens,+ compressed_tokens,+ ..+ } => {+ assert!(+ compressed_tokens >= original_tokens,+ "RejectedNotSmaller invariant: compressed_tokens ({compressed_tokens}) \+ must be >= original_tokens ({original_tokens})"+ );+ }+ // Compressor declined before producing output — also safe.+ BlockAction::NoCompressionApplied { .. } => {}+ // Compressor produced output AND it shrunk the token count —+ // also acceptable (this means our fixture wasn't actually+ // pathological for this tokenizer).+ BlockAction::Compressed {+ original_tokens,+ compressed_tokens,+ ..+ } => {+ assert!(+ compressed_tokens < original_tokens,+ "Compressed invariant: compressed_tokens ({compressed_tokens}) \+ must be < original_tokens ({original_tokens})"+ );+ }+ BlockAction::BelowByteThreshold { .. } => {+ panic!(+ "fixture was supposed to clear the byte threshold; got BelowByteThreshold. \+ payload len = {}",+ payload.len()+ );+ }+ other => panic!("unexpected dispatcher outcome: {other:?}"),+ }+}++#[test]+fn compressed_fewer_tokens_accepted() {+ // Well-formed JSON array of homogeneous dicts — SmartCrusher's+ // bread-and-butter shape. The token gate must accept this.+ let array_of_dicts: Vec<Value> = (0..200)+ .map(|i| {+ json!({+ "id": i,+ "status": "ok",+ "value": format!("repeat-pattern-{}", i % 3),+ })+ })+ .collect();+ let payload = serde_json::to_string(&array_of_dicts).unwrap();++ let body = body_with_tool_result(&payload);+ let out = dispatch(&body);+ let action = first_tool_result_action(&out);++ match action {+ BlockAction::Compressed {+ strategy,+ original_tokens,+ compressed_tokens,+ ..+ } => {+ assert_eq!(strategy, "smart_crusher");+ assert!(+ compressed_tokens < original_tokens,+ "tokenizer-validated gate must produce strict token shrinkage \+ ({compressed_tokens} < {original_tokens})"+ );+ }+ other => panic!(+ "expected token-shrinking Compressed for 200-dict SmartCrusher fodder, got {other:?}"+ ),+ }+}++proptest! {+ /// Invariant: for ANY well-formed Anthropic body the dispatcher+ /// either passes through unchanged or emits a body whose token+ /// count is <= the input's token count. There is no input shape+ /// for which the dispatcher must inflate tokens.+ ///+ /// The strategy generates JSON-array-of-dicts payloads whose+ /// dict shapes vary (matching the sort of tool-result content+ /// the dispatcher actually sees in production). Sub-threshold+ /// payloads short-circuit to `NoChange` and therefore trivially+ /// satisfy the invariant.+ #[test]+ fn live_zone_compression_token_count_non_increasing(+ n in 0usize..150,+ key_a in "[a-z]{1,8}",+ key_b in "[a-z]{1,8}",+ seed in any::<u32>(),+ ) {+ let arr: Vec<Value> = (0..n)+ .map(|i| {+ let v: u64 = (seed as u64).wrapping_add(i as u64);+ json!({+ key_a.as_str(): v % 1000,+ key_b.as_str(): format!("v{}", v % 7),+ })+ })+ .collect();+ let payload = serde_json::to_string(&arr).unwrap();+ let body = body_with_tool_result(&payload);+ let body_tokens_in = token_count_of(&body);++ let outcome = compress_anthropic_live_zone(+ &body,+ 0,+ AuthMode::Payg,+ DEFAULT_MODEL,+ )+ .expect("dispatcher returns Ok on valid bodies");++ let body_tokens_out = match &outcome {+ LiveZoneOutcome::NoChange { .. } => body_tokens_in,+ LiveZoneOutcome::Modified { new_body, .. } => {+ token_count_of(new_body.get().as_bytes())+ }+ };++ prop_assert!(+ body_tokens_out <= body_tokens_in,+ "dispatcher must never inflate token count: \+ tokens_in={body_tokens_in}, tokens_out={body_tokens_out}, payload_len={}",+ payload.len(),+ );+ }+}diff --git a/crates/headroom-proxy/src/compression/live_zone_anthropic.rs b/crates/headroom-proxy/src/compression/live_zone_anthropic.rsindex 31ecc5fb2..7c4cb5597 100644--- a/crates/headroom-proxy/src/compression/live_zone_anthropic.rs+++ b/crates/headroom-proxy/src/compression/live_zone_anthropic.rs@@ -39,6 +39,7 @@//! SHA-256 prefix-and-suffix invariant in CI.use bytes::Bytes;+use headroom_core::transforms::live_zone::DEFAULT_MODEL;use headroom_core::transforms::{compress_anthropic_live_zone, AuthMode, BlockAction, ExclusionReason, LiveZoneError,LiveZoneOutcome,@@ -153,13 +154,28 @@ pub fn compress_anthropic_request(let frozen_count = resolve_frozen_count(&parsed, cache_control_policy, request_id);+ // PR-B4: extract `body["model"]` so the live-zone dispatcher can+ // route the tokenizer registry to the right backend for the+ // per-block token-count rejection gate. Anthropic+ // `/v1/messages` always carries a `model` string per the API+ // schema, but the proxy never breaks on a missing field — we+ // fall back to `DEFAULT_MODEL` (a Claude name, so the+ // chars-per-token estimator picks the calibrated 3.5 cpt+ // density) and continue.+ let model = parsed+ .get("model")+ .and_then(serde_json::Value::as_str)+ .unwrap_or(DEFAULT_MODEL);+// Run the live-zone dispatcher. PR-B3 wires per-type compressors:// SmartCrusher / LogCompressor / SearchCompressor / DiffCompressor.- // The dispatcher returns `Modified` whenever at least one block- // was rewritten and `NoChange` otherwise (live zone empty, every- // compressor declined, or every compressor produced output that- // wasn't smaller than the input).- match compress_anthropic_live_zone(body, frozen_count, AuthMode::Payg) {+ // PR-B4 added the per-content-type byte-threshold gate and the+ // tokenizer-validated rejection check. The dispatcher returns+ // `Modified` whenever at least one block was rewritten and+ // `NoChange` otherwise (live zone empty, every compressor+ // declined, or every compressor produced output whose token+ // count was not strictly less than the input's).+ match compress_anthropic_live_zone(body, frozen_count, AuthMode::Payg, model) {Ok(LiveZoneOutcome::NoChange { manifest }) => {let block_count = manifest.block_outcomes.len();let blocks_excluded = manifest@@ -193,21 +209,28 @@ pub fn compress_anthropic_request(}Ok(LiveZoneOutcome::Modified { new_body, manifest }) => {// Aggregate manifest into the proxy's `Compressed` payload.- // PR-B3 reports byte-counts (not token counts); PR-B4- // upgrades the gate to a tokenizer-validated count and- // updates these fields with the real numbers.+ // PR-B4 reports token counts via the same tokenizer the+ // dispatcher used to gate per-block acceptance — so the+ // saving the proxy logs is the saving the cache will+ // actually see.let mut original_bytes_total: usize = 0;let mut compressed_bytes_total: usize = 0;+ let mut original_tokens_total: usize = 0;+ let mut compressed_tokens_total: usize = 0;let mut strategies: Vec<&'static str> = Vec::new();for entry in &manifest.block_outcomes {if let BlockAction::Compressed {strategy,original_bytes,compressed_bytes,+ original_tokens,+ compressed_tokens,} = entry.action{original_bytes_total += original_bytes;compressed_bytes_total += compressed_bytes;+ original_tokens_total += original_tokens;+ compressed_tokens_total += compressed_tokens;if !strategies.contains(&strategy) {strategies.push(strategy);}@@ -234,15 +257,15 @@ pub fn compress_anthropic_request(live_zone_strategies = ?strategies,live_zone_block_original_bytes = original_bytes_total,live_zone_block_compressed_bytes = compressed_bytes_total,+ live_zone_block_original_tokens = original_tokens_total,+ live_zone_block_compressed_tokens = compressed_tokens_total,+ model = model,"anthropic live-zone dispatch");Outcome::Compressed {body: new_body_bytes,- // PR-B4 will replace these with tokenizer counts.- // For now report block-content byte counts so- // observers can compute the savings ratio.- tokens_before: original_bytes_total,- tokens_after: compressed_bytes_total,+ tokens_before: original_tokens_total,+ tokens_after: compressed_tokens_total,strategies_applied: strategies,// PR-B7 wires CCR retrieval-marker injection.markers_inserted: Vec::new(),diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.jsonindex 67c874fa2..5366f6fb8 100644--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.20.12",+ "version": "0.20.13","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.jsonindex 36427b792..d3bb0de99 100644--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.20.12",+ "version": "0.20.13","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",