Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): scaffold smart_crusher module + foundational helpers
chopratejasd219beecab01
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.jsonindex 78381697b..579b17788 100644--- a/.claude-plugin/marketplace.json+++ b/.claude-plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.11.0"+ "version": "0.10.17"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.11.0",+ "version": "0.10.17","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.jsonindex 78381697b..579b17788 100644--- a/.github/plugin/marketplace.json+++ b/.github/plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.11.0"+ "version": "0.10.17"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.11.0",+ "version": "0.10.17","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/Cargo.lock b/Cargo.lockindex cbf163166..4d52bfd86 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -933,6 +933,7 @@ dependencies = ["regex","serde","serde_json",+ "sha2","thiserror 1.0.69","tiktoken-rs","tokenizers",@@ -2238,6 +2239,17 @@ dependencies = ["digest",]+[[package]]+name = "sha2"+version = "0.10.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"+dependencies = [+ "cfg-if",+ "cpufeatures",+ "digest",+]+[[package]]name = "sharded-slab"version = "0.1.7"diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 929230211..d522455d7 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -26,6 +26,9 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq", "rustl# `md5` for the CCR cache_key. Python's compression_store hashes the original# diff with MD5 truncated to 24 hex chars; we must match byte-for-byte.md-5 = "0.10"+# `sha2` for `_hash_field_name` in smart_crusher (SHA256 truncated to 16+# hex chars). Python uses `hashlib.sha256` so we need byte-exact parity.+sha2 = "0.10"# `regex` is already a transitive dep of tokenizers; depend on it directly so# our hunk-header parser and priority-pattern matcher have a stable surface.regex = "1"diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rsindex f0fc4bc49..abaf33363 100644--- a/crates/headroom-core/src/transforms/mod.rs+++ b/crates/headroom-core/src/transforms/mod.rs@@ -16,6 +16,7 @@//! prod and are returned alongside the parity-equal output for tests.pub mod diff_compressor;+pub mod smart_crusher;pub use diff_compressor::{DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,diff --git a/crates/headroom-core/src/transforms/smart_crusher/anchors.rs b/crates/headroom-core/src/transforms/smart_crusher/anchors.rsnew file mode 100644index 000000000..6a6f55eae--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/anchors.rs@@ -0,0 +1,251 @@+//! Legacy regex-based query anchor extraction.+//!+//! Direct port of `extract_query_anchors` and `item_matches_anchors`+//! (`smart_crusher.py:99-168`). The Python doc-comment marks both as+//! DEPRECATED in favor of `RelevanceScorer`, but they're still called+//! by the live SmartCrusher path on every invocation, so we port them+//! faithfully.+//!+//! # Why regex parity matters+//!+//! These regexes drive which array items survive compression. A subtle+//! difference between Python's `re` engine and Rust's `regex` crate+//! (e.g. word-boundary behavior on Unicode, or repetition greediness)+//! would silently change which anchors are detected and which items+//! survive. The patterns below are pinned to lowercase ASCII inputs+//! and use only ASCII-safe constructs to keep behavior identical.++use regex::Regex;+use serde_json::Value;+use std::collections::HashSet;+use std::sync::LazyLock;++// ---------------------------------------------------------------+// Pattern definitions — direct ports of the module-level Python regexes+// at `smart_crusher.py:85-93`. `std::sync::LazyLock` (stable since Rust+// 1.80) is the modern equivalent of `once_cell::sync::Lazy`, mirroring+// Python's `re.compile` at module import time.+// ---------------------------------------------------------------++/// `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`+static UUID_PATTERN: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")+ .expect("UUID_PATTERN")+});++/// 4+ digit numbers (likely IDs). Python: `r"\b\d{4,}\b"`.+static NUMERIC_ID_PATTERN: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"\b\d{4,}\b").expect("NUMERIC_ID_PATTERN"));++/// Hostname pattern. Matches `host.tld` with optional `.tld2`. Python:+/// `r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b"`.+static HOSTNAME_PATTERN: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b")+ .expect("HOSTNAME_PATTERN")+});++/// Short quoted strings (single OR double quotes), 1-50 chars between+/// quotes. Python: `r"['\"]([^'\"]{1,50})['\"]"`.+static QUOTED_STRING_PATTERN: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r#"['"]([^'"]{1,50})['"]"#).expect("QUOTED_STRING_PATTERN"));++/// Email addresses. Python: `r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"`.+/// (Note Python's `[A-Z|a-z]` includes a literal `|` in the character+/// class — almost certainly a typo, but we faithfully port it for+/// parity. Real-world impact is nil since `|` doesn't appear in TLDs.)+static EMAIL_PATTERN: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b").expect("EMAIL_PATTERN")+});++/// Hostname false-positive blocklist. Python uses a set literal at+/// `smart_crusher.py:137`. We mirror exactly — these strings get+/// dropped from anchor results.+const HOSTNAME_FALSE_POSITIVES: &[&str] = &["e.g", "i.e", "etc."];++/// Extract query anchors from user text. **DEPRECATED** in Python in+/// favor of `RelevanceScorer`, but still called by the live path —+/// ported as-is.+///+/// Output is a set of lowercased anchor strings. Order is not+/// significant (Python returns `set[str]`).+pub fn extract_query_anchors(text: &str) -> HashSet<String> {+ let mut anchors = HashSet::new();++ if text.is_empty() {+ return anchors;+ }++ // UUIDs — lowercase the match.+ for m in UUID_PATTERN.find_iter(text) {+ anchors.insert(m.as_str().to_lowercase());+ }++ // Numeric IDs — Python keeps original case (digits, no transform needed).+ for m in NUMERIC_ID_PATTERN.find_iter(text) {+ anchors.insert(m.as_str().to_string());+ }++ // Hostnames — lowercase, filter false positives.+ for m in HOSTNAME_PATTERN.find_iter(text) {+ let lc = m.as_str().to_lowercase();+ if !HOSTNAME_FALSE_POSITIVES.contains(&lc.as_str()) {+ anchors.insert(lc);+ }+ }++ // Quoted strings — capture group 1 (the content between quotes),+ // require trim().len() >= 2 (Python's `if len(match.strip()) >= 2`).+ for caps in QUOTED_STRING_PATTERN.captures_iter(text) {+ if let Some(inner) = caps.get(1) {+ if inner.as_str().trim().len() >= 2 {+ anchors.insert(inner.as_str().to_lowercase());+ }+ }+ }++ // Emails — lowercase.+ for m in EMAIL_PATTERN.find_iter(text) {+ anchors.insert(m.as_str().to_lowercase());+ }++ anchors+}++/// Check if a JSON object matches any query anchors.+///+/// Direct port of `item_matches_anchors` (Python `smart_crusher.py:152-168`).+/// Python uses `str(item).lower()` which produces Python's `dict.__str__`+/// representation. We mirror by serializing with `serde_json` and+/// lowercasing — this isn't byte-identical to Python's `str(dict)`+/// (Python uses single quotes, JSON uses double; Python's bool is+/// `True`/`False`, JSON's is `true`/`false`), so for cross-language+/// parity we need a string form that matches Python's. We document this+/// gap and fix it in the analyzer integration.+///+/// **WARNING:** `str(item).lower()` in Python produces:+/// `{'key': 'value', 'count': 5, 'ok': True}`+/// while `serde_json::to_string(&item)` produces:+/// `{"key":"value","count":5,"ok":true}`+///+/// The anchor matching is substring-based (`anchor in item_str`), so+/// this difference matters: if an anchor is `"true"` it matches the+/// JSON form but not the Python form, and vice versa for `"True"`.+///+/// **Resolution:** when items reach the matcher they're already+/// lowercased, so `True` → `true` after `.lower()`, removing one source+/// of drift. The remaining drift (single vs double quotes, trailing+/// whitespace) is unlikely to affect anchor matching in practice. We+/// pin behavior with fixtures and move on.+pub fn item_matches_anchors(item: &Value, anchors: &HashSet<String>) -> bool {+ if anchors.is_empty() {+ return false;+ }++ // Python: `str(item).lower()`. We approximate via JSON serialization+ // followed by `.lower()` — see WARNING above for the gap.+ let item_str = match serde_json::to_string(item) {+ Ok(s) => s.to_lowercase(),+ Err(_) => return false,+ };++ anchors.iter().any(|a| item_str.contains(a))+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn empty_text_no_anchors() {+ assert!(extract_query_anchors("").is_empty());+ }++ #[test]+ fn extracts_uuid_lowercased() {+ let anchors = extract_query_anchors("see id 550E8400-E29B-41D4-A716-446655440000 plz");+ assert!(anchors.contains("550e8400-e29b-41d4-a716-446655440000"));+ }++ #[test]+ fn extracts_numeric_id_unchanged() {+ let anchors = extract_query_anchors("user 12345 reported issue");+ assert!(anchors.contains("12345"));+ }++ #[test]+ fn three_digit_number_not_anchor() {+ // Pattern requires 4+ digits.+ let anchors = extract_query_anchors("user 123 reported issue");+ assert!(!anchors.iter().any(|a| a == "123"));+ }++ #[test]+ fn extracts_hostname() {+ let anchors = extract_query_anchors("connect to api.example.com asap");+ assert!(anchors.contains("api.example.com"));+ }++ #[test]+ fn hostname_false_positive_filtered() {+ // "e.g" is in the blocklist — must NOT appear as an anchor even+ // though it matches the regex.+ let anchors = extract_query_anchors("test e.g.com endpoint");+ // "e.g" is filtered, but "e.g.com" or other longer matches may+ // pass; we only assert "e.g" itself is gone.+ assert!(!anchors.contains("e.g"));+ }++ #[test]+ fn extracts_quoted_string_double() {+ let anchors = extract_query_anchors(r#"find the "user_name" field"#);+ assert!(anchors.contains("user_name"));+ }++ #[test]+ fn extracts_quoted_string_single() {+ let anchors = extract_query_anchors("find the 'user_name' field");+ assert!(anchors.contains("user_name"));+ }++ #[test]+ fn very_short_quoted_skipped() {+ // Less than 2 chars after trim — skipped.+ let anchors = extract_query_anchors(r#"the "x" thing"#);+ assert!(!anchors.contains("x"));+ }++ #[test]+ fn extracts_email() {+ let anchors = extract_query_anchors("contact [email protected] please");+ assert!(anchors.contains("[email protected]"));+ }++ #[test]+ fn item_matches_anchors_empty_set() {+ let empty = HashSet::new();+ assert!(!item_matches_anchors(&json!({"a": 1}), &empty));+ }++ #[test]+ fn item_matches_anchor_in_value() {+ let anchors: HashSet<String> = ["alice".to_string()].into_iter().collect();+ assert!(item_matches_anchors(&json!({"name": "Alice"}), &anchors));+ }++ #[test]+ fn item_matches_anchor_in_key() {+ let anchors: HashSet<String> = ["status".to_string()].into_iter().collect();+ // The anchor "status" appears in the JSON-serialized key.+ assert!(item_matches_anchors(+ &json!({"status": "ok"}),+ &anchors+ ));+ }++ #[test]+ fn item_no_match_with_unrelated_anchor() {+ let anchors: HashSet<String> = ["xyz123".to_string()].into_iter().collect();+ assert!(!item_matches_anchors(&json!({"a": "b"}), &anchors));+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/classifier.rs b/crates/headroom-core/src/transforms/smart_crusher/classifier.rsnew file mode 100644index 000000000..d8a04892d--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/classifier.rs@@ -0,0 +1,203 @@+//! JSON array element-type classification.+//!+//! Direct port of `_classify_array` (Python `smart_crusher.py:341-368`).+//! Classification drives compression strategy: dict arrays go through+//! `_crush_array`, string arrays through `_crush_string_array`, etc.+//!+//! # Python parity note: bool vs int+//!+//! Python's `True`/`False` are an int subclass, so a list `[True, False, 1]`+//! has `types == {bool, int}` but `[True, False]` has `types == {bool}`.+//! The Python code uses two checks to disambiguate:+//! 1. `has_bool` flag set during the type-walk+//! 2. `all(isinstance(i, bool) for i in items)` for pure-bool arrays+//!+//! The Rust `serde_json::Value` enum has separate `Bool` and `Number`+//! variants — no inheritance — so the disambiguation is naturally cleaner+//! here. We still walk every element (not a sample) to guarantee correct+//! classification on adversarial inputs.++use serde_json::Value;++/// JSON array element type classification.+///+/// Mirrors Python's `ArrayType` enum at `smart_crusher.py:329-338`. The+/// string variants in `Display`/`Debug` match Python's lowercase `value=`+/// strings exactly, which is required for parity with serialized strategy+/// debug output (e.g. `"dict_array(100->10)"`).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum ArrayType {+ /// `[{...}, {...}, ...]` — dict array, full statistical path.+ DictArray,+ /// `["a", "b", "c", ...]` — string array.+ StringArray,+ /// `[1, 2.5, 3, ...]` — number array (excludes bools).+ NumberArray,+ /// `[true, false, ...]` — pure bool array.+ BoolArray,+ /// `[[...], [...], ...]` — array of arrays.+ NestedArray,+ /// Anything else: heterogeneous or unclassifiable.+ MixedArray,+ /// `[]` — empty array.+ Empty,+}++impl ArrayType {+ /// Lowercase string representation matching Python's `Enum.value`.+ /// Used in strategy debug strings; must match Python exactly.+ pub fn as_str(self) -> &'static str {+ match self {+ ArrayType::DictArray => "dict_array",+ ArrayType::StringArray => "string_array",+ ArrayType::NumberArray => "number_array",+ ArrayType::BoolArray => "bool_array",+ ArrayType::NestedArray => "nested_array",+ ArrayType::MixedArray => "mixed_array",+ ArrayType::Empty => "empty",+ }+ }+}++/// Classify a JSON array by its element types.+///+/// Walks every element (not a sample) to guarantee correct classification+/// even on adversarial inputs where the first few items hide a type+/// transition deeper in the list. `Value::is_*` is O(1), so the full+/// walk is fine.+///+/// Returns `ArrayType::Empty` for an empty slice.+pub fn classify_array(items: &[Value]) -> ArrayType {+ if items.is_empty() {+ return ArrayType::Empty;+ }++ // Track which Value variants we've seen. We collapse Number into+ // either "int-like" or "float-like" once below; here we only need to+ // know whether there's at least one of each high-level kind.+ let mut has_bool = false;+ let mut has_number = false;+ let mut has_string = false;+ let mut has_object = false;+ let mut has_array = false;+ let mut has_null = false;++ for item in items {+ match item {+ Value::Bool(_) => has_bool = true,+ Value::Number(_) => has_number = true,+ Value::String(_) => has_string = true,+ Value::Object(_) => has_object = true,+ Value::Array(_) => has_array = true,+ Value::Null => has_null = true,+ }+ }++ // Pure bool array — Python's check is `all(isinstance(i, bool))`.+ // Note Python `[True, False, 1]` evaluates to `types == {bool, int}`+ // because bool is an int subclass; that maps to MixedArray here.+ if has_bool && !has_number && !has_string && !has_object && !has_array && !has_null {+ return ArrayType::BoolArray;+ }++ // Pure dict array.+ if has_object && !has_bool && !has_number && !has_string && !has_array && !has_null {+ return ArrayType::DictArray;+ }++ // Pure string array.+ if has_string && !has_bool && !has_number && !has_object && !has_array && !has_null {+ return ArrayType::StringArray;+ }++ // Pure number array — Python explicitly excludes bool here.+ if has_number && !has_bool && !has_string && !has_object && !has_array && !has_null {+ return ArrayType::NumberArray;+ }++ // Pure nested array.+ if has_array && !has_bool && !has_number && !has_string && !has_object && !has_null {+ return ArrayType::NestedArray;+ }++ // Anything else — heterogeneous types, or types involving null.+ ArrayType::MixedArray+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn empty_array() {+ let items: Vec<Value> = vec![];+ assert_eq!(classify_array(&items), ArrayType::Empty);+ }++ #[test]+ fn pure_dict_array() {+ let items = vec![json!({"a": 1}), json!({"b": 2})];+ assert_eq!(classify_array(&items), ArrayType::DictArray);+ }++ #[test]+ fn pure_string_array() {+ let items = vec![json!("a"), json!("b"), json!("c")];+ assert_eq!(classify_array(&items), ArrayType::StringArray);+ }++ #[test]+ fn pure_number_array_int_and_float() {+ let items = vec![json!(1), json!(2.5), json!(3)];+ assert_eq!(classify_array(&items), ArrayType::NumberArray);+ }++ #[test]+ fn pure_bool_array() {+ let items = vec![json!(true), json!(false), json!(true)];+ assert_eq!(classify_array(&items), ArrayType::BoolArray);+ }++ #[test]+ fn nested_array() {+ let items = vec![json!([1, 2]), json!([3, 4])];+ assert_eq!(classify_array(&items), ArrayType::NestedArray);+ }++ #[test]+ fn mixed_dict_and_string_is_mixed() {+ let items = vec![json!({"a": 1}), json!("str")];+ assert_eq!(classify_array(&items), ArrayType::MixedArray);+ }++ #[test]+ fn bool_with_number_is_mixed_not_bool_or_number() {+ // Python's `[True, False, 1]` matches `types <= {bool, int}` BUT+ // fails `all(isinstance(i, bool))`, so falls through to NUMBER_ARRAY+ // check which has `not has_bool` — fails. Then nested check fails.+ // Final: MIXED_ARRAY. Same here.+ let items = vec![json!(true), json!(false), json!(1)];+ assert_eq!(classify_array(&items), ArrayType::MixedArray);+ }++ #[test]+ fn null_in_array_is_mixed() {+ // Python's `types == {dict}` check fails when None (NoneType) is+ // present, so a dict array with one null falls to MIXED_ARRAY.+ let items = vec![json!({"a": 1}), json!(null)];+ assert_eq!(classify_array(&items), ArrayType::MixedArray);+ }++ #[test]+ fn as_str_matches_python_values() {+ // Strategy debug strings depend on these exact lowercase forms.+ assert_eq!(ArrayType::DictArray.as_str(), "dict_array");+ assert_eq!(ArrayType::StringArray.as_str(), "string_array");+ assert_eq!(ArrayType::NumberArray.as_str(), "number_array");+ assert_eq!(ArrayType::BoolArray.as_str(), "bool_array");+ assert_eq!(ArrayType::NestedArray.as_str(), "nested_array");+ assert_eq!(ArrayType::MixedArray.as_str(), "mixed_array");+ assert_eq!(ArrayType::Empty.as_str(), "empty");+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/config.rs b/crates/headroom-core/src/transforms/smart_crusher/config.rsnew file mode 100644index 000000000..cb7959856--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/config.rs@@ -0,0 +1,99 @@+//! SmartCrusher configuration.+//!+//! Direct port of `SmartCrusherConfig` at `smart_crusher.py:927-957`. The+//! defaults must match Python exactly — they're consulted everywhere+//! during compression and any drift breaks parity fixtures.++/// Configuration for SmartCrusher.+///+/// SCHEMA-PRESERVING: Output contains only items from the original array.+/// No wrappers, no generated text, no metadata keys. (Python comment at+/// line 930-931.)+#[derive(Debug, Clone)]+pub struct SmartCrusherConfig {+ pub enabled: bool,+ /// Don't analyze arrays smaller than this. Default 5.+ pub min_items_to_analyze: usize,+ /// Only crush content with more than this many tokens. Default 200.+ pub min_tokens_to_crush: usize,+ /// Standard deviations from the mean to count as a change point.+ /// Default 2.0.+ pub variance_threshold: f64,+ /// Below this unique-ratio, a field is treated as nearly constant.+ /// Default 0.1.+ pub uniqueness_threshold: f64,+ /// Similarity score above which strings cluster together. Default 0.8.+ pub similarity_threshold: f64,+ /// Target maximum items in the output. Default 15.+ pub max_items_after_crush: usize,+ /// Whether to preserve detected change points. Default true.+ pub preserve_change_points: bool,+ /// Factor out fields with constant values across all items. Default+ /// false (disabled — preserves original schema).+ pub factor_out_constants: bool,+ /// Include generated text summaries in output. Default false (disabled+ /// — no generated text).+ pub include_summaries: bool,+ /// Use feedback hints to adjust compression aggressiveness. Default true.+ pub use_feedback_hints: bool,+ /// Minimum confidence required to apply TOIN recommendations.+ /// Default 0.5. (Python LOW FIX #21.)+ pub toin_confidence_threshold: f64,+ /// Drop content-identical items before sampling. Default true.+ pub dedup_identical_items: bool,+ /// Fraction of K to allocate to the start of the array. Default 0.3.+ pub first_fraction: f64,+ /// Fraction of K to allocate to the end of the array. Default 0.15.+ pub last_fraction: f64,+}++impl Default for SmartCrusherConfig {+ fn default() -> Self {+ // These defaults must match smart_crusher.py:934-957 byte-for-byte.+ SmartCrusherConfig {+ enabled: true,+ min_items_to_analyze: 5,+ min_tokens_to_crush: 200,+ variance_threshold: 2.0,+ uniqueness_threshold: 0.1,+ similarity_threshold: 0.8,+ max_items_after_crush: 15,+ preserve_change_points: true,+ factor_out_constants: false,+ include_summaries: false,+ use_feedback_hints: true,+ toin_confidence_threshold: 0.5,+ dedup_identical_items: true,+ first_fraction: 0.3,+ last_fraction: 0.15,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn defaults_match_python() {+ // Pin every default. Each field is consulted by some compression+ // path and a drift would break parity. If Python ever changes a+ // default, this test must be updated in lockstep.+ let c = SmartCrusherConfig::default();+ assert!(c.enabled);+ assert_eq!(c.min_items_to_analyze, 5);+ assert_eq!(c.min_tokens_to_crush, 200);+ assert_eq!(c.variance_threshold, 2.0);+ assert_eq!(c.uniqueness_threshold, 0.1);+ assert_eq!(c.similarity_threshold, 0.8);+ assert_eq!(c.max_items_after_crush, 15);+ assert!(c.preserve_change_points);+ assert!(!c.factor_out_constants);+ assert!(!c.include_summaries);+ assert!(c.use_feedback_hints);+ assert_eq!(c.toin_confidence_threshold, 0.5);+ assert!(c.dedup_identical_items);+ assert_eq!(c.first_fraction, 0.3);+ assert_eq!(c.last_fraction, 0.15);+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/hashing.rs b/crates/headroom-core/src/transforms/smart_crusher/hashing.rsnew file mode 100644index 000000000..4a4a404de--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs@@ -0,0 +1,58 @@+//! Field-name hashing for cache keys.+//!+//! Direct port of `_hash_field_name` (Python `smart_crusher.py:171-176`).+//! Used to generate stable cache keys for compression hints — must match+//! Python byte-for-byte or cache lookups will miss.++use sha2::{Digest, Sha256};++/// SHA-256 of the UTF-8 bytes, hex-encoded, truncated to 16 chars.+///+/// Python equivalent: `hashlib.sha256(field_name.encode()).hexdigest()[:16]`.+/// We use lowercase hex (the default for both Python and Rust's `sha2`+/// crate) — the test below pins this.+pub fn hash_field_name(field_name: &str) -> String {+ let mut hasher = Sha256::new();+ hasher.update(field_name.as_bytes());+ let digest = hasher.finalize();+ // Convert to lowercase hex, then truncate to first 16 chars (8 bytes).+ let hex = format!("{:x}", digest);+ hex[..16].to_string()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn matches_python_sha256_truncated_to_16() {+ // Verified against Python: hashlib.sha256(b"customer_id").hexdigest()[:16]+ assert_eq!(hash_field_name("customer_id"), "1e38d67dbe8f47d2");+ }++ #[test]+ fn empty_string() {+ // Verified against Python: hashlib.sha256(b"").hexdigest()[:16]+ assert_eq!(hash_field_name(""), "e3b0c44298fc1c14");+ }++ #[test]+ fn unicode_field_name() {+ // Verified against Python: hashlib.sha256("café".encode()).hexdigest()[:16]+ // UTF-8 bytes for "café" are 63 61 66 c3 a9 — must encode same way.+ assert_eq!(hash_field_name("café"), "850f7dc43910ff89");+ }++ #[test]+ fn deterministic() {+ // Same input → same output across calls.+ assert_eq!(hash_field_name("test"), hash_field_name("test"));+ }++ #[test]+ fn output_length_is_16() {+ // Always exactly 16 hex chars regardless of input length.+ assert_eq!(hash_field_name("a").len(), 16);+ assert_eq!(hash_field_name(&"x".repeat(1000)).len(), 16);+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rsnew file mode 100644index 000000000..9c408cb39--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs@@ -0,0 +1,50 @@+//! Smart statistical tool output compression — Rust port of+//! `headroom/transforms/smart_crusher.py`.+//!+//! # Stage 3c.1: like-for-like parity port+//!+//! This module is a literal Rust port of the Python `SmartCrusher`+//! implementation. The goal of Stage 3c.1 is **byte-equal output parity** for+//! every fixture in `tests/parity/fixtures/smart_crusher/`. Architectural+//! improvements (lossless-first, unified saliency score, structured CCR+//! markers, token budget) are deferred to Stage 3c.2 and tracked in+//! `~/Desktop/SmartCrusher-Architecture-Improvements.md`.+//!+//! # Bugs fixed in BOTH Python and Rust during 3c.1+//!+//! Four defects in the Python source (`headroom/transforms/smart_crusher.py`)+//! were caught during port review. They're fixed in both languages+//! simultaneously so the parity fixtures continue to byte-match:+//!+//! - **k-split overshoot** (line 2722): `_compute_k_split` keeps 2 items when+//! `k_total = 1` because `max(1, round(k_total * fraction))` floors both+//! first and last to 1. Violates `max_items_after_crush`.+//! - **sequential-pattern false positive** (line 444): `_detect_sequential_pattern`+//! does `int("001")` and silently loses zero padding. Padded string IDs+//! misclassified as sequential numeric IDs.+//! - **rare-status detection short-circuit** (line 674): `_detect_rare_status_values`+//! exits early at >10 distinct values. Datasets with 50+ error codes lose+//! rare-error preservation.+//! - **percentile off-by-one** (line 2844): For `len < 8`, integer-division+//! percentile indices are off by one. Cosmetic — only affects strategy+//! debug strings.+//!+//! Each fix has a fixture entry in the parity harness and a corresponding+//! test in `tests/test_transforms/test_smart_crusher_bugs.py`.++mod anchors;+mod classifier;+mod config;+mod hashing;+mod statistics;+mod types;++pub use anchors::{extract_query_anchors, item_matches_anchors};+pub use classifier::{classify_array, ArrayType};+pub use config::SmartCrusherConfig;+pub use hashing::hash_field_name;+pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};+pub use types::{+ ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,+ FieldStats,+};diff --git a/crates/headroom-core/src/transforms/smart_crusher/statistics.rs b/crates/headroom-core/src/transforms/smart_crusher/statistics.rsnew file mode 100644index 000000000..d3530ddd2--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/statistics.rs@@ -0,0 +1,375 @@+//! Statistical helpers for field characterization.+//!+//! Direct port of the helpers at `smart_crusher.py:378-481`. These are+//! used by the analyzer to classify fields (ID-like, score-like, etc.).+//! Detection logic is heuristic; small numeric drift between Python and+//! Rust would change classifications and break fixtures, so the math+//! here mirrors Python step-by-step.++use serde_json::Value;+use std::collections::HashMap;++/// Check if a string looks like a UUID.+///+/// Direct port of `_is_uuid_format` (Python `smart_crusher.py:378-392`).+/// Format check only — no version-bit validation. Hex chars are lower+/// or upper case, matching Python.+pub fn is_uuid_format(value: &str) -> bool {+ if value.len() != 36 {+ return false;+ }++ // Expected segment lengths: 8-4-4-4-12.+ let parts: Vec<&str> = value.split('-').collect();+ if parts.len() != 5 {+ return false;+ }+ let expected_lens = [8, 4, 4, 4, 12];+ for (part, &expected_len) in parts.iter().zip(expected_lens.iter()) {+ if part.len() != expected_len {+ return false;+ }+ for c in part.chars() {+ if !c.is_ascii_hexdigit() {+ return false;+ }+ }+ }+ true+}++/// Shannon entropy of a string, normalized to `[0, 1]`.+///+/// Direct port of `_calculate_string_entropy` (`smart_crusher.py:395-423`).+/// High entropy (>0.7) suggests random/ID-like content. Low entropy+/// (<0.3) suggests repetitive/predictable content. Used by ID detection.+///+/// # Edge cases (matched to Python)+/// - Empty or single-character strings return `0.0`.+/// - All-identical chars: `freq` has 1 entry, `max_entropy = log2(min(1, n)) = 0.0`,+/// we return `0.0` to avoid division by zero.+pub fn calculate_string_entropy(s: &str) -> f64 {+ // Python uses `len(s) < 2` and computes by character. Rust strings+ // are UTF-8 so we iterate `chars()` to match Python's character-level+ // semantics (Python iterates code points; Rust's `chars()` yields+ // Unicode scalar values — same thing for non-surrogate text).+ let n = s.chars().count();+ if n < 2 {+ return 0.0;+ }++ let mut freq: HashMap<char, usize> = HashMap::new();+ for c in s.chars() {+ *freq.entry(c).or_insert(0) += 1;+ }++ let length = n as f64;+ let mut entropy = 0.0_f64;+ for &count in freq.values() {+ let p = count as f64 / length;+ if p > 0.0 {+ entropy -= p * p.log2();+ }+ }++ // Normalize by the maximum possible entropy at this length:+ // Python: max_entropy = log2(min(len(freq), length))+ let max_entropy = (freq.len().min(n) as f64).log2();+ if max_entropy > 0.0 {+ entropy / max_entropy+ } else {+ 0.0+ }+}++/// Detect if numeric values form a sequential pattern (like IDs:+/// 1, 2, 3, ...).+///+/// Direct port of `_detect_sequential_pattern` (`smart_crusher.py:426-481`)+/// **with BUG #2 FIXED**.+///+/// # Bug #2 — string-padding misclassification+/// Python's original implementation calls `int("001") == 1` and silently+/// loses zero padding, so a list of padded string IDs like+/// `["001", "002", ..., "100"]` looks like a sequential numeric pattern+/// when in reality it's a categorical string field where the padding+/// matters. The fix: when a value is a string that parses as a number,+/// flag the input as "had string-encoded numerics". If ALL parsed values+/// originated as strings, refuse to classify as a sequential numeric+/// pattern. Mixed numeric+string inputs still parse as sequential because+/// the unambiguous numeric values dominate the signal.+///+/// This fix is applied in BOTH languages simultaneously (Python `smart_crusher.py`+/// gets the same fix in the same PR) so the parity fixtures continue to+/// match. Tests covering this bug live at+/// `tests/test_transforms/test_smart_crusher_bugs.py`.+///+/// # Args+/// - `values`: items to inspect.+/// - `check_order`: when true, also require ascending order in the+/// original array (the Python flag — IDs are usually ascending in+/// source order, scores are usually descending).+pub fn detect_sequential_pattern(values: &[Value], check_order: bool) -> bool {+ if values.len() < 5 {+ return false;+ }++ // Collect numeric values, tracking whether each value originated as+ // a string. This is the BUG #2 fix: we still parse strings into+ // numbers (so legitimate mixed-type fields work), but we'll refuse+ // to flag the field as sequential if EVERY parseable value was a+ // string.+ let mut nums: Vec<f64> = Vec::new();+ let mut had_non_string_numeric = false;++ for v in values {+ match v {+ Value::Number(n) => {+ if let Some(f) = n.as_f64() {+ nums.push(f);+ had_non_string_numeric = true;+ }+ }+ Value::Bool(_) => {+ // Python: `isinstance(v, int | float) and not isinstance(v, bool)` —+ // bools are explicitly excluded.+ }+ Value::String(s) => {+ // Python: `try: nums.append(int(v))`. `int("3.14")` raises+ // in Python, so we mirror by trying integer parse first and+ // only succeeding for pure-integer strings.+ if let Ok(parsed) = s.parse::<i64>() {+ nums.push(parsed as f64);+ // BUG #2 fix: do NOT set had_non_string_numeric.+ // If we later find this is the ONLY source of numeric+ // values, we refuse to call it sequential.+ }+ }+ _ => {}+ }+ }++ if nums.len() < 5 {+ return false;+ }++ // BUG #2 fix gate: if every numeric value originated as a string,+ // the field is categorical (e.g. zero-padded codes); not sequential.+ if !had_non_string_numeric {+ return false;+ }++ // Need at least 2 elements for pairwise comparison. (Python checks+ // this redundantly after `len(nums) < 5`.)+ if nums.len() < 2 {+ return false;+ }++ // Sort and compute pairwise diffs.+ let mut sorted_nums = nums.clone();+ sorted_nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));+ let diffs: Vec<f64> = sorted_nums+ .windows(2)+ .map(|w| w[1] - w[0])+ .collect();+ if diffs.is_empty() {+ return false;+ }++ let avg_diff: f64 = diffs.iter().sum::<f64>() / diffs.len() as f64;+ if !(0.5..=2.0).contains(&avg_diff) {+ return false;+ }++ // Most diffs in [0.5, 2.0] => sequential candidate.+ let consistent_count = diffs.iter().filter(|&&d| (0.5..=2.0).contains(&d)).count();+ let is_sequential = consistent_count as f64 / diffs.len() as f64 > 0.8;+ if !is_sequential {+ return false;+ }++ if check_order {+ // Python: ascending count over original (not-sorted) sequence.+ // IDs ascend in array order; scores typically descend.+ let ascending_count = nums+ .windows(2)+ .filter(|w| w[0] <= w[1])+ .count();+ let n_pairs = nums.len() - 1;+ let is_ascending = ascending_count as f64 / n_pairs as f64 > 0.7;+ return is_ascending;+ }++ is_sequential+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ // ---------- is_uuid_format ----------++ #[test]+ fn uuid_format_canonical_lowercase() {+ assert!(is_uuid_format("550e8400-e29b-41d4-a716-446655440000"));+ }++ #[test]+ fn uuid_format_uppercase() {+ assert!(is_uuid_format("550E8400-E29B-41D4-A716-446655440000"));+ }++ #[test]+ fn uuid_format_wrong_length_rejected() {+ assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000")); // 1 short+ assert!(!is_uuid_format("550e8400-e29b-41d4-a716-4466554400000")); // 1 long+ }++ #[test]+ fn uuid_format_wrong_segment_count() {+ assert!(!is_uuid_format("550e8400e29b41d4a716446655440000"));+ }++ #[test]+ fn uuid_format_non_hex_rejected() {+ assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000z"));+ }++ #[test]+ fn uuid_format_empty_rejected() {+ assert!(!is_uuid_format(""));+ }++ // ---------- calculate_string_entropy ----------++ #[test]+ fn entropy_empty_string_is_zero() {+ assert_eq!(calculate_string_entropy(""), 0.0);+ }++ #[test]+ fn entropy_single_char_is_zero() {+ assert_eq!(calculate_string_entropy("a"), 0.0);+ }++ #[test]+ fn entropy_all_same_chars_is_zero() {+ // "aaaa" — freq has 1 entry, max_entropy = log2(1) = 0.0,+ // we return 0.0 from the guard.+ assert_eq!(calculate_string_entropy("aaaa"), 0.0);+ }++ #[test]+ fn entropy_perfectly_uniform_normalized_to_one() {+ // Two distinct chars, 50/50: raw entropy = 1.0, max = log2(2) = 1.0,+ // normalized = 1.0.+ let e = calculate_string_entropy("ab");+ assert!((e - 1.0).abs() < 1e-9);+ }++ #[test]+ fn entropy_mostly_repeated_low() {+ // "aaaaaab" — 6/7 'a', 1/7 'b' — should be small.+ let e = calculate_string_entropy("aaaaaab");+ assert!(e < 0.7);+ }++ #[test]+ fn entropy_high_for_random_looking_string() {+ // Approximation of a UUID-ish hex string. Should be > 0.7.+ let e = calculate_string_entropy("a3f7b2c9d8e1f4a7");+ assert!(e > 0.7);+ }++ // ---------- detect_sequential_pattern ----------++ #[test]+ fn sequential_simple_int_ascending() {+ let v: Vec<Value> = (1..=10).map(|i| json!(i)).collect();+ assert!(detect_sequential_pattern(&v, true));+ }++ #[test]+ fn sequential_too_few_values() {+ let v = vec![json!(1), json!(2), json!(3)];+ assert!(!detect_sequential_pattern(&v, true));+ }++ #[test]+ fn sequential_random_numbers_not_detected() {+ let v: Vec<Value> = vec![+ json!(100),+ json!(2),+ json!(85),+ json!(7),+ json!(43),+ json!(17),+ ];+ assert!(!detect_sequential_pattern(&v, true));+ }++ #[test]+ fn sequential_descending_with_check_order_rejected() {+ // Descending sequence — if check_order=true, must NOT be flagged+ // as sequential (Python: scores descend, IDs ascend).+ let v: Vec<Value> = (1..=10).rev().map(|i| json!(i)).collect();+ assert!(!detect_sequential_pattern(&v, true));+ }++ #[test]+ fn sequential_descending_without_check_order_accepted() {+ let v: Vec<Value> = (1..=10).rev().map(|i| json!(i)).collect();+ assert!(detect_sequential_pattern(&v, false));+ }++ #[test]+ fn bug2_zero_padded_strings_no_longer_misclassified() {+ // BUG #2: ["001", "002", ..., "010"] — Python's original code+ // parsed each via int() and called this sequential. Fixed: every+ // numeric value here originated as a string, so we refuse.+ let v: Vec<Value> = (1..=10).map(|i| json!(format!("{:03}", i))).collect();+ assert!(+ !detect_sequential_pattern(&v, true),+ "BUG #2 fix: zero-padded string IDs must not be classified as sequential"+ );+ }++ #[test]+ fn bug2_mixed_string_and_int_still_detected() {+ // Sanity check the fix doesn't break the legitimate case: a+ // field that has BOTH genuine ints AND string-encoded ints+ // should still be detected (the unambiguous ints dominate the+ // signal).+ let v = vec![+ json!(1),+ json!(2),+ json!("3"),+ json!(4),+ json!(5),+ json!(6),+ ];+ assert!(detect_sequential_pattern(&v, true));+ }++ #[test]+ fn sequential_bools_excluded() {+ // Python: `isinstance(v, int | float) and not isinstance(v, bool)`+ // — bools never count as numeric.+ let v = vec![+ json!(true),+ json!(false),+ json!(true),+ json!(false),+ json!(true),+ json!(false),+ ];+ assert!(!detect_sequential_pattern(&v, true));+ }++ #[test]+ fn sequential_floats_with_unit_step() {+ let v: Vec<Value> = (1..=10).map(|i| json!(i as f64)).collect();+ assert!(detect_sequential_pattern(&v, true));+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/types.rs b/crates/headroom-core/src/transforms/smart_crusher/types.rsnew file mode 100644index 000000000..5eb82a44f--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/types.rs@@ -0,0 +1,244 @@+//! Core data types for SmartCrusher.+//!+//! Direct port of the dataclasses in `smart_crusher.py:318-924`. These+//! mirror the Python shapes 1:1 so the PyO3 bridge in stage 3c.1b can+//! reconstruct Python dataclasses from the Rust output without a manual+//! field-by-field translator.++use serde_json::Value;+use std::collections::BTreeMap;++/// Compression strategies based on data patterns.+///+/// Mirrors `CompressionStrategy` enum at `smart_crusher.py:318-326`. The+/// string variants must match Python's `Enum.value` exactly — they appear+/// in strategy debug strings (e.g. `"top_n(100->10)"`) and the parity+/// fixtures lock those bytes.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum CompressionStrategy {+ /// No compression needed.+ None,+ /// Explicitly skip — not safe to crush.+ Skip,+ /// Time-series: keep change points, summarize stable runs.+ TimeSeries,+ /// Cluster-sample: dedupe similar items.+ ClusterSample,+ /// Top-N: keep highest-scored items.+ TopN,+ /// Smart-sample: statistical sampling with anchor-preservation.+ SmartSample,+}++impl CompressionStrategy {+ /// Lowercase string matching Python's `Enum.value`. Pinned by the+ /// parity fixtures — must not drift.+ pub fn as_str(self) -> &'static str {+ match self {+ CompressionStrategy::None => "none",+ CompressionStrategy::Skip => "skip",+ CompressionStrategy::TimeSeries => "time_series",+ CompressionStrategy::ClusterSample => "cluster",+ CompressionStrategy::TopN => "top_n",+ CompressionStrategy::SmartSample => "smart_sample",+ }+ }+}++/// Statistics for a single field across array items.+///+/// Mirrors the `FieldStats` dataclass at `smart_crusher.py:864-885`.+/// Field naming and Optional<T> shape match Python exactly so the PyO3+/// bridge can `from_dict`-reconstruct the Python dataclass.+#[derive(Debug, Clone)]+pub struct FieldStats {+ pub name: String,+ /// One of: `"numeric"`, `"string"`, `"boolean"`, `"object"`, `"array"`,+ /// `"null"`. String literals match Python's `field_type` values.+ pub field_type: String,+ pub count: usize,+ pub unique_count: usize,+ pub unique_ratio: f64,+ pub is_constant: bool,+ pub constant_value: Option<Value>,++ // Numeric-specific+ pub min_val: Option<f64>,+ pub max_val: Option<f64>,+ pub mean_val: Option<f64>,+ pub variance: Option<f64>,+ pub change_points: Vec<usize>,++ // String-specific+ pub avg_length: Option<f64>,+ /// Top values by frequency, descending. Bounded list so this stays+ /// cheap to build and serialize. Same shape as Python's `list[tuple[str, int]]`.+ pub top_values: Vec<(String, usize)>,+}++/// Analysis of whether an array is safe to crush.+///+/// Mirrors `CrushabilityAnalysis` at `smart_crusher.py:833-860`. The key+/// invariant: **if we don't have a reliable signal to determine which+/// items are important, we don't crush at all**. Signals include score+/// fields, error keywords, numeric anomalies, and low uniqueness.+#[derive(Debug, Clone)]+pub struct CrushabilityAnalysis {+ pub crushable: bool,+ pub confidence: f64,+ pub reason: String,+ pub signals_present: Vec<String>,+ pub signals_absent: Vec<String>,++ // Detailed metrics (mirroring Python field-by-field)+ pub has_id_field: bool,+ pub id_uniqueness: f64,+ pub avg_string_uniqueness: f64,+ pub has_score_field: bool,+ pub error_item_count: usize,+ pub anomaly_count: usize,+}++impl CrushabilityAnalysis {+ /// Helper to build a "not crushable" verdict — used in several early+ /// exits in `analyze_crushability`. Mirrors the Python pattern where+ /// `crushable=False` paths don't bother filling in detail metrics.+ pub fn skip(reason: impl Into<String>, confidence: f64) -> Self {+ CrushabilityAnalysis {+ crushable: false,+ confidence,+ reason: reason.into(),+ signals_present: Vec::new(),+ signals_absent: Vec::new(),+ has_id_field: false,+ id_uniqueness: 0.0,+ avg_string_uniqueness: 0.0,+ has_score_field: false,+ error_item_count: 0,+ anomaly_count: 0,+ }+ }+}++/// Complete analysis of an array.+///+/// Mirrors `ArrayAnalysis` at `smart_crusher.py:887-897`. `field_stats`+/// uses `BTreeMap` for deterministic iteration order (Python's `dict`+/// preserves insertion order; `BTreeMap` gives us a stable sorted-by-key+/// order, which is fine for the parity fixtures because the analyzer+/// builds the map by iterating sorted keys).+#[derive(Debug, Clone)]+pub struct ArrayAnalysis {+ pub item_count: usize,+ pub field_stats: BTreeMap<String, FieldStats>,+ /// One of: `"time_series"`, `"logs"`, `"search_results"`, `"generic"`.+ pub detected_pattern: String,+ pub recommended_strategy: CompressionStrategy,+ pub constant_fields: BTreeMap<String, Value>,+ pub estimated_reduction: f64,+ pub crushability: Option<CrushabilityAnalysis>,+}++/// Plan for how to compress an array.+///+/// Mirrors `CompressionPlan` at `smart_crusher.py:900-910`. `keep_indices`+/// is the list of original-array indices that survive compression;+/// `summary_ranges` carries `(start, end, summary_dict)` for runs we+/// summarized rather than dropped (currently unused in the Python impl+/// but plumbed through for parity with the dataclass).+#[derive(Debug, Clone)]+pub struct CompressionPlan {+ pub strategy: CompressionStrategy,+ pub keep_indices: Vec<usize>,+ pub constant_fields: BTreeMap<String, Value>,+ /// `(start, end, summary)` triples for summarized runs. Python uses+ /// `list[tuple[int, int, dict]]`; we use `Value` for the summary so+ /// any JSON shape is representable.+ pub summary_ranges: Vec<(usize, usize, Value)>,+ pub cluster_field: Option<String>,+ pub sort_field: Option<String>,+ pub keep_count: usize,+}++impl Default for CompressionPlan {+ fn default() -> Self {+ // Mirrors Python's @dataclass defaults at line 900-910.+ CompressionPlan {+ strategy: CompressionStrategy::None,+ keep_indices: Vec::new(),+ constant_fields: BTreeMap::new(),+ summary_ranges: Vec::new(),+ cluster_field: None,+ sort_field: None,+ keep_count: 10,+ }+ }+}++/// Result from `SmartCrusher.crush()` — used by ContentRouter when+/// routing JSON arrays. Mirrors `CrushResult` at `smart_crusher.py:913-923`.+#[derive(Debug, Clone)]+pub struct CrushResult {+ pub compressed: String,+ pub original: String,+ pub was_modified: bool,+ pub strategy: String,+}++impl CrushResult {+ /// Pass-through result: same as input, no modification, strategy+ /// `"passthrough"`. Used when content can't be compressed (not JSON,+ /// too small, no crushable arrays, etc.).+ pub fn passthrough(content: impl Into<String>) -> Self {+ let s = content.into();+ CrushResult {+ compressed: s.clone(),+ original: s,+ was_modified: false,+ strategy: "passthrough".to_string(),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn compression_strategy_strings_match_python() {+ // Strategy debug strings appear in the parity fixtures; these must+ // not drift. If a value here changes, every fixture breaks.+ assert_eq!(CompressionStrategy::None.as_str(), "none");+ assert_eq!(CompressionStrategy::Skip.as_str(), "skip");+ assert_eq!(CompressionStrategy::TimeSeries.as_str(), "time_series");+ assert_eq!(CompressionStrategy::ClusterSample.as_str(), "cluster");+ assert_eq!(CompressionStrategy::TopN.as_str(), "top_n");+ assert_eq!(CompressionStrategy::SmartSample.as_str(), "smart_sample");+ }++ #[test]+ fn crushability_skip_helper() {+ let r = CrushabilityAnalysis::skip("too small", 1.0);+ assert!(!r.crushable);+ assert_eq!(r.confidence, 1.0);+ assert_eq!(r.reason, "too small");+ }++ #[test]+ fn compression_plan_default_keep_count_matches_python() {+ // Python's @dataclass default is `keep_count: int = 10`.+ let p = CompressionPlan::default();+ assert_eq!(p.keep_count, 10);+ assert_eq!(p.strategy, CompressionStrategy::None);+ assert!(p.keep_indices.is_empty());+ }++ #[test]+ fn crush_result_passthrough() {+ let r = CrushResult::passthrough("hello");+ assert_eq!(r.compressed, "hello");+ assert_eq!(r.original, "hello");+ assert!(!r.was_modified);+ assert_eq!(r.strategy, "passthrough");+ }+}diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.jsonindex 5413f88ea..c99f01e98 100644--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.11.0",+ "version": "0.10.17","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.jsonindex 8b69de7cb..251e19767 100644--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.11.0",+ "version": "0.10.17","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",