Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): scaffold smart_crusher module + foundational helpers

chopratejasd219beecab01
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
index 78381697b..579b17788 100644
--- a/.claude-plugin/marketplace.json
+++ b/.claude-plugin/marketplace.json
@@ -5,14 +5,14 @@
},
"metadata": {
"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
- "version": "0.11.0"
+ "version": "0.10.17"
},
"plugins": [
{
"name": "headroom",
"source": "./plugins/headroom-agent-hooks",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "version": "0.11.0",
+ "version": "0.10.17",
"author": {
"name": "Headroom Contributors",
"url": "https://github.com/chopratejas/headroom"
diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json
index 78381697b..579b17788 100644
--- a/.github/plugin/marketplace.json
+++ b/.github/plugin/marketplace.json
@@ -5,14 +5,14 @@
},
"metadata": {
"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
- "version": "0.11.0"
+ "version": "0.10.17"
},
"plugins": [
{
"name": "headroom",
"source": "./plugins/headroom-agent-hooks",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "version": "0.11.0",
+ "version": "0.10.17",
"author": {
"name": "Headroom Contributors",
"url": "https://github.com/chopratejas/headroom"
diff --git a/Cargo.lock b/Cargo.lock
index cbf163166..4d52bfd86 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -933,6 +933,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
+ "sha2",
"thiserror 1.0.69",
"tiktoken-rs",
"tokenizers",
@@ -2238,6 +2239,17 @@ dependencies = [
"digest",
]
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
[[package]]
name = "sharded-slab"
version = "0.1.7"
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml
index 929230211..d522455d7 100644
--- a/crates/headroom-core/Cargo.toml
+++ b/crates/headroom-core/Cargo.toml
@@ -26,6 +26,9 @@ hf-hub = { version = "0.4", default-features = false, features = ["ureq", "rustl
# `md5` for the CCR cache_key. Python's compression_store hashes the original
# diff with MD5 truncated to 24 hex chars; we must match byte-for-byte.
md-5 = "0.10"
+# `sha2` for `_hash_field_name` in smart_crusher (SHA256 truncated to 16
+# hex chars). Python uses `hashlib.sha256` so we need byte-exact parity.
+sha2 = "0.10"
# `regex` is already a transitive dep of tokenizers; depend on it directly so
# our hunk-header parser and priority-pattern matcher have a stable surface.
regex = "1"
diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rs
index f0fc4bc49..abaf33363 100644
--- a/crates/headroom-core/src/transforms/mod.rs
+++ b/crates/headroom-core/src/transforms/mod.rs
@@ -16,6 +16,7 @@
//! prod and are returned alongside the parity-equal output for tests.
pub mod diff_compressor;
+pub mod smart_crusher;
pub use diff_compressor::{
DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
diff --git a/crates/headroom-core/src/transforms/smart_crusher/anchors.rs b/crates/headroom-core/src/transforms/smart_crusher/anchors.rs
new file mode 100644
index 000000000..6a6f55eae
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/anchors.rs
@@ -0,0 +1,251 @@
+//! Legacy regex-based query anchor extraction.
+//!
+//! Direct port of `extract_query_anchors` and `item_matches_anchors`
+//! (`smart_crusher.py:99-168`). The Python doc-comment marks both as
+//! DEPRECATED in favor of `RelevanceScorer`, but they're still called
+//! by the live SmartCrusher path on every invocation, so we port them
+//! faithfully.
+//!
+//! # Why regex parity matters
+//!
+//! These regexes drive which array items survive compression. A subtle
+//! difference between Python's `re` engine and Rust's `regex` crate
+//! (e.g. word-boundary behavior on Unicode, or repetition greediness)
+//! would silently change which anchors are detected and which items
+//! survive. The patterns below are pinned to lowercase ASCII inputs
+//! and use only ASCII-safe constructs to keep behavior identical.
+
+use regex::Regex;
+use serde_json::Value;
+use std::collections::HashSet;
+use std::sync::LazyLock;
+
+// ---------------------------------------------------------------
+// Pattern definitions — direct ports of the module-level Python regexes
+// at `smart_crusher.py:85-93`. `std::sync::LazyLock` (stable since Rust
+// 1.80) is the modern equivalent of `once_cell::sync::Lazy`, mirroring
+// Python's `re.compile` at module import time.
+// ---------------------------------------------------------------
+
+/// `\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`
+static UUID_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
+ .expect("UUID_PATTERN")
+});
+
+/// 4+ digit numbers (likely IDs). Python: `r"\b\d{4,}\b"`.
+static NUMERIC_ID_PATTERN: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"\b\d{4,}\b").expect("NUMERIC_ID_PATTERN"));
+
+/// Hostname pattern. Matches `host.tld` with optional `.tld2`. Python:
+/// `r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b"`.
+static HOSTNAME_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b")
+ .expect("HOSTNAME_PATTERN")
+});
+
+/// Short quoted strings (single OR double quotes), 1-50 chars between
+/// quotes. Python: `r"['\"]([^'\"]{1,50})['\"]"`.
+static QUOTED_STRING_PATTERN: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r#"['"]([^'"]{1,50})['"]"#).expect("QUOTED_STRING_PATTERN"));
+
+/// Email addresses. Python: `r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"`.
+/// (Note Python's `[A-Z|a-z]` includes a literal `|` in the character
+/// class — almost certainly a typo, but we faithfully port it for
+/// parity. Real-world impact is nil since `|` doesn't appear in TLDs.)
+static EMAIL_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b").expect("EMAIL_PATTERN")
+});
+
+/// Hostname false-positive blocklist. Python uses a set literal at
+/// `smart_crusher.py:137`. We mirror exactly — these strings get
+/// dropped from anchor results.
+const HOSTNAME_FALSE_POSITIVES: &[&str] = &["e.g", "i.e", "etc."];
+
+/// Extract query anchors from user text. **DEPRECATED** in Python in
+/// favor of `RelevanceScorer`, but still called by the live path —
+/// ported as-is.
+///
+/// Output is a set of lowercased anchor strings. Order is not
+/// significant (Python returns `set[str]`).
+pub fn extract_query_anchors(text: &str) -> HashSet<String> {
+ let mut anchors = HashSet::new();
+
+ if text.is_empty() {
+ return anchors;
+ }
+
+ // UUIDs — lowercase the match.
+ for m in UUID_PATTERN.find_iter(text) {
+ anchors.insert(m.as_str().to_lowercase());
+ }
+
+ // Numeric IDs — Python keeps original case (digits, no transform needed).
+ for m in NUMERIC_ID_PATTERN.find_iter(text) {
+ anchors.insert(m.as_str().to_string());
+ }
+
+ // Hostnames — lowercase, filter false positives.
+ for m in HOSTNAME_PATTERN.find_iter(text) {
+ let lc = m.as_str().to_lowercase();
+ if !HOSTNAME_FALSE_POSITIVES.contains(&lc.as_str()) {
+ anchors.insert(lc);
+ }
+ }
+
+ // Quoted strings — capture group 1 (the content between quotes),
+ // require trim().len() >= 2 (Python's `if len(match.strip()) >= 2`).
+ for caps in QUOTED_STRING_PATTERN.captures_iter(text) {
+ if let Some(inner) = caps.get(1) {
+ if inner.as_str().trim().len() >= 2 {
+ anchors.insert(inner.as_str().to_lowercase());
+ }
+ }
+ }
+
+ // Emails — lowercase.
+ for m in EMAIL_PATTERN.find_iter(text) {
+ anchors.insert(m.as_str().to_lowercase());
+ }
+
+ anchors
+}
+
+/// Check if a JSON object matches any query anchors.
+///
+/// Direct port of `item_matches_anchors` (Python `smart_crusher.py:152-168`).
+/// Python uses `str(item).lower()` which produces Python's `dict.__str__`
+/// representation. We mirror by serializing with `serde_json` and
+/// lowercasing — this isn't byte-identical to Python's `str(dict)`
+/// (Python uses single quotes, JSON uses double; Python's bool is
+/// `True`/`False`, JSON's is `true`/`false`), so for cross-language
+/// parity we need a string form that matches Python's. We document this
+/// gap and fix it in the analyzer integration.
+///
+/// **WARNING:** `str(item).lower()` in Python produces:
+/// `{'key': 'value', 'count': 5, 'ok': True}`
+/// while `serde_json::to_string(&item)` produces:
+/// `{"key":"value","count":5,"ok":true}`
+///
+/// The anchor matching is substring-based (`anchor in item_str`), so
+/// this difference matters: if an anchor is `"true"` it matches the
+/// JSON form but not the Python form, and vice versa for `"True"`.
+///
+/// **Resolution:** when items reach the matcher they're already
+/// lowercased, so `True` → `true` after `.lower()`, removing one source
+/// of drift. The remaining drift (single vs double quotes, trailing
+/// whitespace) is unlikely to affect anchor matching in practice. We
+/// pin behavior with fixtures and move on.
+pub fn item_matches_anchors(item: &Value, anchors: &HashSet<String>) -> bool {
+ if anchors.is_empty() {
+ return false;
+ }
+
+ // Python: `str(item).lower()`. We approximate via JSON serialization
+ // followed by `.lower()` — see WARNING above for the gap.
+ let item_str = match serde_json::to_string(item) {
+ Ok(s) => s.to_lowercase(),
+ Err(_) => return false,
+ };
+
+ anchors.iter().any(|a| item_str.contains(a))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn empty_text_no_anchors() {
+ assert!(extract_query_anchors("").is_empty());
+ }
+
+ #[test]
+ fn extracts_uuid_lowercased() {
+ let anchors = extract_query_anchors("see id 550E8400-E29B-41D4-A716-446655440000 plz");
+ assert!(anchors.contains("550e8400-e29b-41d4-a716-446655440000"));
+ }
+
+ #[test]
+ fn extracts_numeric_id_unchanged() {
+ let anchors = extract_query_anchors("user 12345 reported issue");
+ assert!(anchors.contains("12345"));
+ }
+
+ #[test]
+ fn three_digit_number_not_anchor() {
+ // Pattern requires 4+ digits.
+ let anchors = extract_query_anchors("user 123 reported issue");
+ assert!(!anchors.iter().any(|a| a == "123"));
+ }
+
+ #[test]
+ fn extracts_hostname() {
+ let anchors = extract_query_anchors("connect to api.example.com asap");
+ assert!(anchors.contains("api.example.com"));
+ }
+
+ #[test]
+ fn hostname_false_positive_filtered() {
+ // "e.g" is in the blocklist — must NOT appear as an anchor even
+ // though it matches the regex.
+ let anchors = extract_query_anchors("test e.g.com endpoint");
+ // "e.g" is filtered, but "e.g.com" or other longer matches may
+ // pass; we only assert "e.g" itself is gone.
+ assert!(!anchors.contains("e.g"));
+ }
+
+ #[test]
+ fn extracts_quoted_string_double() {
+ let anchors = extract_query_anchors(r#"find the "user_name" field"#);
+ assert!(anchors.contains("user_name"));
+ }
+
+ #[test]
+ fn extracts_quoted_string_single() {
+ let anchors = extract_query_anchors("find the 'user_name' field");
+ assert!(anchors.contains("user_name"));
+ }
+
+ #[test]
+ fn very_short_quoted_skipped() {
+ // Less than 2 chars after trim — skipped.
+ let anchors = extract_query_anchors(r#"the "x" thing"#);
+ assert!(!anchors.contains("x"));
+ }
+
+ #[test]
+ fn extracts_email() {
+ let anchors = extract_query_anchors("contact [email protected] please");
+ assert!(anchors.contains("[email protected]"));
+ }
+
+ #[test]
+ fn item_matches_anchors_empty_set() {
+ let empty = HashSet::new();
+ assert!(!item_matches_anchors(&json!({"a": 1}), &empty));
+ }
+
+ #[test]
+ fn item_matches_anchor_in_value() {
+ let anchors: HashSet<String> = ["alice".to_string()].into_iter().collect();
+ assert!(item_matches_anchors(&json!({"name": "Alice"}), &anchors));
+ }
+
+ #[test]
+ fn item_matches_anchor_in_key() {
+ let anchors: HashSet<String> = ["status".to_string()].into_iter().collect();
+ // The anchor "status" appears in the JSON-serialized key.
+ assert!(item_matches_anchors(
+ &json!({"status": "ok"}),
+ &anchors
+ ));
+ }
+
+ #[test]
+ fn item_no_match_with_unrelated_anchor() {
+ let anchors: HashSet<String> = ["xyz123".to_string()].into_iter().collect();
+ assert!(!item_matches_anchors(&json!({"a": "b"}), &anchors));
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/classifier.rs b/crates/headroom-core/src/transforms/smart_crusher/classifier.rs
new file mode 100644
index 000000000..d8a04892d
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/classifier.rs
@@ -0,0 +1,203 @@
+//! JSON array element-type classification.
+//!
+//! Direct port of `_classify_array` (Python `smart_crusher.py:341-368`).
+//! Classification drives compression strategy: dict arrays go through
+//! `_crush_array`, string arrays through `_crush_string_array`, etc.
+//!
+//! # Python parity note: bool vs int
+//!
+//! Python's `True`/`False` are an int subclass, so a list `[True, False, 1]`
+//! has `types == {bool, int}` but `[True, False]` has `types == {bool}`.
+//! The Python code uses two checks to disambiguate:
+//! 1. `has_bool` flag set during the type-walk
+//! 2. `all(isinstance(i, bool) for i in items)` for pure-bool arrays
+//!
+//! The Rust `serde_json::Value` enum has separate `Bool` and `Number`
+//! variants — no inheritance — so the disambiguation is naturally cleaner
+//! here. We still walk every element (not a sample) to guarantee correct
+//! classification on adversarial inputs.
+
+use serde_json::Value;
+
+/// JSON array element type classification.
+///
+/// Mirrors Python's `ArrayType` enum at `smart_crusher.py:329-338`. The
+/// string variants in `Display`/`Debug` match Python's lowercase `value=`
+/// strings exactly, which is required for parity with serialized strategy
+/// debug output (e.g. `"dict_array(100->10)"`).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum ArrayType {
+ /// `[{...}, {...}, ...]` — dict array, full statistical path.
+ DictArray,
+ /// `["a", "b", "c", ...]` — string array.
+ StringArray,
+ /// `[1, 2.5, 3, ...]` — number array (excludes bools).
+ NumberArray,
+ /// `[true, false, ...]` — pure bool array.
+ BoolArray,
+ /// `[[...], [...], ...]` — array of arrays.
+ NestedArray,
+ /// Anything else: heterogeneous or unclassifiable.
+ MixedArray,
+ /// `[]` — empty array.
+ Empty,
+}
+
+impl ArrayType {
+ /// Lowercase string representation matching Python's `Enum.value`.
+ /// Used in strategy debug strings; must match Python exactly.
+ pub fn as_str(self) -> &'static str {
+ match self {
+ ArrayType::DictArray => "dict_array",
+ ArrayType::StringArray => "string_array",
+ ArrayType::NumberArray => "number_array",
+ ArrayType::BoolArray => "bool_array",
+ ArrayType::NestedArray => "nested_array",
+ ArrayType::MixedArray => "mixed_array",
+ ArrayType::Empty => "empty",
+ }
+ }
+}
+
+/// Classify a JSON array by its element types.
+///
+/// Walks every element (not a sample) to guarantee correct classification
+/// even on adversarial inputs where the first few items hide a type
+/// transition deeper in the list. `Value::is_*` is O(1), so the full
+/// walk is fine.
+///
+/// Returns `ArrayType::Empty` for an empty slice.
+pub fn classify_array(items: &[Value]) -> ArrayType {
+ if items.is_empty() {
+ return ArrayType::Empty;
+ }
+
+ // Track which Value variants we've seen. We collapse Number into
+ // either "int-like" or "float-like" once below; here we only need to
+ // know whether there's at least one of each high-level kind.
+ let mut has_bool = false;
+ let mut has_number = false;
+ let mut has_string = false;
+ let mut has_object = false;
+ let mut has_array = false;
+ let mut has_null = false;
+
+ for item in items {
+ match item {
+ Value::Bool(_) => has_bool = true,
+ Value::Number(_) => has_number = true,
+ Value::String(_) => has_string = true,
+ Value::Object(_) => has_object = true,
+ Value::Array(_) => has_array = true,
+ Value::Null => has_null = true,
+ }
+ }
+
+ // Pure bool array — Python's check is `all(isinstance(i, bool))`.
+ // Note Python `[True, False, 1]` evaluates to `types == {bool, int}`
+ // because bool is an int subclass; that maps to MixedArray here.
+ if has_bool && !has_number && !has_string && !has_object && !has_array && !has_null {
+ return ArrayType::BoolArray;
+ }
+
+ // Pure dict array.
+ if has_object && !has_bool && !has_number && !has_string && !has_array && !has_null {
+ return ArrayType::DictArray;
+ }
+
+ // Pure string array.
+ if has_string && !has_bool && !has_number && !has_object && !has_array && !has_null {
+ return ArrayType::StringArray;
+ }
+
+ // Pure number array — Python explicitly excludes bool here.
+ if has_number && !has_bool && !has_string && !has_object && !has_array && !has_null {
+ return ArrayType::NumberArray;
+ }
+
+ // Pure nested array.
+ if has_array && !has_bool && !has_number && !has_string && !has_object && !has_null {
+ return ArrayType::NestedArray;
+ }
+
+ // Anything else — heterogeneous types, or types involving null.
+ ArrayType::MixedArray
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn empty_array() {
+ let items: Vec<Value> = vec![];
+ assert_eq!(classify_array(&items), ArrayType::Empty);
+ }
+
+ #[test]
+ fn pure_dict_array() {
+ let items = vec![json!({"a": 1}), json!({"b": 2})];
+ assert_eq!(classify_array(&items), ArrayType::DictArray);
+ }
+
+ #[test]
+ fn pure_string_array() {
+ let items = vec![json!("a"), json!("b"), json!("c")];
+ assert_eq!(classify_array(&items), ArrayType::StringArray);
+ }
+
+ #[test]
+ fn pure_number_array_int_and_float() {
+ let items = vec![json!(1), json!(2.5), json!(3)];
+ assert_eq!(classify_array(&items), ArrayType::NumberArray);
+ }
+
+ #[test]
+ fn pure_bool_array() {
+ let items = vec![json!(true), json!(false), json!(true)];
+ assert_eq!(classify_array(&items), ArrayType::BoolArray);
+ }
+
+ #[test]
+ fn nested_array() {
+ let items = vec![json!([1, 2]), json!([3, 4])];
+ assert_eq!(classify_array(&items), ArrayType::NestedArray);
+ }
+
+ #[test]
+ fn mixed_dict_and_string_is_mixed() {
+ let items = vec![json!({"a": 1}), json!("str")];
+ assert_eq!(classify_array(&items), ArrayType::MixedArray);
+ }
+
+ #[test]
+ fn bool_with_number_is_mixed_not_bool_or_number() {
+ // Python's `[True, False, 1]` matches `types <= {bool, int}` BUT
+ // fails `all(isinstance(i, bool))`, so falls through to NUMBER_ARRAY
+ // check which has `not has_bool` — fails. Then nested check fails.
+ // Final: MIXED_ARRAY. Same here.
+ let items = vec![json!(true), json!(false), json!(1)];
+ assert_eq!(classify_array(&items), ArrayType::MixedArray);
+ }
+
+ #[test]
+ fn null_in_array_is_mixed() {
+ // Python's `types == {dict}` check fails when None (NoneType) is
+ // present, so a dict array with one null falls to MIXED_ARRAY.
+ let items = vec![json!({"a": 1}), json!(null)];
+ assert_eq!(classify_array(&items), ArrayType::MixedArray);
+ }
+
+ #[test]
+ fn as_str_matches_python_values() {
+ // Strategy debug strings depend on these exact lowercase forms.
+ assert_eq!(ArrayType::DictArray.as_str(), "dict_array");
+ assert_eq!(ArrayType::StringArray.as_str(), "string_array");
+ assert_eq!(ArrayType::NumberArray.as_str(), "number_array");
+ assert_eq!(ArrayType::BoolArray.as_str(), "bool_array");
+ assert_eq!(ArrayType::NestedArray.as_str(), "nested_array");
+ assert_eq!(ArrayType::MixedArray.as_str(), "mixed_array");
+ assert_eq!(ArrayType::Empty.as_str(), "empty");
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/config.rs b/crates/headroom-core/src/transforms/smart_crusher/config.rs
new file mode 100644
index 000000000..cb7959856
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/config.rs
@@ -0,0 +1,99 @@
+//! SmartCrusher configuration.
+//!
+//! Direct port of `SmartCrusherConfig` at `smart_crusher.py:927-957`. The
+//! defaults must match Python exactly — they're consulted everywhere
+//! during compression and any drift breaks parity fixtures.
+
+/// Configuration for SmartCrusher.
+///
+/// SCHEMA-PRESERVING: Output contains only items from the original array.
+/// No wrappers, no generated text, no metadata keys. (Python comment at
+/// line 930-931.)
+#[derive(Debug, Clone)]
+pub struct SmartCrusherConfig {
+ pub enabled: bool,
+ /// Don't analyze arrays smaller than this. Default 5.
+ pub min_items_to_analyze: usize,
+ /// Only crush content with more than this many tokens. Default 200.
+ pub min_tokens_to_crush: usize,
+ /// Standard deviations from the mean to count as a change point.
+ /// Default 2.0.
+ pub variance_threshold: f64,
+ /// Below this unique-ratio, a field is treated as nearly constant.
+ /// Default 0.1.
+ pub uniqueness_threshold: f64,
+ /// Similarity score above which strings cluster together. Default 0.8.
+ pub similarity_threshold: f64,
+ /// Target maximum items in the output. Default 15.
+ pub max_items_after_crush: usize,
+ /// Whether to preserve detected change points. Default true.
+ pub preserve_change_points: bool,
+ /// Factor out fields with constant values across all items. Default
+ /// false (disabled — preserves original schema).
+ pub factor_out_constants: bool,
+ /// Include generated text summaries in output. Default false (disabled
+ /// — no generated text).
+ pub include_summaries: bool,
+ /// Use feedback hints to adjust compression aggressiveness. Default true.
+ pub use_feedback_hints: bool,
+ /// Minimum confidence required to apply TOIN recommendations.
+ /// Default 0.5. (Python LOW FIX #21.)
+ pub toin_confidence_threshold: f64,
+ /// Drop content-identical items before sampling. Default true.
+ pub dedup_identical_items: bool,
+ /// Fraction of K to allocate to the start of the array. Default 0.3.
+ pub first_fraction: f64,
+ /// Fraction of K to allocate to the end of the array. Default 0.15.
+ pub last_fraction: f64,
+}
+
+impl Default for SmartCrusherConfig {
+ fn default() -> Self {
+ // These defaults must match smart_crusher.py:934-957 byte-for-byte.
+ SmartCrusherConfig {
+ enabled: true,
+ min_items_to_analyze: 5,
+ min_tokens_to_crush: 200,
+ variance_threshold: 2.0,
+ uniqueness_threshold: 0.1,
+ similarity_threshold: 0.8,
+ max_items_after_crush: 15,
+ preserve_change_points: true,
+ factor_out_constants: false,
+ include_summaries: false,
+ use_feedback_hints: true,
+ toin_confidence_threshold: 0.5,
+ dedup_identical_items: true,
+ first_fraction: 0.3,
+ last_fraction: 0.15,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn defaults_match_python() {
+ // Pin every default. Each field is consulted by some compression
+ // path and a drift would break parity. If Python ever changes a
+ // default, this test must be updated in lockstep.
+ let c = SmartCrusherConfig::default();
+ assert!(c.enabled);
+ assert_eq!(c.min_items_to_analyze, 5);
+ assert_eq!(c.min_tokens_to_crush, 200);
+ assert_eq!(c.variance_threshold, 2.0);
+ assert_eq!(c.uniqueness_threshold, 0.1);
+ assert_eq!(c.similarity_threshold, 0.8);
+ assert_eq!(c.max_items_after_crush, 15);
+ assert!(c.preserve_change_points);
+ assert!(!c.factor_out_constants);
+ assert!(!c.include_summaries);
+ assert!(c.use_feedback_hints);
+ assert_eq!(c.toin_confidence_threshold, 0.5);
+ assert!(c.dedup_identical_items);
+ assert_eq!(c.first_fraction, 0.3);
+ assert_eq!(c.last_fraction, 0.15);
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/hashing.rs b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs
new file mode 100644
index 000000000..4a4a404de
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs
@@ -0,0 +1,58 @@
+//! Field-name hashing for cache keys.
+//!
+//! Direct port of `_hash_field_name` (Python `smart_crusher.py:171-176`).
+//! Used to generate stable cache keys for compression hints — must match
+//! Python byte-for-byte or cache lookups will miss.
+
+use sha2::{Digest, Sha256};
+
+/// SHA-256 of the UTF-8 bytes, hex-encoded, truncated to 16 chars.
+///
+/// Python equivalent: `hashlib.sha256(field_name.encode()).hexdigest()[:16]`.
+/// We use lowercase hex (the default for both Python and Rust's `sha2`
+/// crate) — the test below pins this.
+pub fn hash_field_name(field_name: &str) -> String {
+ let mut hasher = Sha256::new();
+ hasher.update(field_name.as_bytes());
+ let digest = hasher.finalize();
+ // Convert to lowercase hex, then truncate to first 16 chars (8 bytes).
+ let hex = format!("{:x}", digest);
+ hex[..16].to_string()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn matches_python_sha256_truncated_to_16() {
+ // Verified against Python: hashlib.sha256(b"customer_id").hexdigest()[:16]
+ assert_eq!(hash_field_name("customer_id"), "1e38d67dbe8f47d2");
+ }
+
+ #[test]
+ fn empty_string() {
+ // Verified against Python: hashlib.sha256(b"").hexdigest()[:16]
+ assert_eq!(hash_field_name(""), "e3b0c44298fc1c14");
+ }
+
+ #[test]
+ fn unicode_field_name() {
+ // Verified against Python: hashlib.sha256("café".encode()).hexdigest()[:16]
+ // UTF-8 bytes for "café" are 63 61 66 c3 a9 — must encode same way.
+ assert_eq!(hash_field_name("café"), "850f7dc43910ff89");
+ }
+
+ #[test]
+ fn deterministic() {
+ // Same input → same output across calls.
+ assert_eq!(hash_field_name("test"), hash_field_name("test"));
+ }
+
+ #[test]
+ fn output_length_is_16() {
+ // Always exactly 16 hex chars regardless of input length.
+ assert_eq!(hash_field_name("a").len(), 16);
+ assert_eq!(hash_field_name(&"x".repeat(1000)).len(), 16);
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
new file mode 100644
index 000000000..9c408cb39
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
@@ -0,0 +1,50 @@
+//! Smart statistical tool output compression — Rust port of
+//! `headroom/transforms/smart_crusher.py`.
+//!
+//! # Stage 3c.1: like-for-like parity port
+//!
+//! This module is a literal Rust port of the Python `SmartCrusher`
+//! implementation. The goal of Stage 3c.1 is **byte-equal output parity** for
+//! every fixture in `tests/parity/fixtures/smart_crusher/`. Architectural
+//! improvements (lossless-first, unified saliency score, structured CCR
+//! markers, token budget) are deferred to Stage 3c.2 and tracked in
+//! `~/Desktop/SmartCrusher-Architecture-Improvements.md`.
+//!
+//! # Bugs fixed in BOTH Python and Rust during 3c.1
+//!
+//! Four defects in the Python source (`headroom/transforms/smart_crusher.py`)
+//! were caught during port review. They're fixed in both languages
+//! simultaneously so the parity fixtures continue to byte-match:
+//!
+//! - **k-split overshoot** (line 2722): `_compute_k_split` keeps 2 items when
+//! `k_total = 1` because `max(1, round(k_total * fraction))` floors both
+//! first and last to 1. Violates `max_items_after_crush`.
+//! - **sequential-pattern false positive** (line 444): `_detect_sequential_pattern`
+//! does `int("001")` and silently loses zero padding. Padded string IDs
+//! misclassified as sequential numeric IDs.
+//! - **rare-status detection short-circuit** (line 674): `_detect_rare_status_values`
+//! exits early at >10 distinct values. Datasets with 50+ error codes lose
+//! rare-error preservation.
+//! - **percentile off-by-one** (line 2844): For `len < 8`, integer-division
+//! percentile indices are off by one. Cosmetic — only affects strategy
+//! debug strings.
+//!
+//! Each fix has a fixture entry in the parity harness and a corresponding
+//! test in `tests/test_transforms/test_smart_crusher_bugs.py`.
+
+mod anchors;
+mod classifier;
+mod config;
+mod hashing;
+mod statistics;
+mod types;
+
+pub use anchors::{extract_query_anchors, item_matches_anchors};
+pub use classifier::{classify_array, ArrayType};
+pub use config::SmartCrusherConfig;
+pub use hashing::hash_field_name;
+pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
+pub use types::{
+ ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,
+ FieldStats,
+};
diff --git a/crates/headroom-core/src/transforms/smart_crusher/statistics.rs b/crates/headroom-core/src/transforms/smart_crusher/statistics.rs
new file mode 100644
index 000000000..d3530ddd2
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/statistics.rs
@@ -0,0 +1,375 @@
+//! Statistical helpers for field characterization.
+//!
+//! Direct port of the helpers at `smart_crusher.py:378-481`. These are
+//! used by the analyzer to classify fields (ID-like, score-like, etc.).
+//! Detection logic is heuristic; small numeric drift between Python and
+//! Rust would change classifications and break fixtures, so the math
+//! here mirrors Python step-by-step.
+
+use serde_json::Value;
+use std::collections::HashMap;
+
+/// Check if a string looks like a UUID.
+///
+/// Direct port of `_is_uuid_format` (Python `smart_crusher.py:378-392`).
+/// Format check only — no version-bit validation. Hex chars are lower
+/// or upper case, matching Python.
+pub fn is_uuid_format(value: &str) -> bool {
+ if value.len() != 36 {
+ return false;
+ }
+
+ // Expected segment lengths: 8-4-4-4-12.
+ let parts: Vec<&str> = value.split('-').collect();
+ if parts.len() != 5 {
+ return false;
+ }
+ let expected_lens = [8, 4, 4, 4, 12];
+ for (part, &expected_len) in parts.iter().zip(expected_lens.iter()) {
+ if part.len() != expected_len {
+ return false;
+ }
+ for c in part.chars() {
+ if !c.is_ascii_hexdigit() {
+ return false;
+ }
+ }
+ }
+ true
+}
+
+/// Shannon entropy of a string, normalized to `[0, 1]`.
+///
+/// Direct port of `_calculate_string_entropy` (`smart_crusher.py:395-423`).
+/// High entropy (>0.7) suggests random/ID-like content. Low entropy
+/// (<0.3) suggests repetitive/predictable content. Used by ID detection.
+///
+/// # Edge cases (matched to Python)
+/// - Empty or single-character strings return `0.0`.
+/// - All-identical chars: `freq` has 1 entry, `max_entropy = log2(min(1, n)) = 0.0`,
+/// we return `0.0` to avoid division by zero.
+pub fn calculate_string_entropy(s: &str) -> f64 {
+ // Python uses `len(s) < 2` and computes by character. Rust strings
+ // are UTF-8 so we iterate `chars()` to match Python's character-level
+ // semantics (Python iterates code points; Rust's `chars()` yields
+ // Unicode scalar values — same thing for non-surrogate text).
+ let n = s.chars().count();
+ if n < 2 {
+ return 0.0;
+ }
+
+ let mut freq: HashMap<char, usize> = HashMap::new();
+ for c in s.chars() {
+ *freq.entry(c).or_insert(0) += 1;
+ }
+
+ let length = n as f64;
+ let mut entropy = 0.0_f64;
+ for &count in freq.values() {
+ let p = count as f64 / length;
+ if p > 0.0 {
+ entropy -= p * p.log2();
+ }
+ }
+
+ // Normalize by the maximum possible entropy at this length:
+ // Python: max_entropy = log2(min(len(freq), length))
+ let max_entropy = (freq.len().min(n) as f64).log2();
+ if max_entropy > 0.0 {
+ entropy / max_entropy
+ } else {
+ 0.0
+ }
+}
+
+/// Detect if numeric values form a sequential pattern (like IDs:
+/// 1, 2, 3, ...).
+///
+/// Direct port of `_detect_sequential_pattern` (`smart_crusher.py:426-481`)
+/// **with BUG #2 FIXED**.
+///
+/// # Bug #2 — string-padding misclassification
+/// Python's original implementation calls `int("001") == 1` and silently
+/// loses zero padding, so a list of padded string IDs like
+/// `["001", "002", ..., "100"]` looks like a sequential numeric pattern
+/// when in reality it's a categorical string field where the padding
+/// matters. The fix: when a value is a string that parses as a number,
+/// flag the input as "had string-encoded numerics". If ALL parsed values
+/// originated as strings, refuse to classify as a sequential numeric
+/// pattern. Mixed numeric+string inputs still parse as sequential because
+/// the unambiguous numeric values dominate the signal.
+///
+/// This fix is applied in BOTH languages simultaneously (Python `smart_crusher.py`
+/// gets the same fix in the same PR) so the parity fixtures continue to
+/// match. Tests covering this bug live at
+/// `tests/test_transforms/test_smart_crusher_bugs.py`.
+///
+/// # Args
+/// - `values`: items to inspect.
+/// - `check_order`: when true, also require ascending order in the
+/// original array (the Python flag — IDs are usually ascending in
+/// source order, scores are usually descending).
+pub fn detect_sequential_pattern(values: &[Value], check_order: bool) -> bool {
+ if values.len() < 5 {
+ return false;
+ }
+
+ // Collect numeric values, tracking whether each value originated as
+ // a string. This is the BUG #2 fix: we still parse strings into
+ // numbers (so legitimate mixed-type fields work), but we'll refuse
+ // to flag the field as sequential if EVERY parseable value was a
+ // string.
+ let mut nums: Vec<f64> = Vec::new();
+ let mut had_non_string_numeric = false;
+
+ for v in values {
+ match v {
+ Value::Number(n) => {
+ if let Some(f) = n.as_f64() {
+ nums.push(f);
+ had_non_string_numeric = true;
+ }
+ }
+ Value::Bool(_) => {
+ // Python: `isinstance(v, int | float) and not isinstance(v, bool)` —
+ // bools are explicitly excluded.
+ }
+ Value::String(s) => {
+ // Python: `try: nums.append(int(v))`. `int("3.14")` raises
+ // in Python, so we mirror by trying integer parse first and
+ // only succeeding for pure-integer strings.
+ if let Ok(parsed) = s.parse::<i64>() {
+ nums.push(parsed as f64);
+ // BUG #2 fix: do NOT set had_non_string_numeric.
+ // If we later find this is the ONLY source of numeric
+ // values, we refuse to call it sequential.
+ }
+ }
+ _ => {}
+ }
+ }
+
+ if nums.len() < 5 {
+ return false;
+ }
+
+ // BUG #2 fix gate: if every numeric value originated as a string,
+ // the field is categorical (e.g. zero-padded codes); not sequential.
+ if !had_non_string_numeric {
+ return false;
+ }
+
+ // Need at least 2 elements for pairwise comparison. (Python checks
+ // this redundantly after `len(nums) < 5`.)
+ if nums.len() < 2 {
+ return false;
+ }
+
+ // Sort and compute pairwise diffs.
+ let mut sorted_nums = nums.clone();
+ sorted_nums.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+ let diffs: Vec<f64> = sorted_nums
+ .windows(2)
+ .map(|w| w[1] - w[0])
+ .collect();
+ if diffs.is_empty() {
+ return false;
+ }
+
+ let avg_diff: f64 = diffs.iter().sum::<f64>() / diffs.len() as f64;
+ if !(0.5..=2.0).contains(&avg_diff) {
+ return false;
+ }
+
+ // Most diffs in [0.5, 2.0] => sequential candidate.
+ let consistent_count = diffs.iter().filter(|&&d| (0.5..=2.0).contains(&d)).count();
+ let is_sequential = consistent_count as f64 / diffs.len() as f64 > 0.8;
+ if !is_sequential {
+ return false;
+ }
+
+ if check_order {
+ // Python: ascending count over original (not-sorted) sequence.
+ // IDs ascend in array order; scores typically descend.
+ let ascending_count = nums
+ .windows(2)
+ .filter(|w| w[0] <= w[1])
+ .count();
+ let n_pairs = nums.len() - 1;
+ let is_ascending = ascending_count as f64 / n_pairs as f64 > 0.7;
+ return is_ascending;
+ }
+
+ is_sequential
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ // ---------- is_uuid_format ----------
+
+ #[test]
+ fn uuid_format_canonical_lowercase() {
+ assert!(is_uuid_format("550e8400-e29b-41d4-a716-446655440000"));
+ }
+
+ #[test]
+ fn uuid_format_uppercase() {
+ assert!(is_uuid_format("550E8400-E29B-41D4-A716-446655440000"));
+ }
+
+ #[test]
+ fn uuid_format_wrong_length_rejected() {
+ assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000")); // 1 short
+ assert!(!is_uuid_format("550e8400-e29b-41d4-a716-4466554400000")); // 1 long
+ }
+
+ #[test]
+ fn uuid_format_wrong_segment_count() {
+ assert!(!is_uuid_format("550e8400e29b41d4a716446655440000"));
+ }
+
+ #[test]
+ fn uuid_format_non_hex_rejected() {
+ assert!(!is_uuid_format("550e8400-e29b-41d4-a716-44665544000z"));
+ }
+
+ #[test]
+ fn uuid_format_empty_rejected() {
+ assert!(!is_uuid_format(""));
+ }
+
+ // ---------- calculate_string_entropy ----------
+
+ #[test]
+ fn entropy_empty_string_is_zero() {
+ assert_eq!(calculate_string_entropy(""), 0.0);
+ }
+
+ #[test]
+ fn entropy_single_char_is_zero() {
+ assert_eq!(calculate_string_entropy("a"), 0.0);
+ }
+
+ #[test]
+ fn entropy_all_same_chars_is_zero() {
+ // "aaaa" — freq has 1 entry, max_entropy = log2(1) = 0.0,
+ // we return 0.0 from the guard.
+ assert_eq!(calculate_string_entropy("aaaa"), 0.0);
+ }
+
+ #[test]
+ fn entropy_perfectly_uniform_normalized_to_one() {
+ // Two distinct chars, 50/50: raw entropy = 1.0, max = log2(2) = 1.0,
+ // normalized = 1.0.
+ let e = calculate_string_entropy("ab");
+ assert!((e - 1.0).abs() < 1e-9);
+ }
+
+ #[test]
+ fn entropy_mostly_repeated_low() {
+ // "aaaaaab" — 6/7 'a', 1/7 'b' — should be small.
+ let e = calculate_string_entropy("aaaaaab");
+ assert!(e < 0.7);
+ }
+
+ #[test]
+ fn entropy_high_for_random_looking_string() {
+ // Approximation of a UUID-ish hex string. Should be > 0.7.
+ let e = calculate_string_entropy("a3f7b2c9d8e1f4a7");
+ assert!(e > 0.7);
+ }
+
+ // ---------- detect_sequential_pattern ----------
+
+ #[test]
+ fn sequential_simple_int_ascending() {
+ let v: Vec<Value> = (1..=10).map(|i| json!(i)).collect();
+ assert!(detect_sequential_pattern(&v, true));
+ }
+
+ #[test]
+ fn sequential_too_few_values() {
+ let v = vec![json!(1), json!(2), json!(3)];
+ assert!(!detect_sequential_pattern(&v, true));
+ }
+
+ #[test]
+ fn sequential_random_numbers_not_detected() {
+ let v: Vec<Value> = vec![
+ json!(100),
+ json!(2),
+ json!(85),
+ json!(7),
+ json!(43),
+ json!(17),
+ ];
+ assert!(!detect_sequential_pattern(&v, true));
+ }
+
+ #[test]
+ fn sequential_descending_with_check_order_rejected() {
+ // Descending sequence — if check_order=true, must NOT be flagged
+ // as sequential (Python: scores descend, IDs ascend).
+ let v: Vec<Value> = (1..=10).rev().map(|i| json!(i)).collect();
+ assert!(!detect_sequential_pattern(&v, true));
+ }
+
+ #[test]
+ fn sequential_descending_without_check_order_accepted() {
+ let v: Vec<Value> = (1..=10).rev().map(|i| json!(i)).collect();
+ assert!(detect_sequential_pattern(&v, false));
+ }
+
+ #[test]
+ fn bug2_zero_padded_strings_no_longer_misclassified() {
+ // BUG #2: ["001", "002", ..., "010"] — Python's original code
+ // parsed each via int() and called this sequential. Fixed: every
+ // numeric value here originated as a string, so we refuse.
+ let v: Vec<Value> = (1..=10).map(|i| json!(format!("{:03}", i))).collect();
+ assert!(
+ !detect_sequential_pattern(&v, true),
+ "BUG #2 fix: zero-padded string IDs must not be classified as sequential"
+ );
+ }
+
+ #[test]
+ fn bug2_mixed_string_and_int_still_detected() {
+ // Sanity check the fix doesn't break the legitimate case: a
+ // field that has BOTH genuine ints AND string-encoded ints
+ // should still be detected (the unambiguous ints dominate the
+ // signal).
+ let v = vec![
+ json!(1),
+ json!(2),
+ json!("3"),
+ json!(4),
+ json!(5),
+ json!(6),
+ ];
+ assert!(detect_sequential_pattern(&v, true));
+ }
+
+ #[test]
+ fn sequential_bools_excluded() {
+ // Python: `isinstance(v, int | float) and not isinstance(v, bool)`
+ // — bools never count as numeric.
+ let v = vec![
+ json!(true),
+ json!(false),
+ json!(true),
+ json!(false),
+ json!(true),
+ json!(false),
+ ];
+ assert!(!detect_sequential_pattern(&v, true));
+ }
+
+ #[test]
+ fn sequential_floats_with_unit_step() {
+ let v: Vec<Value> = (1..=10).map(|i| json!(i as f64)).collect();
+ assert!(detect_sequential_pattern(&v, true));
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/types.rs b/crates/headroom-core/src/transforms/smart_crusher/types.rs
new file mode 100644
index 000000000..5eb82a44f
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/types.rs
@@ -0,0 +1,244 @@
+//! Core data types for SmartCrusher.
+//!
+//! Direct port of the dataclasses in `smart_crusher.py:318-924`. These
+//! mirror the Python shapes 1:1 so the PyO3 bridge in stage 3c.1b can
+//! reconstruct Python dataclasses from the Rust output without a manual
+//! field-by-field translator.
+
+use serde_json::Value;
+use std::collections::BTreeMap;
+
+/// Compression strategies based on data patterns.
+///
+/// Mirrors `CompressionStrategy` enum at `smart_crusher.py:318-326`. The
+/// string variants must match Python's `Enum.value` exactly — they appear
+/// in strategy debug strings (e.g. `"top_n(100->10)"`) and the parity
+/// fixtures lock those bytes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum CompressionStrategy {
+ /// No compression needed.
+ None,
+ /// Explicitly skip — not safe to crush.
+ Skip,
+ /// Time-series: keep change points, summarize stable runs.
+ TimeSeries,
+ /// Cluster-sample: dedupe similar items.
+ ClusterSample,
+ /// Top-N: keep highest-scored items.
+ TopN,
+ /// Smart-sample: statistical sampling with anchor-preservation.
+ SmartSample,
+}
+
+impl CompressionStrategy {
+ /// Lowercase string matching Python's `Enum.value`. Pinned by the
+ /// parity fixtures — must not drift.
+ pub fn as_str(self) -> &'static str {
+ match self {
+ CompressionStrategy::None => "none",
+ CompressionStrategy::Skip => "skip",
+ CompressionStrategy::TimeSeries => "time_series",
+ CompressionStrategy::ClusterSample => "cluster",
+ CompressionStrategy::TopN => "top_n",
+ CompressionStrategy::SmartSample => "smart_sample",
+ }
+ }
+}
+
+/// Statistics for a single field across array items.
+///
+/// Mirrors the `FieldStats` dataclass at `smart_crusher.py:864-885`.
+/// Field naming and Optional<T> shape match Python exactly so the PyO3
+/// bridge can `from_dict`-reconstruct the Python dataclass.
+#[derive(Debug, Clone)]
+pub struct FieldStats {
+ pub name: String,
+ /// One of: `"numeric"`, `"string"`, `"boolean"`, `"object"`, `"array"`,
+ /// `"null"`. String literals match Python's `field_type` values.
+ pub field_type: String,
+ pub count: usize,
+ pub unique_count: usize,
+ pub unique_ratio: f64,
+ pub is_constant: bool,
+ pub constant_value: Option<Value>,
+
+ // Numeric-specific
+ pub min_val: Option<f64>,
+ pub max_val: Option<f64>,
+ pub mean_val: Option<f64>,
+ pub variance: Option<f64>,
+ pub change_points: Vec<usize>,
+
+ // String-specific
+ pub avg_length: Option<f64>,
+ /// Top values by frequency, descending. Bounded list so this stays
+ /// cheap to build and serialize. Same shape as Python's `list[tuple[str, int]]`.
+ pub top_values: Vec<(String, usize)>,
+}
+
+/// Analysis of whether an array is safe to crush.
+///
+/// Mirrors `CrushabilityAnalysis` at `smart_crusher.py:833-860`. The key
+/// invariant: **if we don't have a reliable signal to determine which
+/// items are important, we don't crush at all**. Signals include score
+/// fields, error keywords, numeric anomalies, and low uniqueness.
+#[derive(Debug, Clone)]
+pub struct CrushabilityAnalysis {
+ pub crushable: bool,
+ pub confidence: f64,
+ pub reason: String,
+ pub signals_present: Vec<String>,
+ pub signals_absent: Vec<String>,
+
+ // Detailed metrics (mirroring Python field-by-field)
+ pub has_id_field: bool,
+ pub id_uniqueness: f64,
+ pub avg_string_uniqueness: f64,
+ pub has_score_field: bool,
+ pub error_item_count: usize,
+ pub anomaly_count: usize,
+}
+
+impl CrushabilityAnalysis {
+ /// Helper to build a "not crushable" verdict — used in several early
+ /// exits in `analyze_crushability`. Mirrors the Python pattern where
+ /// `crushable=False` paths don't bother filling in detail metrics.
+ pub fn skip(reason: impl Into<String>, confidence: f64) -> Self {
+ CrushabilityAnalysis {
+ crushable: false,
+ confidence,
+ reason: reason.into(),
+ signals_present: Vec::new(),
+ signals_absent: Vec::new(),
+ has_id_field: false,
+ id_uniqueness: 0.0,
+ avg_string_uniqueness: 0.0,
+ has_score_field: false,
+ error_item_count: 0,
+ anomaly_count: 0,
+ }
+ }
+}
+
+/// Complete analysis of an array.
+///
+/// Mirrors `ArrayAnalysis` at `smart_crusher.py:887-897`. `field_stats`
+/// uses `BTreeMap` for deterministic iteration order (Python's `dict`
+/// preserves insertion order; `BTreeMap` gives us a stable sorted-by-key
+/// order, which is fine for the parity fixtures because the analyzer
+/// builds the map by iterating sorted keys).
+#[derive(Debug, Clone)]
+pub struct ArrayAnalysis {
+ pub item_count: usize,
+ pub field_stats: BTreeMap<String, FieldStats>,
+ /// One of: `"time_series"`, `"logs"`, `"search_results"`, `"generic"`.
+ pub detected_pattern: String,
+ pub recommended_strategy: CompressionStrategy,
+ pub constant_fields: BTreeMap<String, Value>,
+ pub estimated_reduction: f64,
+ pub crushability: Option<CrushabilityAnalysis>,
+}
+
+/// Plan for how to compress an array.
+///
+/// Mirrors `CompressionPlan` at `smart_crusher.py:900-910`. `keep_indices`
+/// is the list of original-array indices that survive compression;
+/// `summary_ranges` carries `(start, end, summary_dict)` for runs we
+/// summarized rather than dropped (currently unused in the Python impl
+/// but plumbed through for parity with the dataclass).
+#[derive(Debug, Clone)]
+pub struct CompressionPlan {
+ pub strategy: CompressionStrategy,
+ pub keep_indices: Vec<usize>,
+ pub constant_fields: BTreeMap<String, Value>,
+ /// `(start, end, summary)` triples for summarized runs. Python uses
+ /// `list[tuple[int, int, dict]]`; we use `Value` for the summary so
+ /// any JSON shape is representable.
+ pub summary_ranges: Vec<(usize, usize, Value)>,
+ pub cluster_field: Option<String>,
+ pub sort_field: Option<String>,
+ pub keep_count: usize,
+}
+
+impl Default for CompressionPlan {
+ fn default() -> Self {
+ // Mirrors Python's @dataclass defaults at line 900-910.
+ CompressionPlan {
+ strategy: CompressionStrategy::None,
+ keep_indices: Vec::new(),
+ constant_fields: BTreeMap::new(),
+ summary_ranges: Vec::new(),
+ cluster_field: None,
+ sort_field: None,
+ keep_count: 10,
+ }
+ }
+}
+
+/// Result from `SmartCrusher.crush()` — used by ContentRouter when
+/// routing JSON arrays. Mirrors `CrushResult` at `smart_crusher.py:913-923`.
+#[derive(Debug, Clone)]
+pub struct CrushResult {
+ pub compressed: String,
+ pub original: String,
+ pub was_modified: bool,
+ pub strategy: String,
+}
+
+impl CrushResult {
+ /// Pass-through result: same as input, no modification, strategy
+ /// `"passthrough"`. Used when content can't be compressed (not JSON,
+ /// too small, no crushable arrays, etc.).
+ pub fn passthrough(content: impl Into<String>) -> Self {
+ let s = content.into();
+ CrushResult {
+ compressed: s.clone(),
+ original: s,
+ was_modified: false,
+ strategy: "passthrough".to_string(),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn compression_strategy_strings_match_python() {
+ // Strategy debug strings appear in the parity fixtures; these must
+ // not drift. If a value here changes, every fixture breaks.
+ assert_eq!(CompressionStrategy::None.as_str(), "none");
+ assert_eq!(CompressionStrategy::Skip.as_str(), "skip");
+ assert_eq!(CompressionStrategy::TimeSeries.as_str(), "time_series");
+ assert_eq!(CompressionStrategy::ClusterSample.as_str(), "cluster");
+ assert_eq!(CompressionStrategy::TopN.as_str(), "top_n");
+ assert_eq!(CompressionStrategy::SmartSample.as_str(), "smart_sample");
+ }
+
+ #[test]
+ fn crushability_skip_helper() {
+ let r = CrushabilityAnalysis::skip("too small", 1.0);
+ assert!(!r.crushable);
+ assert_eq!(r.confidence, 1.0);
+ assert_eq!(r.reason, "too small");
+ }
+
+ #[test]
+ fn compression_plan_default_keep_count_matches_python() {
+ // Python's @dataclass default is `keep_count: int = 10`.
+ let p = CompressionPlan::default();
+ assert_eq!(p.keep_count, 10);
+ assert_eq!(p.strategy, CompressionStrategy::None);
+ assert!(p.keep_indices.is_empty());
+ }
+
+ #[test]
+ fn crush_result_passthrough() {
+ let r = CrushResult::passthrough("hello");
+ assert_eq!(r.compressed, "hello");
+ assert_eq!(r.original, "hello");
+ assert!(!r.was_modified);
+ assert_eq!(r.strategy, "passthrough");
+ }
+}
diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
index 5413f88ea..c99f01e98 100644
--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "headroom",
- "version": "0.11.0",
+ "version": "0.10.17",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"author": {
"name": "Headroom Contributors",
diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.json
index 8b69de7cb..251e19767 100644
--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json
+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "headroom",
- "version": "0.11.0",
+ "version": "0.10.17",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"author": {
"name": "Headroom Contributors",