Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(python+rust): smart_crusher bugs #1, #2, #3, #4 + sorted iteration
chopratejasc829dfa53997
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs b/crates/headroom-core/src/transforms/smart_crusher/crushers.rsindex e81b7c511..e60477cf1 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs@@ -254,18 +254,17 @@ pub fn crush_number_array(let mut sorted_finite: Vec<f64> = finite.clone();sorted_finite.sort_by(f64::total_cmp);- // BUG #1 (faithful port): integer-division index. For `len(sorted_finite) < 4`,- // fall back to min/max. Otherwise `sorted_finite[len/4]` / `sorted_finite[3*len/4]`.- let p25 = if sorted_finite.len() >= 4 {- sorted_finite[sorted_finite.len() / 4]- } else {- finite_min(&finite)- };- let p75 = if sorted_finite.len() >= 4 {- sorted_finite[3 * sorted_finite.len() / 4]- } else {- finite_max(&finite)- };+ // BUG #1 FIX (lockstep with Python `_percentile_linear`): replace+ // integer-division indexing with proper linear interpolation.+ // Matches numpy's "linear" method exactly:+ // index = q * (n - 1)+ // if integer: sorted[index]+ // else: linear interpolate between floor and ceil+ // The Python source's `_percentile_linear` helper uses the same+ // formula; both languages now agree byte-for-byte on the strategy+ // string's p25/p75 values.+ let p25 = percentile_linear(&sorted_finite, 0.25);+ let p75 = percentile_linear(&sorted_finite, 0.75);// Outliers (>variance_threshold σ from mean).let mut outlier_indices: BTreeSet<usize> = BTreeSet::new();@@ -490,6 +489,24 @@ pub fn crush_object(// ---------- helpers ----------+/// Linear-interpolation percentile (numpy "linear" method).+/// Mirrors Python's `_percentile_linear` helper for byte-equal+/// strategy-string parity (BUG #1 FIX).+fn percentile_linear(sorted_values: &[f64], q: f64) -> f64 {+ let n = sorted_values.len();+ if n == 0 {+ return 0.0;+ }+ if n == 1 {+ return sorted_values[0];+ }+ let pos = q * (n - 1) as f64;+ let lo = pos as usize;+ let hi = if lo + 1 < n { lo + 1 } else { lo };+ let frac = pos - lo as f64;+ sorted_values[lo] * (1.0 - frac) + sorted_values[hi] * frac+}+fn finite_min(values: &[f64]) -> f64 {values.iter().cloned().reduce(f64::min).unwrap_or(0.0)}@@ -782,23 +799,46 @@ mod tests {// ---------- BUG #1 documentation test ----------#[test]- fn bug1_percentile_off_by_one_documented() {- // Pin Python's buggy index choice for 9-element finite list:- // sorted_finite[9 // 4] = sorted_finite[2] (3rd element).- // sorted_finite[3 * 9 // 4] = sorted_finite[6] (7th element).- //- // For sorted [1,2,3,4,5,6,7,8,9]:- // buggy p25 = sorted[2] = 3- // buggy p75 = sorted[6] = 7- //- // Stage 3c.1 commit 7 will fix this in BOTH languages and- // regenerate fixtures. Until then, this test pins parity.+ fn bug1_percentile_proper_linear_interpolation() {+ // BUG #1 FIX (Rust + Python in lockstep): proper linear-+ // interpolation percentile. For sorted [1,2,3,4,5,6,7,8,9],+ // n=9 so:+ // p25 index = 0.25 * 8 = 2.0 → sorted[2] = 3.0+ // p75 index = 0.75 * 8 = 6.0 → sorted[6] = 7.0+ // (Both p25 and p75 land on integer indices for n=9.)let mut items: Vec<Value> = (1..=9).map(|i| json!(i)).collect();- // Pad past n>8 threshold and keep finite list exactly 9.items.extend(vec![json!(null); 5]); // nulls drop out of `finite`let (_out, strat) = crush_number_array(&items, &cfg(), 1.0);- // Verify the buggy p25/p75 land in the strategy string.assert!(strat.contains("p25=3"), "got: {}", strat);assert!(strat.contains("p75=7"), "got: {}", strat);}++ #[test]+ fn bug1_percentile_interpolates_when_index_non_integer() {+ // For sorted [10, 20, 30, 40, 50] (n=5):+ // p25 = 0.25 * 4 = 1.0 → sorted[1] = 20+ // p75 = 0.75 * 4 = 3.0 → sorted[3] = 40+ // For sorted with n=10, n=11, etc., the index is non-integer+ // and we interpolate. Pin a case where interpolation actually+ // happens to verify the fix.+ // n=10 finite: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]+ // p25 = 0.25 * 9 = 2.25 → sorted[2] * 0.75 + sorted[3] * 0.25+ // = 30 * 0.75 + 40 * 0.25 = 32.5+ // p75 = 0.75 * 9 = 6.75 → sorted[6] * 0.25 + sorted[7] * 0.75+ // = 70 * 0.25 + 80 * 0.75 = 77.5+ let items: Vec<Value> = (1..=10).map(|i| json!(i * 10)).collect();+ let (_out, strat) = crush_number_array(&items, &cfg(), 1.0);+ // Pre-fix would have given p25=sorted[10/4]=sorted[2]=30 (wrong).+ // Post-fix gives 32.5.+ assert!(+ strat.contains("p25=32.5"),+ "expected proper-percentile p25=32.5, got: {}",+ strat+ );+ assert!(+ strat.contains("p75=77.5"),+ "expected proper-percentile p75=77.5, got: {}",+ strat+ );+ }}diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 5d354a621..db8d284e8 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -286,6 +286,34 @@ def _get_preserve_field_values(return matches+def _percentile_linear(sorted_values: list[float], q: float) -> float:+ """Linear-interpolation percentile, matching numpy's "linear" method.++ BUG #1 FIX (replaces integer-division indexing in _crush_number_array+ that was off by one for `len < 8`). Computes:+ index = q * (n - 1)+ if index is integer: return sorted_values[index]+ else: linear interpolate between floor and ceil++ Args:+ sorted_values: Pre-sorted ascending list of finite numbers.+ q: Quantile in [0, 1] (e.g. 0.25 for p25, 0.75 for p75).++ Returns:+ The interpolated quantile value. Empty input returns 0.0.+ """+ n = len(sorted_values)+ if n == 0:+ return 0.0+ if n == 1:+ return float(sorted_values[0])+ pos = q * (n - 1)+ lo = int(pos)+ hi = lo + 1 if lo + 1 < n else lo+ frac = pos - lo+ return sorted_values[lo] * (1 - frac) + sorted_values[hi] * frac++def _item_has_preserve_field_match(item: dict,preserve_field_hashes: list[str],@@ -436,20 +464,35 @@ def _detect_sequential_pattern(values: list[Any], check_order: bool = True) -> bif len(values) < 5:return False- # Get numeric values+ # BUG #2 FIX: track whether ANY value was already numeric (not+ # just a stringified number). If every "number" came from `int(s)`+ # of a string value, the field is almost certainly a categorical+ # string ID — possibly zero-padded — and should NOT be flagged+ # as sequential. Without this guard, `["001", "002", "003"]`+ # silently becomes `[1, 2, 3]` and gets misclassified.nums = []+ had_non_string_numeric = Falsefor v in values:if isinstance(v, int | float) and not isinstance(v, bool):nums.append(v)+ had_non_string_numeric = Trueelif isinstance(v, str):try:nums.append(int(v))+ # Intentionally do NOT set had_non_string_numeric —+ # see fix doc above.except ValueError:passif len(nums) < 5:return False+ # BUG #2 FIX: if every parseable value came from a string, the+ # field is categorical (zero-padded codes, alphanumeric IDs that+ # happen to be int-parseable, etc.). Don't classify as sequential.+ if not had_non_string_numeric:+ return False+# Need at least 2 elements for pairwise comparisonif len(nums) < 2:return False@@ -670,8 +713,13 @@ def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> liexcept Exception:continue- # Status field = low cardinality (2-10 distinct values)- if not (2 <= len(unique_values) <= 10):+ # BUG #3 FIX: cardinality cap raised from 10 to 50 so that+ # higher-cardinality but Pareto-distributed fields (e.g.+ # 60 INFO + 25 WARN + 15 distinct error codes) still get+ # their rare values flagged. Above 50 distinct values, the+ # field is almost certainly an ID/free-form column, not a+ # status enum.+ if not (2 <= len(unique_values) <= 50):continue# Count value frequencies@@ -680,23 +728,39 @@ def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> likey = str(v) if v is not None else "__none__"value_counts[key] = value_counts.get(key, 0) + 1- # Find the dominant valueif not value_counts:continue- max_count = max(value_counts.values())total = len(values)- # If one value dominates (>90%), others are interesting- if max_count >= total * 0.9:- dominant_value = max(value_counts.keys(), key=lambda k: value_counts[k])+ # BUG #3 FIX: replace single-dominant check with Pareto top-K.+ # Sort frequencies descending; tiebreak by key ascending so the+ # outcome is deterministic when multiple values have the same+ # frequency. Find the smallest K such that top-K covers >=80%+ # of items.+ sorted_counts = sorted(value_counts.items(), key=lambda x: (-x[1], x[0]))+ threshold = math.ceil(total * 0.8)+ cumulative = 0+ top_k_values: set[str] = set()+ for value, count in sorted_counts:+ cumulative += count+ top_k_values.add(value)+ if cumulative >= threshold:+ break- for i, item in enumerate(items):- if not isinstance(item, dict) or field_name not in item:- continue- item_value = str(item[field_name]) if item[field_name] is not None else "__none__"- if item_value != dominant_value:- outlier_indices.append(i)+ # K must be small (<=5) for any value to count as "rare".+ # Above this the distribution is too uniform to label a value+ # rare.+ if len(top_k_values) > 5:+ continue++ # Items NOT in top_k_values are outliers.+ for i, item in enumerate(items):+ if not isinstance(item, dict) or field_name not in item:+ continue+ item_value = str(item[field_name]) if item[field_name] is not None else "__none__"+ if item_value not in top_k_values:+ outlier_indices.append(i)return outlier_indices@@ -982,7 +1046,14 @@ class SmartAnalyzer:if isinstance(item, dict):all_keys.update(item.keys())- for key in all_keys:+ # PARITY FIX (Stage 3c.1): iterate in sorted key order. Python's+ # set iteration is non-deterministic across PYTHONHASHSEED, so+ # downstream short-circuits (`_select_strategy` "message" lookup,+ # `_detect_pattern` first-score-field) become non-deterministic.+ # Rust uses BTreeMap which iterates ASCII-sorted; sorting here+ # locks both languages to the same iteration order so parity+ # fixtures byte-match.+ for key in sorted(all_keys):field_stats[key] = self._analyze_field(key, items)# Detect pattern@@ -2719,8 +2790,14 @@ class SmartCrusher(Transform):min_k=3,max_k=self.config.max_items_after_crush or None,)+ # BUG #4 FIX: clamp k_first and k_last so their sum never exceeds+ # k_total. Without the clamp, k_total=1 produces k_first=k_last=1+ # (both `max(1, round(0))`), violating max_items_after_crush.+ # No-op for k_total >= 2 (the common path).k_first = max(1, round(k_total * self.config.first_fraction))+ k_first = min(k_first, k_total)k_last = max(1, round(k_total * self.config.last_fraction))+ k_last = min(k_last, max(0, k_total - k_first))k_importance = max(0, k_total - k_first - k_last)return k_total, k_first, k_last, k_importance@@ -2841,8 +2918,12 @@ class SmartCrusher(Transform):median_val = statistics.median(finite)std_val = statistics.stdev(finite) if len(finite) > 1 else 0.0sorted_finite = sorted(finite)- p25 = sorted_finite[len(sorted_finite) // 4] if len(sorted_finite) >= 4 else min(finite)- p75 = sorted_finite[3 * len(sorted_finite) // 4] if len(sorted_finite) >= 4 else max(finite)+ # BUG #1 FIX: replace integer-division indexing (off-by-one+ # for len < 8) with proper linear-interpolation percentile.+ # `_percentile_linear` matches numpy's "linear" method exactly+ # — index = q * (n - 1), interpolate between floor and ceil.+ p25 = _percentile_linear(sorted_finite, 0.25)+ p75 = _percentile_linear(sorted_finite, 0.75)# Outliers (> variance_threshold σ from mean)outlier_indices: set[int] = set()diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.pyindex 00310dc40..e09f2deee 100644--- a/tests/test_transforms/test_smart_crusher_bugs.py+++ b/tests/test_transforms/test_smart_crusher_bugs.py@@ -210,3 +210,81 @@ class TestRecursionDepthLimit:result, was_modified, info = crusher._smart_crush_content(content)parsed = json.loads(result)assert isinstance(parsed, list)+++# ---------------------------------------------------------------------------+# Stage 3c.1 lockstep bug fixes (#1 percentile, #2 sequential, #3 rare-status,+# #4 k-split). Each test pins the Python behavior post-fix; Rust has matching+# tests so parity fixtures byte-equal both languages.+# ---------------------------------------------------------------------------+++class TestStage3c1BugFixes:+ """Bugs fixed in lockstep with the Rust port at Stage 3c.1."""++ # Bug #1 — percentile off-by-one (cosmetic, strategy string only).+ def test_bug1_percentile_uses_linear_interpolation(self) -> None:+ from headroom.transforms.smart_crusher import _percentile_linear++ # n=10 [10..100]: p25 index = 0.25 * 9 = 2.25 → 30*0.75 + 40*0.25 = 32.5+ sorted_vals = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]+ assert _percentile_linear(sorted_vals, 0.25) == 32.5+ assert _percentile_linear(sorted_vals, 0.75) == 77.5+ # n=1 → return the single value.+ assert _percentile_linear([42.0], 0.25) == 42.0+ # n=0 → 0.0 (defensive).+ assert _percentile_linear([], 0.25) == 0.0++ # Bug #2 — zero-padded string IDs misclassified as sequential.+ def test_bug2_zero_padded_strings_not_sequential(self) -> None:+ from headroom.transforms.smart_crusher import _detect_sequential_pattern++ # Pre-fix: int("001") → 1, so ["001",...,"005"] looked like 1..5+ # and was classified as sequential. Post-fix: had_non_string_numeric+ # stays False because every value came from a string → return False.+ zero_padded = ["001", "002", "003", "004", "005"]+ assert _detect_sequential_pattern(zero_padded, check_order=False) is False++ # Real ints (not strings) still classify as sequential.+ real_ints = [1, 2, 3, 4, 5]+ assert _detect_sequential_pattern(real_ints, check_order=False) is True++ # Bug #3 — rare-status detection cardinality cap.+ def test_bug3_high_cardinality_pareto(self) -> None:+ from headroom.transforms.smart_crusher import _detect_rare_status_values++ # 60×INFO + 25×WARN + 15 distinct error codes (cardinality 17).+ # Pre-fix: 17 > 10 → field skipped → 0 outliers.+ # Post-fix: top-2 covers 85% (60+25=85), K=2 ≤ 5, the 15 rare codes flagged.+ items = []+ for _ in range(60):+ items.append({"code": "INFO"})+ for _ in range(25):+ items.append({"code": "WARN"})+ for i in range(15):+ items.append({"code": f"ERR_{i}"})+ outliers = _detect_rare_status_values(items, common_fields={"code"})+ assert len(outliers) == 15++ def test_bug3_uniform_distribution_no_outliers(self) -> None:+ # 50 distinct values, 1 each → top-K never reaches 80% with K<=5.+ # Field correctly identified as non-categorical.+ from headroom.transforms.smart_crusher import _detect_rare_status_values++ items = [{"code": f"CAT_{i}"} for i in range(50)]+ assert _detect_rare_status_values(items, common_fields={"code"}) == []++ # Bug #4 — k-split overshoot when k_total=1.+ def test_bug4_k_split_no_overshoot_when_k_total_one(self) -> None:+ from headroom import SmartCrusherConfig+ from headroom.transforms.smart_crusher import SmartCrusher++ config = SmartCrusherConfig(min_items_to_analyze=1)+ crusher = SmartCrusher(config=config)+ # Force k_total=1 by passing a single-item list — the n<=8 fast+ # path returns n=1, so k_total=1.+ k_total, k_first, k_last, k_importance = crusher._compute_k_split(["only"], 1.0)+ assert k_total == 1+ assert k_first + k_last <= k_total, (+ f"BUG #4: k_first={k_first} + k_last={k_last} must not exceed k_total={k_total}"+ )