Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(python+rust): smart_crusher bugs #1, #2, #3, #4 + sorted iteration

chopratejasc829dfa53997
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs
index e81b7c511..e60477cf1 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs
@@ -254,18 +254,17 @@ pub fn crush_number_array(
let mut sorted_finite: Vec<f64> = finite.clone();
sorted_finite.sort_by(f64::total_cmp);
- // BUG #1 (faithful port): integer-division index. For `len(sorted_finite) < 4`,
- // fall back to min/max. Otherwise `sorted_finite[len/4]` / `sorted_finite[3*len/4]`.
- let p25 = if sorted_finite.len() >= 4 {
- sorted_finite[sorted_finite.len() / 4]
- } else {
- finite_min(&finite)
- };
- let p75 = if sorted_finite.len() >= 4 {
- sorted_finite[3 * sorted_finite.len() / 4]
- } else {
- finite_max(&finite)
- };
+ // BUG #1 FIX (lockstep with Python `_percentile_linear`): replace
+ // integer-division indexing with proper linear interpolation.
+ // Matches numpy's "linear" method exactly:
+ // index = q * (n - 1)
+ // if integer: sorted[index]
+ // else: linear interpolate between floor and ceil
+ // The Python source's `_percentile_linear` helper uses the same
+ // formula; both languages now agree byte-for-byte on the strategy
+ // string's p25/p75 values.
+ let p25 = percentile_linear(&sorted_finite, 0.25);
+ let p75 = percentile_linear(&sorted_finite, 0.75);
// Outliers (>variance_threshold σ from mean).
let mut outlier_indices: BTreeSet<usize> = BTreeSet::new();
@@ -490,6 +489,24 @@ pub fn crush_object(
// ---------- helpers ----------
+/// Linear-interpolation percentile (numpy "linear" method).
+/// Mirrors Python's `_percentile_linear` helper for byte-equal
+/// strategy-string parity (BUG #1 FIX).
+fn percentile_linear(sorted_values: &[f64], q: f64) -> f64 {
+ let n = sorted_values.len();
+ if n == 0 {
+ return 0.0;
+ }
+ if n == 1 {
+ return sorted_values[0];
+ }
+ let pos = q * (n - 1) as f64;
+ let lo = pos as usize;
+ let hi = if lo + 1 < n { lo + 1 } else { lo };
+ let frac = pos - lo as f64;
+ sorted_values[lo] * (1.0 - frac) + sorted_values[hi] * frac
+}
+
fn finite_min(values: &[f64]) -> f64 {
values.iter().cloned().reduce(f64::min).unwrap_or(0.0)
}
@@ -782,23 +799,46 @@ mod tests {
// ---------- BUG #1 documentation test ----------
#[test]
- fn bug1_percentile_off_by_one_documented() {
- // Pin Python's buggy index choice for 9-element finite list:
- // sorted_finite[9 // 4] = sorted_finite[2] (3rd element).
- // sorted_finite[3 * 9 // 4] = sorted_finite[6] (7th element).
- //
- // For sorted [1,2,3,4,5,6,7,8,9]:
- // buggy p25 = sorted[2] = 3
- // buggy p75 = sorted[6] = 7
- //
- // Stage 3c.1 commit 7 will fix this in BOTH languages and
- // regenerate fixtures. Until then, this test pins parity.
+ fn bug1_percentile_proper_linear_interpolation() {
+ // BUG #1 FIX (Rust + Python in lockstep): proper linear-
+ // interpolation percentile. For sorted [1,2,3,4,5,6,7,8,9],
+ // n=9 so:
+ // p25 index = 0.25 * 8 = 2.0 → sorted[2] = 3.0
+ // p75 index = 0.75 * 8 = 6.0 → sorted[6] = 7.0
+ // (Both p25 and p75 land on integer indices for n=9.)
let mut items: Vec<Value> = (1..=9).map(|i| json!(i)).collect();
- // Pad past n>8 threshold and keep finite list exactly 9.
items.extend(vec![json!(null); 5]); // nulls drop out of `finite`
let (_out, strat) = crush_number_array(&items, &cfg(), 1.0);
- // Verify the buggy p25/p75 land in the strategy string.
assert!(strat.contains("p25=3"), "got: {}", strat);
assert!(strat.contains("p75=7"), "got: {}", strat);
}
+
+ #[test]
+ fn bug1_percentile_interpolates_when_index_non_integer() {
+ // For sorted [10, 20, 30, 40, 50] (n=5):
+ // p25 = 0.25 * 4 = 1.0 → sorted[1] = 20
+ // p75 = 0.75 * 4 = 3.0 → sorted[3] = 40
+ // For sorted with n=10, n=11, etc., the index is non-integer
+ // and we interpolate. Pin a case where interpolation actually
+ // happens to verify the fix.
+ // n=10 finite: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
+ // p25 = 0.25 * 9 = 2.25 → sorted[2] * 0.75 + sorted[3] * 0.25
+ // = 30 * 0.75 + 40 * 0.25 = 32.5
+ // p75 = 0.75 * 9 = 6.75 → sorted[6] * 0.25 + sorted[7] * 0.75
+ // = 70 * 0.25 + 80 * 0.75 = 77.5
+ let items: Vec<Value> = (1..=10).map(|i| json!(i * 10)).collect();
+ let (_out, strat) = crush_number_array(&items, &cfg(), 1.0);
+ // Pre-fix would have given p25=sorted[10/4]=sorted[2]=30 (wrong).
+ // Post-fix gives 32.5.
+ assert!(
+ strat.contains("p25=32.5"),
+ "expected proper-percentile p25=32.5, got: {}",
+ strat
+ );
+ assert!(
+ strat.contains("p75=77.5"),
+ "expected proper-percentile p75=77.5, got: {}",
+ strat
+ );
+ }
}
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 5d354a621..db8d284e8 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -286,6 +286,34 @@ def _get_preserve_field_values(
return matches
+def _percentile_linear(sorted_values: list[float], q: float) -> float:
+ """Linear-interpolation percentile, matching numpy's "linear" method.
+
+ BUG #1 FIX (replaces integer-division indexing in _crush_number_array
+ that was off by one for `len < 8`). Computes:
+ index = q * (n - 1)
+ if index is integer: return sorted_values[index]
+ else: linear interpolate between floor and ceil
+
+ Args:
+ sorted_values: Pre-sorted ascending list of finite numbers.
+ q: Quantile in [0, 1] (e.g. 0.25 for p25, 0.75 for p75).
+
+ Returns:
+ The interpolated quantile value. Empty input returns 0.0.
+ """
+ n = len(sorted_values)
+ if n == 0:
+ return 0.0
+ if n == 1:
+ return float(sorted_values[0])
+ pos = q * (n - 1)
+ lo = int(pos)
+ hi = lo + 1 if lo + 1 < n else lo
+ frac = pos - lo
+ return sorted_values[lo] * (1 - frac) + sorted_values[hi] * frac
+
+
def _item_has_preserve_field_match(
item: dict,
preserve_field_hashes: list[str],
@@ -436,20 +464,35 @@ def _detect_sequential_pattern(values: list[Any], check_order: bool = True) -> b
if len(values) < 5:
return False
- # Get numeric values
+ # BUG #2 FIX: track whether ANY value was already numeric (not
+ # just a stringified number). If every "number" came from `int(s)`
+ # of a string value, the field is almost certainly a categorical
+ # string ID — possibly zero-padded — and should NOT be flagged
+ # as sequential. Without this guard, `["001", "002", "003"]`
+ # silently becomes `[1, 2, 3]` and gets misclassified.
nums = []
+ had_non_string_numeric = False
for v in values:
if isinstance(v, int | float) and not isinstance(v, bool):
nums.append(v)
+ had_non_string_numeric = True
elif isinstance(v, str):
try:
nums.append(int(v))
+ # Intentionally do NOT set had_non_string_numeric —
+ # see fix doc above.
except ValueError:
pass
if len(nums) < 5:
return False
+ # BUG #2 FIX: if every parseable value came from a string, the
+ # field is categorical (zero-padded codes, alphanumeric IDs that
+ # happen to be int-parseable, etc.). Don't classify as sequential.
+ if not had_non_string_numeric:
+ return False
+
# Need at least 2 elements for pairwise comparison
if len(nums) < 2:
return False
@@ -670,8 +713,13 @@ def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> li
except Exception:
continue
- # Status field = low cardinality (2-10 distinct values)
- if not (2 <= len(unique_values) <= 10):
+ # BUG #3 FIX: cardinality cap raised from 10 to 50 so that
+ # higher-cardinality but Pareto-distributed fields (e.g.
+ # 60 INFO + 25 WARN + 15 distinct error codes) still get
+ # their rare values flagged. Above 50 distinct values, the
+ # field is almost certainly an ID/free-form column, not a
+ # status enum.
+ if not (2 <= len(unique_values) <= 50):
continue
# Count value frequencies
@@ -680,23 +728,39 @@ def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> li
key = str(v) if v is not None else "__none__"
value_counts[key] = value_counts.get(key, 0) + 1
- # Find the dominant value
if not value_counts:
continue
- max_count = max(value_counts.values())
total = len(values)
- # If one value dominates (>90%), others are interesting
- if max_count >= total * 0.9:
- dominant_value = max(value_counts.keys(), key=lambda k: value_counts[k])
+ # BUG #3 FIX: replace single-dominant check with Pareto top-K.
+ # Sort frequencies descending; tiebreak by key ascending so the
+ # outcome is deterministic when multiple values have the same
+ # frequency. Find the smallest K such that top-K covers >=80%
+ # of items.
+ sorted_counts = sorted(value_counts.items(), key=lambda x: (-x[1], x[0]))
+ threshold = math.ceil(total * 0.8)
+ cumulative = 0
+ top_k_values: set[str] = set()
+ for value, count in sorted_counts:
+ cumulative += count
+ top_k_values.add(value)
+ if cumulative >= threshold:
+ break
- for i, item in enumerate(items):
- if not isinstance(item, dict) or field_name not in item:
- continue
- item_value = str(item[field_name]) if item[field_name] is not None else "__none__"
- if item_value != dominant_value:
- outlier_indices.append(i)
+ # K must be small (<=5) for any value to count as "rare".
+ # Above this the distribution is too uniform to label a value
+ # rare.
+ if len(top_k_values) > 5:
+ continue
+
+ # Items NOT in top_k_values are outliers.
+ for i, item in enumerate(items):
+ if not isinstance(item, dict) or field_name not in item:
+ continue
+ item_value = str(item[field_name]) if item[field_name] is not None else "__none__"
+ if item_value not in top_k_values:
+ outlier_indices.append(i)
return outlier_indices
@@ -982,7 +1046,14 @@ class SmartAnalyzer:
if isinstance(item, dict):
all_keys.update(item.keys())
- for key in all_keys:
+ # PARITY FIX (Stage 3c.1): iterate in sorted key order. Python's
+ # set iteration is non-deterministic across PYTHONHASHSEED, so
+ # downstream short-circuits (`_select_strategy` "message" lookup,
+ # `_detect_pattern` first-score-field) become non-deterministic.
+ # Rust uses BTreeMap which iterates ASCII-sorted; sorting here
+ # locks both languages to the same iteration order so parity
+ # fixtures byte-match.
+ for key in sorted(all_keys):
field_stats[key] = self._analyze_field(key, items)
# Detect pattern
@@ -2719,8 +2790,14 @@ class SmartCrusher(Transform):
min_k=3,
max_k=self.config.max_items_after_crush or None,
)
+ # BUG #4 FIX: clamp k_first and k_last so their sum never exceeds
+ # k_total. Without the clamp, k_total=1 produces k_first=k_last=1
+ # (both `max(1, round(0))`), violating max_items_after_crush.
+ # No-op for k_total >= 2 (the common path).
k_first = max(1, round(k_total * self.config.first_fraction))
+ k_first = min(k_first, k_total)
k_last = max(1, round(k_total * self.config.last_fraction))
+ k_last = min(k_last, max(0, k_total - k_first))
k_importance = max(0, k_total - k_first - k_last)
return k_total, k_first, k_last, k_importance
@@ -2841,8 +2918,12 @@ class SmartCrusher(Transform):
median_val = statistics.median(finite)
std_val = statistics.stdev(finite) if len(finite) > 1 else 0.0
sorted_finite = sorted(finite)
- p25 = sorted_finite[len(sorted_finite) // 4] if len(sorted_finite) >= 4 else min(finite)
- p75 = sorted_finite[3 * len(sorted_finite) // 4] if len(sorted_finite) >= 4 else max(finite)
+ # BUG #1 FIX: replace integer-division indexing (off-by-one
+ # for len < 8) with proper linear-interpolation percentile.
+ # `_percentile_linear` matches numpy's "linear" method exactly
+ # — index = q * (n - 1), interpolate between floor and ceil.
+ p25 = _percentile_linear(sorted_finite, 0.25)
+ p75 = _percentile_linear(sorted_finite, 0.75)
# Outliers (> variance_threshold σ from mean)
outlier_indices: set[int] = set()
diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.py
index 00310dc40..e09f2deee 100644
--- a/tests/test_transforms/test_smart_crusher_bugs.py
+++ b/tests/test_transforms/test_smart_crusher_bugs.py
@@ -210,3 +210,81 @@ class TestRecursionDepthLimit:
result, was_modified, info = crusher._smart_crush_content(content)
parsed = json.loads(result)
assert isinstance(parsed, list)
+
+
+# ---------------------------------------------------------------------------
+# Stage 3c.1 lockstep bug fixes (#1 percentile, #2 sequential, #3 rare-status,
+# #4 k-split). Each test pins the Python behavior post-fix; Rust has matching
+# tests so parity fixtures byte-equal both languages.
+# ---------------------------------------------------------------------------
+
+
+class TestStage3c1BugFixes:
+ """Bugs fixed in lockstep with the Rust port at Stage 3c.1."""
+
+ # Bug #1 — percentile off-by-one (cosmetic, strategy string only).
+ def test_bug1_percentile_uses_linear_interpolation(self) -> None:
+ from headroom.transforms.smart_crusher import _percentile_linear
+
+ # n=10 [10..100]: p25 index = 0.25 * 9 = 2.25 → 30*0.75 + 40*0.25 = 32.5
+ sorted_vals = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
+ assert _percentile_linear(sorted_vals, 0.25) == 32.5
+ assert _percentile_linear(sorted_vals, 0.75) == 77.5
+ # n=1 → return the single value.
+ assert _percentile_linear([42.0], 0.25) == 42.0
+ # n=0 → 0.0 (defensive).
+ assert _percentile_linear([], 0.25) == 0.0
+
+ # Bug #2 — zero-padded string IDs misclassified as sequential.
+ def test_bug2_zero_padded_strings_not_sequential(self) -> None:
+ from headroom.transforms.smart_crusher import _detect_sequential_pattern
+
+ # Pre-fix: int("001") → 1, so ["001",...,"005"] looked like 1..5
+ # and was classified as sequential. Post-fix: had_non_string_numeric
+ # stays False because every value came from a string → return False.
+ zero_padded = ["001", "002", "003", "004", "005"]
+ assert _detect_sequential_pattern(zero_padded, check_order=False) is False
+
+ # Real ints (not strings) still classify as sequential.
+ real_ints = [1, 2, 3, 4, 5]
+ assert _detect_sequential_pattern(real_ints, check_order=False) is True
+
+ # Bug #3 — rare-status detection cardinality cap.
+ def test_bug3_high_cardinality_pareto(self) -> None:
+ from headroom.transforms.smart_crusher import _detect_rare_status_values
+
+ # 60×INFO + 25×WARN + 15 distinct error codes (cardinality 17).
+ # Pre-fix: 17 > 10 → field skipped → 0 outliers.
+ # Post-fix: top-2 covers 85% (60+25=85), K=2 ≤ 5, the 15 rare codes flagged.
+ items = []
+ for _ in range(60):
+ items.append({"code": "INFO"})
+ for _ in range(25):
+ items.append({"code": "WARN"})
+ for i in range(15):
+ items.append({"code": f"ERR_{i}"})
+ outliers = _detect_rare_status_values(items, common_fields={"code"})
+ assert len(outliers) == 15
+
+ def test_bug3_uniform_distribution_no_outliers(self) -> None:
+ # 50 distinct values, 1 each → top-K never reaches 80% with K<=5.
+ # Field correctly identified as non-categorical.
+ from headroom.transforms.smart_crusher import _detect_rare_status_values
+
+ items = [{"code": f"CAT_{i}"} for i in range(50)]
+ assert _detect_rare_status_values(items, common_fields={"code"}) == []
+
+ # Bug #4 — k-split overshoot when k_total=1.
+ def test_bug4_k_split_no_overshoot_when_k_total_one(self) -> None:
+ from headroom import SmartCrusherConfig
+ from headroom.transforms.smart_crusher import SmartCrusher
+
+ config = SmartCrusherConfig(min_items_to_analyze=1)
+ crusher = SmartCrusher(config=config)
+ # Force k_total=1 by passing a single-item list — the n<=8 fast
+ # path returns n=1, so k_total=1.
+ k_total, k_first, k_last, k_importance = crusher._compute_k_split(["only"], 1.0)
+ assert k_total == 1
+ assert k_first + k_last <= k_total, (
+ f"BUG #4: k_first={k_first} + k_last={k_last} must not exceed k_total={k_total}"
+ )