Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): smart_crusher analyzer helpers — id/score/outlier detectors + bug #3 fix

chopratejasf029055229bb
diff --git a/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs b/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs
new file mode 100644
index 000000000..7661b8b2c
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs
@@ -0,0 +1,63 @@
+//! Canonical error keyword set for item preservation.
+//!
+//! Direct port of `ERROR_KEYWORDS` from `headroom/transforms/error_detection.py:18-33`.
+//! These are the **FALLBACK** preservation signal when TOIN field
+//! semantics aren't available yet (per the Python module-level
+//! comment). Intentionally broad — better to over-preserve than to
+//! drop a real error item.
+//!
+//! Used by `detect_error_items_for_preservation`. The list is small
+//! enough to keep as a `&[&str]`; if we ever cross ~50 keywords, switch
+//! to a `phf::Set` or pre-built FST for sub-linear lookup.
+
+/// 12 error/failure keywords. Order doesn't matter for correctness, but
+/// matches Python's set-literal order so reading both side-by-side is
+/// easier. Lowercase by construction; callers must lowercase the
+/// haystack before substring-matching.
+pub const ERROR_KEYWORDS: &[&str] = &[
+ "error",
+ "exception",
+ "failed",
+ "failure",
+ "critical",
+ "fatal",
+ "crash",
+ "panic",
+ "abort",
+ "timeout",
+ "denied",
+ "rejected",
+];
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn matches_python_count() {
+ // Python `len(ERROR_KEYWORDS) == 12`. If this drifts, the
+ // Python set was edited and the Rust list needs a matching
+ // update.
+ assert_eq!(ERROR_KEYWORDS.len(), 12);
+ }
+
+ #[test]
+ fn all_lowercase_invariant() {
+ for &kw in ERROR_KEYWORDS {
+ assert_eq!(kw, kw.to_lowercase(), "ERROR_KEYWORDS must all be lowercase");
+ }
+ }
+
+ #[test]
+ fn pinned_membership() {
+ // Pin the exact set so accidental edits surface in CI rather
+ // than silently changing item-preservation behavior.
+ let expected = [
+ "error", "exception", "failed", "failure", "critical", "fatal", "crash", "panic",
+ "abort", "timeout", "denied", "rejected",
+ ];
+ let actual: std::collections::BTreeSet<&str> = ERROR_KEYWORDS.iter().copied().collect();
+ let expected: std::collections::BTreeSet<&str> = expected.iter().copied().collect();
+ assert_eq!(actual, expected);
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs b/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs
new file mode 100644
index 000000000..34d91b023
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs
@@ -0,0 +1,400 @@
+//! Statistical detectors for ID-like and score-like fields.
+//!
+//! Direct ports of `_detect_id_field_statistically` and
+//! `_detect_score_field_statistically` from `smart_crusher.py:484-603`.
+//!
+//! These run *after* per-field statistics are computed and consume a
+//! `FieldStats` plus the raw values. They're called by the analyzer's
+//! crushability logic to decide whether a field carries a meaningful
+//! ranking signal (score) or is just a unique identifier (ID) that
+//! shouldn't drive compression decisions.
+
+use serde_json::Value;
+
+use super::statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
+use super::types::FieldStats;
+
+/// Detect whether a field is an "ID field" — high-uniqueness column
+/// that doesn't carry semantic information.
+///
+/// Direct port of `_detect_id_field_statistically` (Python
+/// `smart_crusher.py:484-530`). Returns `(is_id, confidence)` where
+/// confidence ∈ [0.0, 1.0]. The caller uses this to decide whether the
+/// field is a strong enough signal to drive crushability analysis.
+///
+/// # Detection rules (mirroring Python step-by-step)
+///
+/// 1. Hard gate: `unique_ratio < 0.9` → not an ID field.
+/// 2. String fields:
+/// - >80% of first-20 sample values look like UUIDs → confidence 0.95.
+/// - Average entropy >0.7 AND `unique_ratio > 0.95` → confidence 0.8.
+/// 3. Numeric fields:
+/// - Sequential pattern (via `detect_sequential_pattern`) AND
+/// `unique_ratio > 0.95` → confidence 0.9.
+/// - Has a value range AND `unique_ratio > 0.95` → confidence 0.85.
+/// 4. Catch-all: very high uniqueness (`> 0.98`) → confidence 0.7.
+pub fn detect_id_field_statistically(stats: &FieldStats, values: &[Value]) -> (bool, f64) {
+ // Hard gate matching Python line 494.
+ if stats.unique_ratio < 0.9 {
+ return (false, 0.0);
+ }
+
+ // String-field branches.
+ if stats.field_type == "string" {
+ // First 20 string-typed values for sampling. Python: `values[:20]`
+ // then filters by `isinstance(v, str)` — order-preserving slice
+ // before filter, so we mirror that.
+ let sample_values: Vec<&str> = values
+ .iter()
+ .take(20)
+ .filter_map(|v| v.as_str())
+ .collect();
+
+ if !sample_values.is_empty() {
+ let uuid_count = sample_values.iter().filter(|s| is_uuid_format(s)).count();
+ // Python: `uuid_count / len(sample_values) > 0.8`.
+ if (uuid_count as f64 / sample_values.len() as f64) > 0.8 {
+ return (true, 0.95);
+ }
+
+ // Python: average entropy across the sample.
+ let avg_entropy = sample_values
+ .iter()
+ .map(|s| calculate_string_entropy(s))
+ .sum::<f64>()
+ / sample_values.len() as f64;
+ if avg_entropy > 0.7 && stats.unique_ratio > 0.95 {
+ return (true, 0.8);
+ }
+ }
+ }
+
+ // Numeric-field branches.
+ if stats.field_type == "numeric" {
+ // Python: passes `values` (full list, may include strings) through
+ // `_detect_sequential_pattern` with default `check_order=True`.
+ if detect_sequential_pattern(values, true) && stats.unique_ratio > 0.95 {
+ return (true, 0.9);
+ }
+
+ // High-uniqueness numeric with non-trivial range — likely an ID
+ // even without sequential structure (e.g., random ints in a wide
+ // band).
+ if let (Some(min_v), Some(max_v)) = (stats.min_val, stats.max_val) {
+ let value_range = max_v - min_v;
+ if value_range > 0.0 && stats.unique_ratio > 0.95 {
+ return (true, 0.85);
+ }
+ }
+ }
+
+ // Catch-all: very high uniqueness alone is a signal.
+ if stats.unique_ratio > 0.98 {
+ return (true, 0.7);
+ }
+
+ (false, 0.0)
+}
+
+/// Detect whether a field is a "score field" — bounded-range numeric
+/// where higher values mean "more relevant".
+///
+/// Direct port of `_detect_score_field_statistically` (Python
+/// `smart_crusher.py:533-603`). Returns `(is_score, confidence)`.
+///
+/// # Detection rules (mirroring Python)
+///
+/// 1. Field must be numeric AND have both `min_val` and `max_val`.
+/// 2. Range must match a "common score range":
+/// - `[0, 1]` (most common ML score range) → +0.4
+/// - `[0, 10]` → +0.3
+/// - `[0, 100]` → +0.25
+/// - `[-1, 1]` (signed similarity) → +0.35
+/// 3. Must NOT be a sequential pattern (IDs are sequential; scores aren't).
+/// 4. If first-50 values appear sorted descending (>70% of pairs) → +0.3.
+/// 5. If >30% of first-20 are non-integer floats → +0.1.
+/// 6. Returns `(confidence >= 0.4, min(confidence, 0.95))`.
+///
+/// `items` is the list of original-array dict items so we can pull the
+/// field's values in array order for the descending-sort check.
+pub fn detect_score_field_statistically(stats: &FieldStats, items: &[Value]) -> (bool, f64) {
+ if stats.field_type != "numeric" {
+ return (false, 0.0);
+ }
+
+ let (min_val, max_val) = match (stats.min_val, stats.max_val) {
+ (Some(min_v), Some(max_v)) => (min_v, max_v),
+ _ => return (false, 0.0),
+ };
+
+ let mut confidence: f64 = 0.0;
+
+ // Range check (Python lines 555-568). The conditions are arranged
+ // exactly as in Python — `if/elif` chain, first match wins.
+ let is_bounded = if (0.0..=1.0).contains(&min_val) && (0.0..=1.0).contains(&max_val) {
+ confidence += 0.4;
+ true
+ } else if (0.0..=10.0).contains(&min_val) && (0.0..=10.0).contains(&max_val) {
+ confidence += 0.3;
+ true
+ } else if (0.0..=100.0).contains(&min_val) && (0.0..=100.0).contains(&max_val) {
+ confidence += 0.25;
+ true
+ } else if min_val >= -1.0 && max_val <= 1.0 {
+ // Python: `elif -1 <= min_val and max_val <= 1`. Note Python's
+ // chained `<=` only on max_val side; min_val is checked
+ // separately. Pinned exactly.
+ confidence += 0.35;
+ true
+ } else {
+ false
+ };
+
+ if !is_bounded {
+ return (false, 0.0);
+ }
+
+ // Pull this field's values from the FIRST 50 items, dict-style.
+ // Python: `[item.get(stats.name) for item in items[:50] if stats.name in item]`.
+ let sample_values: Vec<&Value> = items
+ .iter()
+ .take(50)
+ .filter_map(|item| item.as_object().and_then(|m| m.get(&stats.name)))
+ .collect();
+
+ // Sequential check — IDs are sequential, scores aren't. We need
+ // owned `Value`s for `detect_sequential_pattern`; clone the sample.
+ let sample_owned: Vec<Value> = sample_values.iter().map(|v| (*v).clone()).collect();
+ if detect_sequential_pattern(&sample_owned, true) {
+ return (false, 0.0);
+ }
+
+ // Descending-sort check on the full items list (Python line 580-593).
+ // Filter to finite-numeric values, preserving array order.
+ let values_in_order: Vec<f64> = items
+ .iter()
+ .filter_map(|item| item.as_object().and_then(|m| m.get(&stats.name)))
+ .filter_map(|v| v.as_f64())
+ .filter(|f| f.is_finite())
+ .collect();
+
+ if values_in_order.len() >= 5 {
+ let num_pairs = values_in_order.len() - 1;
+ let descending_count = values_in_order
+ .windows(2)
+ .filter(|w| w[0] >= w[1])
+ .count();
+ if num_pairs > 0 && (descending_count as f64 / num_pairs as f64) > 0.7 {
+ confidence += 0.3;
+ }
+ }
+
+ // Float-fraction check on first 20 ordered values (Python line 597-601).
+ // "Not equal to its int-truncation" ≈ has a fractional part.
+ let first_20: &[f64] = if values_in_order.len() > 20 {
+ &values_in_order[..20]
+ } else {
+ &values_in_order[..]
+ };
+ let float_count = first_20
+ .iter()
+ .filter(|&&v| v.is_finite() && v != v.trunc())
+ .count();
+ if !first_20.is_empty() && (float_count as f64) > (first_20.len() as f64 * 0.3) {
+ confidence += 0.1;
+ }
+
+ let is_score = confidence >= 0.4;
+ let bounded_confidence = confidence.min(0.95);
+ (is_score, bounded_confidence)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn stats(name: &str, field_type: &str, unique_ratio: f64) -> FieldStats {
+ FieldStats {
+ name: name.to_string(),
+ field_type: field_type.to_string(),
+ count: 100,
+ unique_count: (100.0 * unique_ratio) as usize,
+ unique_ratio,
+ is_constant: false,
+ constant_value: None,
+ min_val: None,
+ max_val: None,
+ mean_val: None,
+ variance: None,
+ change_points: Vec::new(),
+ avg_length: None,
+ top_values: Vec::new(),
+ }
+ }
+
+ fn stats_with_range(name: &str, min_v: f64, max_v: f64) -> FieldStats {
+ let mut s = stats(name, "numeric", 1.0);
+ s.min_val = Some(min_v);
+ s.max_val = Some(max_v);
+ s
+ }
+
+ // ---------- detect_id_field_statistically ----------
+
+ #[test]
+ fn id_field_low_uniqueness_rejected() {
+ let s = stats("status", "string", 0.5);
+ let values: Vec<Value> = vec![json!("ok"), json!("error"), json!("ok"), json!("ok")];
+ assert_eq!(detect_id_field_statistically(&s, &values), (false, 0.0));
+ }
+
+ #[test]
+ fn id_field_uuid_strings_high_confidence() {
+ let s = stats("uid", "string", 1.0);
+ let values: Vec<Value> = (0..20)
+ .map(|i| json!(format!("550e8400-e29b-41d4-a716-{:012x}", i)))
+ .collect();
+ let (is_id, conf) = detect_id_field_statistically(&s, &values);
+ assert!(is_id);
+ assert_eq!(conf, 0.95);
+ }
+
+ #[test]
+ fn id_field_high_entropy_strings() {
+ let mut s = stats("uid", "string", 1.0);
+ s.unique_ratio = 0.96;
+ // 20 random-looking hex-ish strings — high entropy.
+ let values: Vec<Value> = (0..20)
+ .map(|i| json!(format!("a3f7b2c{:06x}d8e1f4a7", i)))
+ .collect();
+ let (is_id, conf) = detect_id_field_statistically(&s, &values);
+ assert!(is_id);
+ assert!((conf - 0.8).abs() < 1e-9);
+ }
+
+ #[test]
+ fn id_field_sequential_numeric() {
+ let mut s = stats("id", "numeric", 1.0);
+ s.unique_ratio = 0.96;
+ s.min_val = Some(1.0);
+ s.max_val = Some(100.0);
+ let values: Vec<Value> = (1..=100).map(|i| json!(i)).collect();
+ let (is_id, conf) = detect_id_field_statistically(&s, &values);
+ assert!(is_id);
+ assert!((conf - 0.9).abs() < 1e-9);
+ }
+
+ #[test]
+ fn id_field_high_uniqueness_alone_triggers_catchall() {
+ // Numeric field with unique_ratio = 0.99 but no other signal.
+ // Falls through to the 0.98 catch-all.
+ let mut s = stats("misc", "numeric", 0.99);
+ s.min_val = Some(0.0);
+ s.max_val = Some(0.0); // zero range → numeric range branch fails
+ let values: Vec<Value> = (0..100).map(|_| json!(0)).collect();
+ let (is_id, conf) = detect_id_field_statistically(&s, &values);
+ assert!(is_id);
+ assert!((conf - 0.7).abs() < 1e-9);
+ }
+
+ // ---------- detect_score_field_statistically ----------
+
+ #[test]
+ fn score_field_unit_range_with_descending_sort() {
+ // Range [0, 1] (+0.4) + descending sort (+0.3) = 0.7, which
+ // exceeds the 0.4 threshold and is below the 0.95 cap.
+ let s = stats_with_range("score", 0.0, 1.0);
+ let items: Vec<Value> = (0..10)
+ .rev()
+ .map(|i| json!({"score": (i as f64) / 10.0}))
+ .collect();
+ let (is_score, conf) = detect_score_field_statistically(&s, &items);
+ assert!(is_score);
+ assert!(conf >= 0.7);
+ assert!(conf <= 0.95);
+ }
+
+ #[test]
+ fn score_field_sequential_rejected() {
+ // Bounded but sequential — must NOT be a score.
+ let s = stats_with_range("score", 1.0, 10.0);
+ let items: Vec<Value> = (1..=10).map(|i| json!({"score": i})).collect();
+ let (is_score, _) = detect_score_field_statistically(&s, &items);
+ assert!(!is_score);
+ }
+
+ #[test]
+ fn score_field_unbounded_range_rejected() {
+ // Range [0, 1000] doesn't match any of the bounded-range cases.
+ let s = stats_with_range("metric", 0.0, 1000.0);
+ let items: Vec<Value> = (0..10).map(|i| json!({"metric": i * 100})).collect();
+ let (is_score, _) = detect_score_field_statistically(&s, &items);
+ assert!(!is_score);
+ }
+
+ #[test]
+ fn score_field_signed_similarity_range() {
+ // Range [-1, 1] is the cosine-similarity bucket (+0.35).
+ let s = stats_with_range("similarity", -0.9, 0.95);
+ let items: Vec<Value> = (0..10)
+ .rev()
+ .map(|i| json!({"similarity": (i as f64) / 10.0 - 0.5}))
+ .collect();
+ let (is_score, _) = detect_score_field_statistically(&s, &items);
+ assert!(is_score);
+ }
+
+ #[test]
+ fn score_field_below_threshold_rejected() {
+ // Range [0, 100] alone (+0.25) is below the 0.4 threshold.
+ // No descending sort, no float fraction → stays below 0.4.
+ let s = stats_with_range("metric", 0.0, 100.0);
+ // Random unsorted ints — no descending hint.
+ let items: Vec<Value> = vec![
+ json!({"metric": 50}),
+ json!({"metric": 10}),
+ json!({"metric": 80}),
+ json!({"metric": 20}),
+ json!({"metric": 90}),
+ ];
+ let (is_score, _) = detect_score_field_statistically(&s, &items);
+ assert!(!is_score);
+ }
+
+ #[test]
+ fn score_field_non_numeric_rejected() {
+ let s = stats("name", "string", 0.5);
+ let items: Vec<Value> = vec![
+ json!({"name": "alice"}),
+ json!({"name": "bob"}),
+ json!({"name": "alice"}),
+ ];
+ let (is_score, _) = detect_score_field_statistically(&s, &items);
+ assert!(!is_score);
+ }
+
+ #[test]
+ fn score_field_missing_min_max_rejected() {
+ let s = stats("score", "numeric", 1.0); // no min/max set
+ let items: Vec<Value> = vec![json!({"score": 0.5})];
+ let (is_score, _) = detect_score_field_statistically(&s, &items);
+ assert!(!is_score);
+ }
+
+ #[test]
+ fn score_field_confidence_capped_at_95() {
+ // Range [0, 1] (+0.4) + descending sort (+0.3) + float fraction
+ // (+0.1) = 0.8. Below cap. To exceed 0.95 we'd need >0.55 from
+ // bonuses, which isn't possible with current rules. Pin the cap
+ // anyway by constructing the highest-scoring case and confirming
+ // it doesn't go above 0.95.
+ let s = stats_with_range("score", 0.0, 1.0);
+ let items: Vec<Value> = (0..50)
+ .rev()
+ .map(|i| json!({"score": (i as f64) / 50.0}))
+ .collect();
+ let (_, conf) = detect_score_field_statistically(&s, &items);
+ assert!(conf <= 0.95);
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
index 9c408cb39..9eed2cdcd 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
@@ -35,14 +35,22 @@
mod anchors;
mod classifier;
mod config;
+mod error_keywords;
+mod field_detect;
mod hashing;
+mod outliers;
mod statistics;
mod types;
pub use anchors::{extract_query_anchors, item_matches_anchors};
pub use classifier::{classify_array, ArrayType};
pub use config::SmartCrusherConfig;
+pub use error_keywords::ERROR_KEYWORDS;
+pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};
pub use hashing::hash_field_name;
+pub use outliers::{
+ detect_error_items_for_preservation, detect_rare_status_values, detect_structural_outliers,
+};
pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
pub use types::{
ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,
diff --git a/crates/headroom-core/src/transforms/smart_crusher/outliers.rs b/crates/headroom-core/src/transforms/smart_crusher/outliers.rs
new file mode 100644
index 000000000..d09150c0e
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/outliers.rs
@@ -0,0 +1,502 @@
+//! Outlier detectors used to mark items as "must preserve" during compression.
+//!
+//! Direct port of `_detect_structural_outliers`, `_detect_rare_status_values`,
+//! and `_detect_error_items_for_preservation` from
+//! `smart_crusher.py:606-748`.
+//!
+//! `_detect_items_by_learned_semantics` is deferred to a later commit
+//! because it depends on the TOIN `FieldSemantics` type, which isn't
+//! ported yet.
+//!
+//! # Bug #3 fix — `detect_rare_status_values`
+//!
+//! Python's original guard at `smart_crusher.py:674`
+//! `if not (2 <= len(unique_values) <= 10): continue`
+//! caps cardinality at 10, so error-code domains with 50+ codes are
+//! skipped entirely — even when one or two codes appear at <1% rates
+//! and clearly deserve outlier flagging.
+//!
+//! The fix replaces the cap-and-dominance approach with a Pareto check:
+//!
+//! 1. Cardinality cap raised to **50** (above which the field is
+//! almost certainly an ID/free-form column, not a status enum).
+//! 2. Sort value frequencies descending. Find the smallest K such
+//! that the top-K values cover ≥80% of items.
+//! 3. If `K ≤ 5`, the remaining values are "rare" and items
+//! containing them are outliers.
+//!
+//! This unifies both cases the original algorithm partially handled:
+//!
+//! - **Low cardinality + dominant**: 95×"ok" + 5 errors → top-1 covers
+//! 95% → 4 rare values flagged. Same as before.
+//! - **Higher cardinality + bimodal**: 60×"info" + 25×"warn" + 15
+//! distinct rare errors → top-2 covers 85% → 15 rare values flagged.
+//! New, correct, and was missed entirely by the old code.
+//! - **Uniform distribution**: 50 distinct values, 2 each → top-K
+//! never reaches 80% with K ≤ 5 → skip. Correctly identifies as
+//! non-categorical.
+//!
+//! The same fix lands in `headroom/transforms/smart_crusher.py` later
+//! in this PR so the parity fixtures continue to byte-match.
+
+use serde_json::Value;
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+
+use super::error_keywords::ERROR_KEYWORDS;
+
+/// Detect items that are structural outliers (error-like or
+/// uncommonly-shaped).
+///
+/// Direct port of `_detect_structural_outliers` (Python
+/// `smart_crusher.py:606-650`). Returns deduplicated, ascending-sorted
+/// indices.
+///
+/// # Detection
+///
+/// 1. **Rare-field outliers**: items containing a field that appears
+/// in <20% of the array.
+/// 2. **Rare-status outliers**: forwarded to `detect_rare_status_values`,
+/// which finds items with statistically rare categorical values
+/// (with bug #3 fixed).
+pub fn detect_structural_outliers(items: &[Value]) -> Vec<usize> {
+ if items.len() < 5 {
+ return Vec::new();
+ }
+
+ // Field counts across the whole array.
+ let mut field_counts: BTreeMap<&str, usize> = BTreeMap::new();
+ for item in items {
+ if let Some(obj) = item.as_object() {
+ for key in obj.keys() {
+ *field_counts.entry(key.as_str()).or_insert(0) += 1;
+ }
+ }
+ }
+
+ let n = items.len();
+ let common_fields: HashSet<String> = field_counts
+ .iter()
+ .filter(|(_, &c)| c as f64 >= n as f64 * 0.8)
+ .map(|(k, _)| (*k).to_string())
+ .collect();
+ let rare_fields: HashSet<&str> = field_counts
+ .iter()
+ .filter(|(_, &c)| (c as f64) < n as f64 * 0.2)
+ .map(|(k, _)| *k)
+ .collect();
+
+ // Use a BTreeSet for stable order — Python uses `set()` then `list(set(...))`
+ // which is non-deterministic order; we pin to ascending for parity tests.
+ let mut outlier_set: BTreeSet<usize> = BTreeSet::new();
+
+ // 1. Rare-field outliers.
+ for (i, item) in items.iter().enumerate() {
+ let Some(obj) = item.as_object() else { continue };
+ let has_rare = obj.keys().any(|k| rare_fields.contains(k.as_str()));
+ if has_rare {
+ outlier_set.insert(i);
+ }
+ }
+
+ // 2. Rare-status outliers.
+ for idx in detect_rare_status_values(items, &common_fields) {
+ outlier_set.insert(idx);
+ }
+
+ outlier_set.into_iter().collect()
+}
+
+/// Detect items with rare values in status-like categorical fields.
+///
+/// **Bug #3 fix** — see module-level doc. Algorithm:
+///
+/// 1. Cardinality 2..=50 (was 2..=10 in Python).
+/// 2. Pareto check: top-K values covering ≥80% with `K ≤ 5`.
+/// 3. Items NOT in top-K → outliers.
+///
+/// Returns indices in the order they were discovered (mirrors
+/// Python's append-order behavior; downstream `detect_structural_outliers`
+/// dedupes via BTreeSet).
+pub fn detect_rare_status_values(items: &[Value], common_fields: &HashSet<String>) -> Vec<usize> {
+ let mut outlier_indices: Vec<usize> = Vec::new();
+
+ // Iterate fields in sorted order for determinism. Python iterates
+ // a `set`, which has non-deterministic order — but the eventual
+ // output is deduped via the caller's set, so order here only
+ // affects which fields drive detection if multiple status-like
+ // fields exist. Sorting gives us a stable, fixture-friendly order.
+ let mut sorted_fields: Vec<&String> = common_fields.iter().collect();
+ sorted_fields.sort();
+
+ for field_name in sorted_fields {
+ // Collect this field's values across all items, mirroring Python:
+ // `[item.get(field_name) for item in items if isinstance(item, dict) and field_name in item]`
+ let values: Vec<&Value> = items
+ .iter()
+ .filter_map(|item| item.as_object())
+ .filter_map(|m| m.get(field_name))
+ .collect();
+
+ // Stringify non-null values and dedupe to get cardinality. Python:
+ // `unique_values = {str(v) for v in values if v is not None}`
+ // We use `python_repr_value`-equivalent stringification: simple
+ // scalars use their natural form; nested values use serde_json
+ // serialization. This stringification is only used for set-
+ // dedup and frequency counting, not surfaced to callers, so the
+ // python_repr-vs-json distinction we made for anchors doesn't
+ // matter here — the SAME stringification is used for both the
+ // "is this rare" computation and the per-item lookup, so the
+ // surface is internally consistent.
+ let stringify = |v: &Value| -> String {
+ match v {
+ Value::Null => unreachable!("null filtered above"),
+ Value::Bool(b) => b.to_string(),
+ Value::Number(n) => n.to_string(),
+ Value::String(s) => s.clone(),
+ _ => v.to_string(),
+ }
+ };
+
+ let unique_values: BTreeSet<String> = values
+ .iter()
+ .filter(|v| !matches!(v, Value::Null))
+ .map(|v| stringify(v))
+ .collect();
+
+ // Cardinality cap (BUG #3 FIX: was 2..=10, now 2..=50).
+ if !(2..=50).contains(&unique_values.len()) {
+ continue;
+ }
+
+ // Frequency count. Python: `value_counts: dict[str, int]`,
+ // `key = str(v) if v is not None else "__none__"`.
+ let mut value_counts: BTreeMap<String, usize> = BTreeMap::new();
+ for v in &values {
+ let key = if matches!(v, Value::Null) {
+ "__none__".to_string()
+ } else {
+ stringify(v)
+ };
+ *value_counts.entry(key).or_insert(0) += 1;
+ }
+ if value_counts.is_empty() {
+ continue;
+ }
+
+ let total = values.len();
+
+ // Pareto check (BUG #3 FIX): find smallest K such that top-K
+ // values cover ≥80% of items.
+ let mut sorted_counts: Vec<(&String, &usize)> = value_counts.iter().collect();
+ // Sort by count descending; tiebreak by key ascending so the
+ // result is deterministic when multiple values have the same
+ // frequency.
+ sorted_counts.sort_by(|a, b| b.1.cmp(a.1).then(a.0.cmp(b.0)));
+
+ let threshold = (total as f64 * 0.8).ceil() as usize;
+ let mut cumulative: usize = 0;
+ let mut top_k_values: HashSet<String> = HashSet::new();
+ for (value, count) in &sorted_counts {
+ cumulative += **count;
+ top_k_values.insert((*value).clone());
+ if cumulative >= threshold {
+ break;
+ }
+ }
+
+ // Only flag rare values if the top-K is small (≤5). Above this
+ // the distribution is too uniform to label any value "rare".
+ if top_k_values.len() > 5 {
+ continue;
+ }
+
+ // Items with values NOT in top_k_values are outliers.
+ for (i, item) in items.iter().enumerate() {
+ let Some(obj) = item.as_object() else { continue };
+ let Some(field_value) = obj.get(field_name) else { continue };
+ let item_value = if matches!(field_value, Value::Null) {
+ "__none__".to_string()
+ } else {
+ stringify(field_value)
+ };
+ if !top_k_values.contains(&item_value) {
+ outlier_indices.push(i);
+ }
+ }
+ }
+
+ outlier_indices
+}
+
+/// Detect items containing error keywords for PRESERVATION.
+///
+/// Direct port of `_detect_error_items_for_preservation` (Python
+/// `smart_crusher.py:711-748`). Used by the orchestrator's
+/// `_prioritize_indices` to ensure error items are NEVER dropped.
+///
+/// # Args
+///
+/// - `items`: array items to scan.
+/// - `item_strings`: pre-computed JSON serializations to avoid
+/// redundant `to_string` work. Pass `None` to serialize on the fly.
+/// When provided, must be the same length as `items` (Python's
+/// bounds-check via `i < len(item_strings)` is mirrored).
+pub fn detect_error_items_for_preservation(
+ items: &[Value],
+ item_strings: Option<&[String]>,
+) -> Vec<usize> {
+ let mut error_indices: Vec<usize> = Vec::new();
+
+ for (i, item) in items.iter().enumerate() {
+ // Python: `if not isinstance(item, dict): continue`. Mirror.
+ if !item.is_object() {
+ continue;
+ }
+
+ // Reuse cached serialization or serialize fresh. Python:
+ // `if item_strings is not None and i < len(item_strings):
+ // item_str = item_strings[i].lower()
+ // else:
+ // item_str = json.dumps(item).lower()`
+ let serialized: String = match item_strings {
+ Some(arr) if i < arr.len() => arr[i].to_lowercase(),
+ _ => match serde_json::to_string(item) {
+ Ok(s) => s.to_lowercase(),
+ Err(_) => continue,
+ },
+ };
+
+ // Python: `for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:
+ // if keyword in item_str: ...; break`
+ if ERROR_KEYWORDS.iter().any(|kw| serialized.contains(kw)) {
+ error_indices.push(i);
+ }
+ }
+
+ error_indices
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ // ---------- detect_structural_outliers ----------
+
+ #[test]
+ fn outliers_too_few_items_returns_empty() {
+ let items: Vec<Value> = (0..3).map(|i| json!({"a": i})).collect();
+ assert_eq!(detect_structural_outliers(&items), Vec::<usize>::new());
+ }
+
+ #[test]
+ fn outliers_rare_field_flags_item() {
+ // 9 items with `{"a"}`, 1 item with extra `{"a", "x"}` — `x`
+ // appears in 10% of items, below the 20% rare-field threshold.
+ let mut items: Vec<Value> = (0..9).map(|i| json!({"a": i})).collect();
+ items.push(json!({"a": 9, "x": "rare"}));
+ let outliers = detect_structural_outliers(&items);
+ assert!(outliers.contains(&9));
+ }
+
+ #[test]
+ fn outliers_no_dict_items_silently_skipped() {
+ // Mixed array: only dict items count. A non-dict in the middle
+ // doesn't crash anything.
+ let items: Vec<Value> = vec![
+ json!({"status": "ok"}),
+ json!("string-not-dict"),
+ json!({"status": "ok"}),
+ json!({"status": "ok"}),
+ json!({"status": "ok"}),
+ ];
+ // Should not panic.
+ let _ = detect_structural_outliers(&items);
+ }
+
+ // ---------- detect_rare_status_values (BUG #3 FIX coverage) ----------
+
+ #[test]
+ fn rare_status_low_cardinality_dominant_value() {
+ // Pre-bug-fix path still works: 95×ok + 5 errors.
+ let mut items: Vec<Value> = (0..95).map(|_| json!({"status": "ok"})).collect();
+ items.push(json!({"status": "error"}));
+ items.push(json!({"status": "timeout"}));
+ items.push(json!({"status": "error"}));
+ items.push(json!({"status": "timeout"}));
+ items.push(json!({"status": "fail"}));
+ let common: HashSet<String> = ["status".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ // All 5 non-"ok" items flagged.
+ assert_eq!(outliers.len(), 5);
+ }
+
+ #[test]
+ fn rare_status_bug3_fix_high_cardinality_bimodal() {
+ // BUG #3 case: cardinality 17 (1 dominant + 1 second + 15
+ // singletons). Old code: 17 > 10 → skip. New code: top-2 covers
+ // 85%, K=2 ≤ 5, remaining 15 values flagged.
+ let mut items: Vec<Value> = Vec::new();
+ for _ in 0..60 {
+ items.push(json!({"code": "INFO"}));
+ }
+ for _ in 0..25 {
+ items.push(json!({"code": "WARN"}));
+ }
+ for i in 0..15 {
+ items.push(json!({"code": format!("ERR_{}", i)}));
+ }
+ let common: HashSet<String> = ["code".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ // 15 rare-error items flagged. Pre-fix this would be 0.
+ assert_eq!(outliers.len(), 15);
+ }
+
+ #[test]
+ fn rare_status_uniform_distribution_no_outliers() {
+ // 50 items, 50 distinct values, 1 each. Top-K never reaches
+ // 80% with K ≤ 5 → no outliers (correctly identified as
+ // non-categorical).
+ let items: Vec<Value> = (0..50)
+ .map(|i| json!({"code": format!("CAT_{}", i)}))
+ .collect();
+ let common: HashSet<String> = ["code".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ assert!(outliers.is_empty(),
+ "uniform distribution must not produce rare-status outliers");
+ }
+
+ #[test]
+ fn rare_status_cardinality_above_50_skipped() {
+ // 60 distinct values → cardinality cap rejects.
+ let items: Vec<Value> = (0..60)
+ .map(|i| json!({"code": format!("V_{}", i)}))
+ .collect();
+ let common: HashSet<String> = ["code".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ assert!(outliers.is_empty());
+ }
+
+ #[test]
+ fn rare_status_cardinality_one_skipped() {
+ // 100 items, all same value → cardinality 1, fails 2..=50 gate.
+ let items: Vec<Value> = (0..100).map(|_| json!({"status": "ok"})).collect();
+ let common: HashSet<String> = ["status".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ assert!(outliers.is_empty());
+ }
+
+ #[test]
+ fn rare_status_nulls_filtered_from_cardinality() {
+ // Pinned Python parity: `unique_values = {str(v) for v in values
+ // if v is not None}` — nulls are excluded from the cardinality
+ // computation. With 95×"ok" + 5×null, cardinality = 1 (just "ok"),
+ // which fails the 2..=50 gate and the field is skipped entirely.
+ // Pre-fix Python had the same behavior; the null-aware
+ // `__none__` mapping only kicks in inside `value_counts` which
+ // is unreachable when the cardinality gate fails first.
+ //
+ // If we ever want null to count as a distinct categorical value
+ // (so missing-status items get flagged), that's a behavior
+ // change beyond bug #3 and lives in Stage 3c.2.
+ let mut items: Vec<Value> = (0..95).map(|_| json!({"s": "ok"})).collect();
+ for _ in 0..5 {
+ items.push(json!({"s": null}));
+ }
+ let common: HashSet<String> = ["s".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ assert!(
+ outliers.is_empty(),
+ "cardinality 1 (after null filter) must skip the field"
+ );
+ }
+
+ #[test]
+ fn rare_status_nulls_count_in_value_counts_when_cardinality_passes() {
+ // Once cardinality >= 2 (with nulls excluded from the set), the
+ // value_counts loop maps null → "__none__" and treats it as a
+ // distinct value for frequency counting. Mirrors Python's
+ // `key = str(v) if v is not None else "__none__"`.
+ //
+ // Setup: 90×"ok" + 5×"warn" + 5×null → unique_values = {"ok", "warn"},
+ // cardinality 2, gate passes. value_counts: ok=90, warn=5,
+ // __none__=5. top-1 = "ok" (90/100 = 90%) covers ≥80%, K=1 ≤ 5.
+ // Items with "warn" or null are flagged.
+ let mut items: Vec<Value> = (0..90).map(|_| json!({"s": "ok"})).collect();
+ for _ in 0..5 {
+ items.push(json!({"s": "warn"}));
+ }
+ for _ in 0..5 {
+ items.push(json!({"s": null}));
+ }
+ let common: HashSet<String> = ["s".to_string()].into_iter().collect();
+ let outliers = detect_rare_status_values(&items, &common);
+ assert_eq!(outliers.len(), 10, "5 warn + 5 null = 10 outliers");
+ }
+
+ // ---------- detect_error_items_for_preservation ----------
+
+ #[test]
+ fn error_keywords_preservation_basic() {
+ let items: Vec<Value> = vec![
+ json!({"status": "ok"}),
+ json!({"status": "error", "msg": "boom"}),
+ json!({"status": "ok"}),
+ json!({"msg": "request failed"}),
+ json!({"status": "ok"}),
+ ];
+ let errs = detect_error_items_for_preservation(&items, None);
+ assert_eq!(errs, vec![1, 3]);
+ }
+
+ #[test]
+ fn error_keywords_case_insensitive() {
+ let items: Vec<Value> = vec![
+ json!({"msg": "FATAL: out of memory"}),
+ json!({"msg": "panic at line 42"}),
+ ];
+ let errs = detect_error_items_for_preservation(&items, None);
+ assert_eq!(errs, vec![0, 1]);
+ }
+
+ #[test]
+ fn error_keywords_no_match() {
+ let items: Vec<Value> = vec![
+ json!({"name": "alice"}),
+ json!({"count": 5}),
+ ];
+ let errs = detect_error_items_for_preservation(&items, None);
+ assert!(errs.is_empty());
+ }
+
+ #[test]
+ fn error_keywords_uses_cached_strings_when_provided() {
+ // If `item_strings` is passed, we use those rather than
+ // re-serializing. Test that a custom cached string can drive
+ // a hit even when the actual item wouldn't.
+ let items: Vec<Value> = vec![json!({"a": 1}), json!({"b": 2})];
+ let cached = vec!["error".to_string(), "ok".to_string()];
+ let errs = detect_error_items_for_preservation(&items, Some(&cached));
+ assert_eq!(errs, vec![0]);
+ }
+
+ #[test]
+ fn error_keywords_falls_back_when_cache_too_short() {
+ // i >= len(item_strings) → fall back to fresh serialization.
+ let items: Vec<Value> = vec![json!({"a": 1}), json!({"msg": "error"})];
+ let cached = vec!["ok".to_string()]; // only 1 entry
+ let errs = detect_error_items_for_preservation(&items, Some(&cached));
+ assert_eq!(errs, vec![1]); // index 1 falls back to serialization, hits "error"
+ }
+
+ #[test]
+ fn error_keywords_skips_non_dict_items() {
+ let items: Vec<Value> = vec![
+ json!({"msg": "error"}),
+ json!("error string"), // not a dict — Python skips
+ json!({"msg": "error"}),
+ ];
+ let errs = detect_error_items_for_preservation(&items, None);
+ assert_eq!(errs, vec![0, 2]);
+ }
+}