Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): smart_crusher analyzer helpers — id/score/outlier detectors + bug #3 fix
chopratejasf029055229bb
diff --git a/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs b/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rsnew file mode 100644index 000000000..7661b8b2c--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs@@ -0,0 +1,63 @@+//! Canonical error keyword set for item preservation.+//!+//! Direct port of `ERROR_KEYWORDS` from `headroom/transforms/error_detection.py:18-33`.+//! These are the **FALLBACK** preservation signal when TOIN field+//! semantics aren't available yet (per the Python module-level+//! comment). Intentionally broad — better to over-preserve than to+//! drop a real error item.+//!+//! Used by `detect_error_items_for_preservation`. The list is small+//! enough to keep as a `&[&str]`; if we ever cross ~50 keywords, switch+//! to a `phf::Set` or pre-built FST for sub-linear lookup.++/// 12 error/failure keywords. Order doesn't matter for correctness, but+/// matches Python's set-literal order so reading both side-by-side is+/// easier. Lowercase by construction; callers must lowercase the+/// haystack before substring-matching.+pub const ERROR_KEYWORDS: &[&str] = &[+ "error",+ "exception",+ "failed",+ "failure",+ "critical",+ "fatal",+ "crash",+ "panic",+ "abort",+ "timeout",+ "denied",+ "rejected",+];++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn matches_python_count() {+ // Python `len(ERROR_KEYWORDS) == 12`. If this drifts, the+ // Python set was edited and the Rust list needs a matching+ // update.+ assert_eq!(ERROR_KEYWORDS.len(), 12);+ }++ #[test]+ fn all_lowercase_invariant() {+ for &kw in ERROR_KEYWORDS {+ assert_eq!(kw, kw.to_lowercase(), "ERROR_KEYWORDS must all be lowercase");+ }+ }++ #[test]+ fn pinned_membership() {+ // Pin the exact set so accidental edits surface in CI rather+ // than silently changing item-preservation behavior.+ let expected = [+ "error", "exception", "failed", "failure", "critical", "fatal", "crash", "panic",+ "abort", "timeout", "denied", "rejected",+ ];+ let actual: std::collections::BTreeSet<&str> = ERROR_KEYWORDS.iter().copied().collect();+ let expected: std::collections::BTreeSet<&str> = expected.iter().copied().collect();+ assert_eq!(actual, expected);+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs b/crates/headroom-core/src/transforms/smart_crusher/field_detect.rsnew file mode 100644index 000000000..34d91b023--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs@@ -0,0 +1,400 @@+//! Statistical detectors for ID-like and score-like fields.+//!+//! Direct ports of `_detect_id_field_statistically` and+//! `_detect_score_field_statistically` from `smart_crusher.py:484-603`.+//!+//! These run *after* per-field statistics are computed and consume a+//! `FieldStats` plus the raw values. They're called by the analyzer's+//! crushability logic to decide whether a field carries a meaningful+//! ranking signal (score) or is just a unique identifier (ID) that+//! shouldn't drive compression decisions.++use serde_json::Value;++use super::statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};+use super::types::FieldStats;++/// Detect whether a field is an "ID field" — high-uniqueness column+/// that doesn't carry semantic information.+///+/// Direct port of `_detect_id_field_statistically` (Python+/// `smart_crusher.py:484-530`). Returns `(is_id, confidence)` where+/// confidence ∈ [0.0, 1.0]. The caller uses this to decide whether the+/// field is a strong enough signal to drive crushability analysis.+///+/// # Detection rules (mirroring Python step-by-step)+///+/// 1. Hard gate: `unique_ratio < 0.9` → not an ID field.+/// 2. String fields:+/// - >80% of first-20 sample values look like UUIDs → confidence 0.95.+/// - Average entropy >0.7 AND `unique_ratio > 0.95` → confidence 0.8.+/// 3. Numeric fields:+/// - Sequential pattern (via `detect_sequential_pattern`) AND+/// `unique_ratio > 0.95` → confidence 0.9.+/// - Has a value range AND `unique_ratio > 0.95` → confidence 0.85.+/// 4. Catch-all: very high uniqueness (`> 0.98`) → confidence 0.7.+pub fn detect_id_field_statistically(stats: &FieldStats, values: &[Value]) -> (bool, f64) {+ // Hard gate matching Python line 494.+ if stats.unique_ratio < 0.9 {+ return (false, 0.0);+ }++ // String-field branches.+ if stats.field_type == "string" {+ // First 20 string-typed values for sampling. Python: `values[:20]`+ // then filters by `isinstance(v, str)` — order-preserving slice+ // before filter, so we mirror that.+ let sample_values: Vec<&str> = values+ .iter()+ .take(20)+ .filter_map(|v| v.as_str())+ .collect();++ if !sample_values.is_empty() {+ let uuid_count = sample_values.iter().filter(|s| is_uuid_format(s)).count();+ // Python: `uuid_count / len(sample_values) > 0.8`.+ if (uuid_count as f64 / sample_values.len() as f64) > 0.8 {+ return (true, 0.95);+ }++ // Python: average entropy across the sample.+ let avg_entropy = sample_values+ .iter()+ .map(|s| calculate_string_entropy(s))+ .sum::<f64>()+ / sample_values.len() as f64;+ if avg_entropy > 0.7 && stats.unique_ratio > 0.95 {+ return (true, 0.8);+ }+ }+ }++ // Numeric-field branches.+ if stats.field_type == "numeric" {+ // Python: passes `values` (full list, may include strings) through+ // `_detect_sequential_pattern` with default `check_order=True`.+ if detect_sequential_pattern(values, true) && stats.unique_ratio > 0.95 {+ return (true, 0.9);+ }++ // High-uniqueness numeric with non-trivial range — likely an ID+ // even without sequential structure (e.g., random ints in a wide+ // band).+ if let (Some(min_v), Some(max_v)) = (stats.min_val, stats.max_val) {+ let value_range = max_v - min_v;+ if value_range > 0.0 && stats.unique_ratio > 0.95 {+ return (true, 0.85);+ }+ }+ }++ // Catch-all: very high uniqueness alone is a signal.+ if stats.unique_ratio > 0.98 {+ return (true, 0.7);+ }++ (false, 0.0)+}++/// Detect whether a field is a "score field" — bounded-range numeric+/// where higher values mean "more relevant".+///+/// Direct port of `_detect_score_field_statistically` (Python+/// `smart_crusher.py:533-603`). Returns `(is_score, confidence)`.+///+/// # Detection rules (mirroring Python)+///+/// 1. Field must be numeric AND have both `min_val` and `max_val`.+/// 2. Range must match a "common score range":+/// - `[0, 1]` (most common ML score range) → +0.4+/// - `[0, 10]` → +0.3+/// - `[0, 100]` → +0.25+/// - `[-1, 1]` (signed similarity) → +0.35+/// 3. Must NOT be a sequential pattern (IDs are sequential; scores aren't).+/// 4. If first-50 values appear sorted descending (>70% of pairs) → +0.3.+/// 5. If >30% of first-20 are non-integer floats → +0.1.+/// 6. Returns `(confidence >= 0.4, min(confidence, 0.95))`.+///+/// `items` is the list of original-array dict items so we can pull the+/// field's values in array order for the descending-sort check.+pub fn detect_score_field_statistically(stats: &FieldStats, items: &[Value]) -> (bool, f64) {+ if stats.field_type != "numeric" {+ return (false, 0.0);+ }++ let (min_val, max_val) = match (stats.min_val, stats.max_val) {+ (Some(min_v), Some(max_v)) => (min_v, max_v),+ _ => return (false, 0.0),+ };++ let mut confidence: f64 = 0.0;++ // Range check (Python lines 555-568). The conditions are arranged+ // exactly as in Python — `if/elif` chain, first match wins.+ let is_bounded = if (0.0..=1.0).contains(&min_val) && (0.0..=1.0).contains(&max_val) {+ confidence += 0.4;+ true+ } else if (0.0..=10.0).contains(&min_val) && (0.0..=10.0).contains(&max_val) {+ confidence += 0.3;+ true+ } else if (0.0..=100.0).contains(&min_val) && (0.0..=100.0).contains(&max_val) {+ confidence += 0.25;+ true+ } else if min_val >= -1.0 && max_val <= 1.0 {+ // Python: `elif -1 <= min_val and max_val <= 1`. Note Python's+ // chained `<=` only on max_val side; min_val is checked+ // separately. Pinned exactly.+ confidence += 0.35;+ true+ } else {+ false+ };++ if !is_bounded {+ return (false, 0.0);+ }++ // Pull this field's values from the FIRST 50 items, dict-style.+ // Python: `[item.get(stats.name) for item in items[:50] if stats.name in item]`.+ let sample_values: Vec<&Value> = items+ .iter()+ .take(50)+ .filter_map(|item| item.as_object().and_then(|m| m.get(&stats.name)))+ .collect();++ // Sequential check — IDs are sequential, scores aren't. We need+ // owned `Value`s for `detect_sequential_pattern`; clone the sample.+ let sample_owned: Vec<Value> = sample_values.iter().map(|v| (*v).clone()).collect();+ if detect_sequential_pattern(&sample_owned, true) {+ return (false, 0.0);+ }++ // Descending-sort check on the full items list (Python line 580-593).+ // Filter to finite-numeric values, preserving array order.+ let values_in_order: Vec<f64> = items+ .iter()+ .filter_map(|item| item.as_object().and_then(|m| m.get(&stats.name)))+ .filter_map(|v| v.as_f64())+ .filter(|f| f.is_finite())+ .collect();++ if values_in_order.len() >= 5 {+ let num_pairs = values_in_order.len() - 1;+ let descending_count = values_in_order+ .windows(2)+ .filter(|w| w[0] >= w[1])+ .count();+ if num_pairs > 0 && (descending_count as f64 / num_pairs as f64) > 0.7 {+ confidence += 0.3;+ }+ }++ // Float-fraction check on first 20 ordered values (Python line 597-601).+ // "Not equal to its int-truncation" ≈ has a fractional part.+ let first_20: &[f64] = if values_in_order.len() > 20 {+ &values_in_order[..20]+ } else {+ &values_in_order[..]+ };+ let float_count = first_20+ .iter()+ .filter(|&&v| v.is_finite() && v != v.trunc())+ .count();+ if !first_20.is_empty() && (float_count as f64) > (first_20.len() as f64 * 0.3) {+ confidence += 0.1;+ }++ let is_score = confidence >= 0.4;+ let bounded_confidence = confidence.min(0.95);+ (is_score, bounded_confidence)+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn stats(name: &str, field_type: &str, unique_ratio: f64) -> FieldStats {+ FieldStats {+ name: name.to_string(),+ field_type: field_type.to_string(),+ count: 100,+ unique_count: (100.0 * unique_ratio) as usize,+ unique_ratio,+ is_constant: false,+ constant_value: None,+ min_val: None,+ max_val: None,+ mean_val: None,+ variance: None,+ change_points: Vec::new(),+ avg_length: None,+ top_values: Vec::new(),+ }+ }++ fn stats_with_range(name: &str, min_v: f64, max_v: f64) -> FieldStats {+ let mut s = stats(name, "numeric", 1.0);+ s.min_val = Some(min_v);+ s.max_val = Some(max_v);+ s+ }++ // ---------- detect_id_field_statistically ----------++ #[test]+ fn id_field_low_uniqueness_rejected() {+ let s = stats("status", "string", 0.5);+ let values: Vec<Value> = vec![json!("ok"), json!("error"), json!("ok"), json!("ok")];+ assert_eq!(detect_id_field_statistically(&s, &values), (false, 0.0));+ }++ #[test]+ fn id_field_uuid_strings_high_confidence() {+ let s = stats("uid", "string", 1.0);+ let values: Vec<Value> = (0..20)+ .map(|i| json!(format!("550e8400-e29b-41d4-a716-{:012x}", i)))+ .collect();+ let (is_id, conf) = detect_id_field_statistically(&s, &values);+ assert!(is_id);+ assert_eq!(conf, 0.95);+ }++ #[test]+ fn id_field_high_entropy_strings() {+ let mut s = stats("uid", "string", 1.0);+ s.unique_ratio = 0.96;+ // 20 random-looking hex-ish strings — high entropy.+ let values: Vec<Value> = (0..20)+ .map(|i| json!(format!("a3f7b2c{:06x}d8e1f4a7", i)))+ .collect();+ let (is_id, conf) = detect_id_field_statistically(&s, &values);+ assert!(is_id);+ assert!((conf - 0.8).abs() < 1e-9);+ }++ #[test]+ fn id_field_sequential_numeric() {+ let mut s = stats("id", "numeric", 1.0);+ s.unique_ratio = 0.96;+ s.min_val = Some(1.0);+ s.max_val = Some(100.0);+ let values: Vec<Value> = (1..=100).map(|i| json!(i)).collect();+ let (is_id, conf) = detect_id_field_statistically(&s, &values);+ assert!(is_id);+ assert!((conf - 0.9).abs() < 1e-9);+ }++ #[test]+ fn id_field_high_uniqueness_alone_triggers_catchall() {+ // Numeric field with unique_ratio = 0.99 but no other signal.+ // Falls through to the 0.98 catch-all.+ let mut s = stats("misc", "numeric", 0.99);+ s.min_val = Some(0.0);+ s.max_val = Some(0.0); // zero range → numeric range branch fails+ let values: Vec<Value> = (0..100).map(|_| json!(0)).collect();+ let (is_id, conf) = detect_id_field_statistically(&s, &values);+ assert!(is_id);+ assert!((conf - 0.7).abs() < 1e-9);+ }++ // ---------- detect_score_field_statistically ----------++ #[test]+ fn score_field_unit_range_with_descending_sort() {+ // Range [0, 1] (+0.4) + descending sort (+0.3) = 0.7, which+ // exceeds the 0.4 threshold and is below the 0.95 cap.+ let s = stats_with_range("score", 0.0, 1.0);+ let items: Vec<Value> = (0..10)+ .rev()+ .map(|i| json!({"score": (i as f64) / 10.0}))+ .collect();+ let (is_score, conf) = detect_score_field_statistically(&s, &items);+ assert!(is_score);+ assert!(conf >= 0.7);+ assert!(conf <= 0.95);+ }++ #[test]+ fn score_field_sequential_rejected() {+ // Bounded but sequential — must NOT be a score.+ let s = stats_with_range("score", 1.0, 10.0);+ let items: Vec<Value> = (1..=10).map(|i| json!({"score": i})).collect();+ let (is_score, _) = detect_score_field_statistically(&s, &items);+ assert!(!is_score);+ }++ #[test]+ fn score_field_unbounded_range_rejected() {+ // Range [0, 1000] doesn't match any of the bounded-range cases.+ let s = stats_with_range("metric", 0.0, 1000.0);+ let items: Vec<Value> = (0..10).map(|i| json!({"metric": i * 100})).collect();+ let (is_score, _) = detect_score_field_statistically(&s, &items);+ assert!(!is_score);+ }++ #[test]+ fn score_field_signed_similarity_range() {+ // Range [-1, 1] is the cosine-similarity bucket (+0.35).+ let s = stats_with_range("similarity", -0.9, 0.95);+ let items: Vec<Value> = (0..10)+ .rev()+ .map(|i| json!({"similarity": (i as f64) / 10.0 - 0.5}))+ .collect();+ let (is_score, _) = detect_score_field_statistically(&s, &items);+ assert!(is_score);+ }++ #[test]+ fn score_field_below_threshold_rejected() {+ // Range [0, 100] alone (+0.25) is below the 0.4 threshold.+ // No descending sort, no float fraction → stays below 0.4.+ let s = stats_with_range("metric", 0.0, 100.0);+ // Random unsorted ints — no descending hint.+ let items: Vec<Value> = vec![+ json!({"metric": 50}),+ json!({"metric": 10}),+ json!({"metric": 80}),+ json!({"metric": 20}),+ json!({"metric": 90}),+ ];+ let (is_score, _) = detect_score_field_statistically(&s, &items);+ assert!(!is_score);+ }++ #[test]+ fn score_field_non_numeric_rejected() {+ let s = stats("name", "string", 0.5);+ let items: Vec<Value> = vec![+ json!({"name": "alice"}),+ json!({"name": "bob"}),+ json!({"name": "alice"}),+ ];+ let (is_score, _) = detect_score_field_statistically(&s, &items);+ assert!(!is_score);+ }++ #[test]+ fn score_field_missing_min_max_rejected() {+ let s = stats("score", "numeric", 1.0); // no min/max set+ let items: Vec<Value> = vec![json!({"score": 0.5})];+ let (is_score, _) = detect_score_field_statistically(&s, &items);+ assert!(!is_score);+ }++ #[test]+ fn score_field_confidence_capped_at_95() {+ // Range [0, 1] (+0.4) + descending sort (+0.3) + float fraction+ // (+0.1) = 0.8. Below cap. To exceed 0.95 we'd need >0.55 from+ // bonuses, which isn't possible with current rules. Pin the cap+ // anyway by constructing the highest-scoring case and confirming+ // it doesn't go above 0.95.+ let s = stats_with_range("score", 0.0, 1.0);+ let items: Vec<Value> = (0..50)+ .rev()+ .map(|i| json!({"score": (i as f64) / 50.0}))+ .collect();+ let (_, conf) = detect_score_field_statistically(&s, &items);+ assert!(conf <= 0.95);+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rsindex 9c408cb39..9eed2cdcd 100644--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs@@ -35,14 +35,22 @@mod anchors;mod classifier;mod config;+mod error_keywords;+mod field_detect;mod hashing;+mod outliers;mod statistics;mod types;pub use anchors::{extract_query_anchors, item_matches_anchors};pub use classifier::{classify_array, ArrayType};pub use config::SmartCrusherConfig;+pub use error_keywords::ERROR_KEYWORDS;+pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};pub use hashing::hash_field_name;+pub use outliers::{+ detect_error_items_for_preservation, detect_rare_status_values, detect_structural_outliers,+};pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};pub use types::{ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,diff --git a/crates/headroom-core/src/transforms/smart_crusher/outliers.rs b/crates/headroom-core/src/transforms/smart_crusher/outliers.rsnew file mode 100644index 000000000..d09150c0e--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/outliers.rs@@ -0,0 +1,502 @@+//! Outlier detectors used to mark items as "must preserve" during compression.+//!+//! Direct port of `_detect_structural_outliers`, `_detect_rare_status_values`,+//! and `_detect_error_items_for_preservation` from+//! `smart_crusher.py:606-748`.+//!+//! `_detect_items_by_learned_semantics` is deferred to a later commit+//! because it depends on the TOIN `FieldSemantics` type, which isn't+//! ported yet.+//!+//! # Bug #3 fix — `detect_rare_status_values`+//!+//! Python's original guard at `smart_crusher.py:674`+//! `if not (2 <= len(unique_values) <= 10): continue`+//! caps cardinality at 10, so error-code domains with 50+ codes are+//! skipped entirely — even when one or two codes appear at <1% rates+//! and clearly deserve outlier flagging.+//!+//! The fix replaces the cap-and-dominance approach with a Pareto check:+//!+//! 1. Cardinality cap raised to **50** (above which the field is+//! almost certainly an ID/free-form column, not a status enum).+//! 2. Sort value frequencies descending. Find the smallest K such+//! that the top-K values cover ≥80% of items.+//! 3. If `K ≤ 5`, the remaining values are "rare" and items+//! containing them are outliers.+//!+//! This unifies both cases the original algorithm partially handled:+//!+//! - **Low cardinality + dominant**: 95×"ok" + 5 errors → top-1 covers+//! 95% → 4 rare values flagged. Same as before.+//! - **Higher cardinality + bimodal**: 60×"info" + 25×"warn" + 15+//! distinct rare errors → top-2 covers 85% → 15 rare values flagged.+//! New, correct, and was missed entirely by the old code.+//! - **Uniform distribution**: 50 distinct values, 2 each → top-K+//! never reaches 80% with K ≤ 5 → skip. Correctly identifies as+//! non-categorical.+//!+//! The same fix lands in `headroom/transforms/smart_crusher.py` later+//! in this PR so the parity fixtures continue to byte-match.++use serde_json::Value;+use std::collections::{BTreeMap, BTreeSet, HashSet};++use super::error_keywords::ERROR_KEYWORDS;++/// Detect items that are structural outliers (error-like or+/// uncommonly-shaped).+///+/// Direct port of `_detect_structural_outliers` (Python+/// `smart_crusher.py:606-650`). Returns deduplicated, ascending-sorted+/// indices.+///+/// # Detection+///+/// 1. **Rare-field outliers**: items containing a field that appears+/// in <20% of the array.+/// 2. **Rare-status outliers**: forwarded to `detect_rare_status_values`,+/// which finds items with statistically rare categorical values+/// (with bug #3 fixed).+pub fn detect_structural_outliers(items: &[Value]) -> Vec<usize> {+ if items.len() < 5 {+ return Vec::new();+ }++ // Field counts across the whole array.+ let mut field_counts: BTreeMap<&str, usize> = BTreeMap::new();+ for item in items {+ if let Some(obj) = item.as_object() {+ for key in obj.keys() {+ *field_counts.entry(key.as_str()).or_insert(0) += 1;+ }+ }+ }++ let n = items.len();+ let common_fields: HashSet<String> = field_counts+ .iter()+ .filter(|(_, &c)| c as f64 >= n as f64 * 0.8)+ .map(|(k, _)| (*k).to_string())+ .collect();+ let rare_fields: HashSet<&str> = field_counts+ .iter()+ .filter(|(_, &c)| (c as f64) < n as f64 * 0.2)+ .map(|(k, _)| *k)+ .collect();++ // Use a BTreeSet for stable order — Python uses `set()` then `list(set(...))`+ // which is non-deterministic order; we pin to ascending for parity tests.+ let mut outlier_set: BTreeSet<usize> = BTreeSet::new();++ // 1. Rare-field outliers.+ for (i, item) in items.iter().enumerate() {+ let Some(obj) = item.as_object() else { continue };+ let has_rare = obj.keys().any(|k| rare_fields.contains(k.as_str()));+ if has_rare {+ outlier_set.insert(i);+ }+ }++ // 2. Rare-status outliers.+ for idx in detect_rare_status_values(items, &common_fields) {+ outlier_set.insert(idx);+ }++ outlier_set.into_iter().collect()+}++/// Detect items with rare values in status-like categorical fields.+///+/// **Bug #3 fix** — see module-level doc. Algorithm:+///+/// 1. Cardinality 2..=50 (was 2..=10 in Python).+/// 2. Pareto check: top-K values covering ≥80% with `K ≤ 5`.+/// 3. Items NOT in top-K → outliers.+///+/// Returns indices in the order they were discovered (mirrors+/// Python's append-order behavior; downstream `detect_structural_outliers`+/// dedupes via BTreeSet).+pub fn detect_rare_status_values(items: &[Value], common_fields: &HashSet<String>) -> Vec<usize> {+ let mut outlier_indices: Vec<usize> = Vec::new();++ // Iterate fields in sorted order for determinism. Python iterates+ // a `set`, which has non-deterministic order — but the eventual+ // output is deduped via the caller's set, so order here only+ // affects which fields drive detection if multiple status-like+ // fields exist. Sorting gives us a stable, fixture-friendly order.+ let mut sorted_fields: Vec<&String> = common_fields.iter().collect();+ sorted_fields.sort();++ for field_name in sorted_fields {+ // Collect this field's values across all items, mirroring Python:+ // `[item.get(field_name) for item in items if isinstance(item, dict) and field_name in item]`+ let values: Vec<&Value> = items+ .iter()+ .filter_map(|item| item.as_object())+ .filter_map(|m| m.get(field_name))+ .collect();++ // Stringify non-null values and dedupe to get cardinality. Python:+ // `unique_values = {str(v) for v in values if v is not None}`+ // We use `python_repr_value`-equivalent stringification: simple+ // scalars use their natural form; nested values use serde_json+ // serialization. This stringification is only used for set-+ // dedup and frequency counting, not surfaced to callers, so the+ // python_repr-vs-json distinction we made for anchors doesn't+ // matter here — the SAME stringification is used for both the+ // "is this rare" computation and the per-item lookup, so the+ // surface is internally consistent.+ let stringify = |v: &Value| -> String {+ match v {+ Value::Null => unreachable!("null filtered above"),+ Value::Bool(b) => b.to_string(),+ Value::Number(n) => n.to_string(),+ Value::String(s) => s.clone(),+ _ => v.to_string(),+ }+ };++ let unique_values: BTreeSet<String> = values+ .iter()+ .filter(|v| !matches!(v, Value::Null))+ .map(|v| stringify(v))+ .collect();++ // Cardinality cap (BUG #3 FIX: was 2..=10, now 2..=50).+ if !(2..=50).contains(&unique_values.len()) {+ continue;+ }++ // Frequency count. Python: `value_counts: dict[str, int]`,+ // `key = str(v) if v is not None else "__none__"`.+ let mut value_counts: BTreeMap<String, usize> = BTreeMap::new();+ for v in &values {+ let key = if matches!(v, Value::Null) {+ "__none__".to_string()+ } else {+ stringify(v)+ };+ *value_counts.entry(key).or_insert(0) += 1;+ }+ if value_counts.is_empty() {+ continue;+ }++ let total = values.len();++ // Pareto check (BUG #3 FIX): find smallest K such that top-K+ // values cover ≥80% of items.+ let mut sorted_counts: Vec<(&String, &usize)> = value_counts.iter().collect();+ // Sort by count descending; tiebreak by key ascending so the+ // result is deterministic when multiple values have the same+ // frequency.+ sorted_counts.sort_by(|a, b| b.1.cmp(a.1).then(a.0.cmp(b.0)));++ let threshold = (total as f64 * 0.8).ceil() as usize;+ let mut cumulative: usize = 0;+ let mut top_k_values: HashSet<String> = HashSet::new();+ for (value, count) in &sorted_counts {+ cumulative += **count;+ top_k_values.insert((*value).clone());+ if cumulative >= threshold {+ break;+ }+ }++ // Only flag rare values if the top-K is small (≤5). Above this+ // the distribution is too uniform to label any value "rare".+ if top_k_values.len() > 5 {+ continue;+ }++ // Items with values NOT in top_k_values are outliers.+ for (i, item) in items.iter().enumerate() {+ let Some(obj) = item.as_object() else { continue };+ let Some(field_value) = obj.get(field_name) else { continue };+ let item_value = if matches!(field_value, Value::Null) {+ "__none__".to_string()+ } else {+ stringify(field_value)+ };+ if !top_k_values.contains(&item_value) {+ outlier_indices.push(i);+ }+ }+ }++ outlier_indices+}++/// Detect items containing error keywords for PRESERVATION.+///+/// Direct port of `_detect_error_items_for_preservation` (Python+/// `smart_crusher.py:711-748`). Used by the orchestrator's+/// `_prioritize_indices` to ensure error items are NEVER dropped.+///+/// # Args+///+/// - `items`: array items to scan.+/// - `item_strings`: pre-computed JSON serializations to avoid+/// redundant `to_string` work. Pass `None` to serialize on the fly.+/// When provided, must be the same length as `items` (Python's+/// bounds-check via `i < len(item_strings)` is mirrored).+pub fn detect_error_items_for_preservation(+ items: &[Value],+ item_strings: Option<&[String]>,+) -> Vec<usize> {+ let mut error_indices: Vec<usize> = Vec::new();++ for (i, item) in items.iter().enumerate() {+ // Python: `if not isinstance(item, dict): continue`. Mirror.+ if !item.is_object() {+ continue;+ }++ // Reuse cached serialization or serialize fresh. Python:+ // `if item_strings is not None and i < len(item_strings):+ // item_str = item_strings[i].lower()+ // else:+ // item_str = json.dumps(item).lower()`+ let serialized: String = match item_strings {+ Some(arr) if i < arr.len() => arr[i].to_lowercase(),+ _ => match serde_json::to_string(item) {+ Ok(s) => s.to_lowercase(),+ Err(_) => continue,+ },+ };++ // Python: `for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:+ // if keyword in item_str: ...; break`+ if ERROR_KEYWORDS.iter().any(|kw| serialized.contains(kw)) {+ error_indices.push(i);+ }+ }++ error_indices+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ // ---------- detect_structural_outliers ----------++ #[test]+ fn outliers_too_few_items_returns_empty() {+ let items: Vec<Value> = (0..3).map(|i| json!({"a": i})).collect();+ assert_eq!(detect_structural_outliers(&items), Vec::<usize>::new());+ }++ #[test]+ fn outliers_rare_field_flags_item() {+ // 9 items with `{"a"}`, 1 item with extra `{"a", "x"}` — `x`+ // appears in 10% of items, below the 20% rare-field threshold.+ let mut items: Vec<Value> = (0..9).map(|i| json!({"a": i})).collect();+ items.push(json!({"a": 9, "x": "rare"}));+ let outliers = detect_structural_outliers(&items);+ assert!(outliers.contains(&9));+ }++ #[test]+ fn outliers_no_dict_items_silently_skipped() {+ // Mixed array: only dict items count. A non-dict in the middle+ // doesn't crash anything.+ let items: Vec<Value> = vec![+ json!({"status": "ok"}),+ json!("string-not-dict"),+ json!({"status": "ok"}),+ json!({"status": "ok"}),+ json!({"status": "ok"}),+ ];+ // Should not panic.+ let _ = detect_structural_outliers(&items);+ }++ // ---------- detect_rare_status_values (BUG #3 FIX coverage) ----------++ #[test]+ fn rare_status_low_cardinality_dominant_value() {+ // Pre-bug-fix path still works: 95×ok + 5 errors.+ let mut items: Vec<Value> = (0..95).map(|_| json!({"status": "ok"})).collect();+ items.push(json!({"status": "error"}));+ items.push(json!({"status": "timeout"}));+ items.push(json!({"status": "error"}));+ items.push(json!({"status": "timeout"}));+ items.push(json!({"status": "fail"}));+ let common: HashSet<String> = ["status".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ // All 5 non-"ok" items flagged.+ assert_eq!(outliers.len(), 5);+ }++ #[test]+ fn rare_status_bug3_fix_high_cardinality_bimodal() {+ // BUG #3 case: cardinality 17 (1 dominant + 1 second + 15+ // singletons). Old code: 17 > 10 → skip. New code: top-2 covers+ // 85%, K=2 ≤ 5, remaining 15 values flagged.+ let mut items: Vec<Value> = Vec::new();+ for _ in 0..60 {+ items.push(json!({"code": "INFO"}));+ }+ for _ in 0..25 {+ items.push(json!({"code": "WARN"}));+ }+ for i in 0..15 {+ items.push(json!({"code": format!("ERR_{}", i)}));+ }+ let common: HashSet<String> = ["code".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ // 15 rare-error items flagged. Pre-fix this would be 0.+ assert_eq!(outliers.len(), 15);+ }++ #[test]+ fn rare_status_uniform_distribution_no_outliers() {+ // 50 items, 50 distinct values, 1 each. Top-K never reaches+ // 80% with K ≤ 5 → no outliers (correctly identified as+ // non-categorical).+ let items: Vec<Value> = (0..50)+ .map(|i| json!({"code": format!("CAT_{}", i)}))+ .collect();+ let common: HashSet<String> = ["code".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ assert!(outliers.is_empty(),+ "uniform distribution must not produce rare-status outliers");+ }++ #[test]+ fn rare_status_cardinality_above_50_skipped() {+ // 60 distinct values → cardinality cap rejects.+ let items: Vec<Value> = (0..60)+ .map(|i| json!({"code": format!("V_{}", i)}))+ .collect();+ let common: HashSet<String> = ["code".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ assert!(outliers.is_empty());+ }++ #[test]+ fn rare_status_cardinality_one_skipped() {+ // 100 items, all same value → cardinality 1, fails 2..=50 gate.+ let items: Vec<Value> = (0..100).map(|_| json!({"status": "ok"})).collect();+ let common: HashSet<String> = ["status".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ assert!(outliers.is_empty());+ }++ #[test]+ fn rare_status_nulls_filtered_from_cardinality() {+ // Pinned Python parity: `unique_values = {str(v) for v in values+ // if v is not None}` — nulls are excluded from the cardinality+ // computation. With 95×"ok" + 5×null, cardinality = 1 (just "ok"),+ // which fails the 2..=50 gate and the field is skipped entirely.+ // Pre-fix Python had the same behavior; the null-aware+ // `__none__` mapping only kicks in inside `value_counts` which+ // is unreachable when the cardinality gate fails first.+ //+ // If we ever want null to count as a distinct categorical value+ // (so missing-status items get flagged), that's a behavior+ // change beyond bug #3 and lives in Stage 3c.2.+ let mut items: Vec<Value> = (0..95).map(|_| json!({"s": "ok"})).collect();+ for _ in 0..5 {+ items.push(json!({"s": null}));+ }+ let common: HashSet<String> = ["s".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ assert!(+ outliers.is_empty(),+ "cardinality 1 (after null filter) must skip the field"+ );+ }++ #[test]+ fn rare_status_nulls_count_in_value_counts_when_cardinality_passes() {+ // Once cardinality >= 2 (with nulls excluded from the set), the+ // value_counts loop maps null → "__none__" and treats it as a+ // distinct value for frequency counting. Mirrors Python's+ // `key = str(v) if v is not None else "__none__"`.+ //+ // Setup: 90×"ok" + 5×"warn" + 5×null → unique_values = {"ok", "warn"},+ // cardinality 2, gate passes. value_counts: ok=90, warn=5,+ // __none__=5. top-1 = "ok" (90/100 = 90%) covers ≥80%, K=1 ≤ 5.+ // Items with "warn" or null are flagged.+ let mut items: Vec<Value> = (0..90).map(|_| json!({"s": "ok"})).collect();+ for _ in 0..5 {+ items.push(json!({"s": "warn"}));+ }+ for _ in 0..5 {+ items.push(json!({"s": null}));+ }+ let common: HashSet<String> = ["s".to_string()].into_iter().collect();+ let outliers = detect_rare_status_values(&items, &common);+ assert_eq!(outliers.len(), 10, "5 warn + 5 null = 10 outliers");+ }++ // ---------- detect_error_items_for_preservation ----------++ #[test]+ fn error_keywords_preservation_basic() {+ let items: Vec<Value> = vec![+ json!({"status": "ok"}),+ json!({"status": "error", "msg": "boom"}),+ json!({"status": "ok"}),+ json!({"msg": "request failed"}),+ json!({"status": "ok"}),+ ];+ let errs = detect_error_items_for_preservation(&items, None);+ assert_eq!(errs, vec![1, 3]);+ }++ #[test]+ fn error_keywords_case_insensitive() {+ let items: Vec<Value> = vec![+ json!({"msg": "FATAL: out of memory"}),+ json!({"msg": "panic at line 42"}),+ ];+ let errs = detect_error_items_for_preservation(&items, None);+ assert_eq!(errs, vec![0, 1]);+ }++ #[test]+ fn error_keywords_no_match() {+ let items: Vec<Value> = vec![+ json!({"name": "alice"}),+ json!({"count": 5}),+ ];+ let errs = detect_error_items_for_preservation(&items, None);+ assert!(errs.is_empty());+ }++ #[test]+ fn error_keywords_uses_cached_strings_when_provided() {+ // If `item_strings` is passed, we use those rather than+ // re-serializing. Test that a custom cached string can drive+ // a hit even when the actual item wouldn't.+ let items: Vec<Value> = vec![json!({"a": 1}), json!({"b": 2})];+ let cached = vec!["error".to_string(), "ok".to_string()];+ let errs = detect_error_items_for_preservation(&items, Some(&cached));+ assert_eq!(errs, vec![0]);+ }++ #[test]+ fn error_keywords_falls_back_when_cache_too_short() {+ // i >= len(item_strings) → fall back to fresh serialization.+ let items: Vec<Value> = vec![json!({"a": 1}), json!({"msg": "error"})];+ let cached = vec!["ok".to_string()]; // only 1 entry+ let errs = detect_error_items_for_preservation(&items, Some(&cached));+ assert_eq!(errs, vec![1]); // index 1 falls back to serialization, hits "error"+ }++ #[test]+ fn error_keywords_skips_non_dict_items() {+ let items: Vec<Value> = vec![+ json!({"msg": "error"}),+ json!("error string"), // not a dict — Python skips+ json!({"msg": "error"}),+ ];+ let errs = detect_error_items_for_preservation(&items, None);+ assert_eq!(errs, vec![0, 2]);+ }+}