Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): SmartCrusher extension surface — Constraint, Observer, Builder
chopratejas57c16b9e2b10
diff --git a/crates/headroom-core/src/transforms/smart_crusher/builder.rs b/crates/headroom-core/src/transforms/smart_crusher/builder.rsnew file mode 100644index 000000000..b4243fad3--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/builder.rs@@ -0,0 +1,219 @@+//! `SmartCrusherBuilder` — explicit composition of the three traits.+//!+//! `SmartCrusher::new(config)` returns the OSS default composition+//! (HybridScorer + KeepErrorsConstraint + KeepStructuralOutliersConstraint+//! + TracingObserver) — drop-in compatible with pre-PR1 callers.+//!+//! Builder is for callers who want to customize the composition:+//!+//! ```ignore+//! use headroom_core::transforms::smart_crusher::{+//! SmartCrusher, SmartCrusherConfig, SmartCrusherBuilder,+//! };+//! // Enterprise: swap the scorer, add a business-rule constraint,+//! // attach an audit observer.+//! let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())+//! .with_scorer(Box::new(my_loop_scorer))+//! .add_default_oss_constraints() // KeepErrors + KeepStructuralOutliers+//! .add_constraint(Box::new(my_business_rule))+//! .add_observer(Box::new(my_audit_observer))+//! .build();+//! ```+//!+//! # Defaults vs explicit+//!+//! `SmartCrusherBuilder::new()` starts EMPTY — no scorer, no+//! constraints, no observers. You get exactly what you ask for. Use+//! [`with_default_oss_setup`](SmartCrusherBuilder::with_default_oss_setup)+//! to start from the OSS default and customize from there. This is+//! the "no silent fallback" rule applied to composition: the builder+//! makes your intent explicit; the `new()` factory shorthand for the+//! OSS preset.++use crate::relevance::{HybridScorer, RelevanceScorer};+use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};++use super::analyzer::SmartAnalyzer;+use super::config::SmartCrusherConfig;+use super::constraints::default_oss_constraints;+use super::crusher::SmartCrusher;+use super::observer::TracingObserver;+use super::traits::{Constraint, Observer};++/// Builder for `SmartCrusher`. See module docs.+pub struct SmartCrusherBuilder {+ config: SmartCrusherConfig,+ anchor_config: Option<AnchorConfig>,+ scorer: Option<Box<dyn RelevanceScorer + Send + Sync>>,+ constraints: Vec<Box<dyn Constraint>>,+ observers: Vec<Box<dyn Observer>>,+}++impl SmartCrusherBuilder {+ /// Empty builder — no scorer, no constraints, no observers.+ pub fn new(config: SmartCrusherConfig) -> Self {+ SmartCrusherBuilder {+ config,+ anchor_config: None,+ scorer: None,+ constraints: Vec::new(),+ observers: Vec::new(),+ }+ }++ /// Override the default `AnchorConfig` (rare — most callers leave+ /// this as the default).+ pub fn anchor_config(mut self, cfg: AnchorConfig) -> Self {+ self.anchor_config = Some(cfg);+ self+ }++ /// Set the relevance scorer. The Enterprise plug-in point — pass+ /// a `LoopScorer`, custom `HybridScorer { adaptive: false, alpha: 0.5 }`,+ /// or any other `RelevanceScorer` impl.+ pub fn with_scorer(mut self, scorer: Box<dyn RelevanceScorer + Send + Sync>) -> Self {+ self.scorer = Some(scorer);+ self+ }++ /// Append a constraint. Constraints stack — the must-keep set is+ /// the union of every constraint's output. Order does not affect+ /// correctness but is preserved in observer event strategy strings+ /// for determinism.+ pub fn add_constraint(mut self, c: Box<dyn Constraint>) -> Self {+ self.constraints.push(c);+ self+ }++ /// Append the OSS default constraint stack (`KeepErrorsConstraint`+ /// plus `KeepStructuralOutliersConstraint`) to the current builder.+ /// Composes naturally with `add_constraint`:+ ///+ /// ```ignore+ /// SmartCrusherBuilder::new(cfg)+ /// .add_default_oss_constraints()+ /// .add_constraint(Box::new(MyBusinessRule))+ /// ```+ pub fn add_default_oss_constraints(mut self) -> Self {+ self.constraints.extend(default_oss_constraints());+ self+ }++ /// Append an observer. Observers stack — every event fires every+ /// observer in registration order.+ pub fn add_observer(mut self, o: Box<dyn Observer>) -> Self {+ self.observers.push(o);+ self+ }++ /// Apply the OSS default setup: `HybridScorer`,+ /// default-OSS-constraints, `TracingObserver`. Equivalent to+ /// `SmartCrusher::new(config)` if no further customization is+ /// applied. Use this when starting from the OSS preset and+ /// adding a few enterprise components.+ pub fn with_default_oss_setup(self) -> Self {+ self.with_scorer(Box::<HybridScorer>::default())+ .add_default_oss_constraints()+ .add_observer(Box::new(TracingObserver))+ }++ /// Construct the `SmartCrusher`. If `with_scorer` was not called,+ /// falls back to `HybridScorer::default()` so a builder with no+ /// other customization still produces a working crusher.+ pub fn build(self) -> SmartCrusher {+ let analyzer = SmartAnalyzer::new(self.config.clone());+ let anchor_selector = AnchorSelector::new(self.anchor_config.unwrap_or_default());+ let scorer = self+ .scorer+ .unwrap_or_else(|| Box::<HybridScorer>::default());+ SmartCrusher::from_parts(+ self.config,+ anchor_selector,+ scorer,+ analyzer,+ self.constraints,+ self.observers,+ )+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::transforms::smart_crusher::traits::{Constraint, CrushEvent, Observer};+ use serde_json::Value;+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::sync::Arc;++ struct MarkerConstraint {+ name: &'static str,+ }+ impl Constraint for MarkerConstraint {+ fn name(&self) -> &str {+ self.name+ }+ fn must_keep(&self, _: &[Value], _: Option<&[String]>) -> Vec<usize> {+ Vec::new()+ }+ }++ struct MarkerObserver {+ count: Arc<AtomicUsize>,+ }+ impl Observer for MarkerObserver {+ fn on_event(&self, _: &CrushEvent) {+ self.count.fetch_add(1, Ordering::SeqCst);+ }+ }++ #[test]+ fn empty_builder_builds_with_default_scorer() {+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default()).build();+ assert!(crusher.constraints.is_empty());+ assert!(crusher.observers.is_empty());+ }++ #[test]+ fn add_default_oss_constraints_appends_two() {+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())+ .add_default_oss_constraints()+ .build();+ assert_eq!(crusher.constraints.len(), 2);+ let names: Vec<&str> = crusher.constraints.iter().map(|c| c.name()).collect();+ assert_eq!(names, vec!["keep_errors", "keep_structural_outliers"]);+ }++ #[test]+ fn add_constraint_preserves_order() {+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())+ .add_constraint(Box::new(MarkerConstraint { name: "first" }))+ .add_constraint(Box::new(MarkerConstraint { name: "second" }))+ .add_constraint(Box::new(MarkerConstraint { name: "third" }))+ .build();+ let names: Vec<&str> = crusher.constraints.iter().map(|c| c.name()).collect();+ assert_eq!(names, vec!["first", "second", "third"]);+ }++ #[test]+ fn with_default_oss_setup_yields_two_constraints_one_observer() {+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())+ .with_default_oss_setup()+ .build();+ assert_eq!(crusher.constraints.len(), 2);+ assert_eq!(crusher.observers.len(), 1);+ }++ #[test]+ fn builder_observer_fires_on_crush() {+ // Wire a counting observer, run a crush, expect exactly one+ // event. Pins the observer integration end-to-end.+ let counter = Arc::new(AtomicUsize::new(0));+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())+ .add_observer(Box::new(MarkerObserver {+ count: counter.clone(),+ }))+ .build();+ let _ = crusher.crush(r#"[1, 2, 3]"#, "", 1.0);+ assert_eq!(counter.load(Ordering::SeqCst), 1);+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/constraints.rs b/crates/headroom-core/src/transforms/smart_crusher/constraints.rsnew file mode 100644index 000000000..a246d73d8--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/constraints.rs@@ -0,0 +1,159 @@+//! OSS default `Constraint` implementations.+//!+//! Two constraints ship in the default OSS composition:+//!+//! - [`KeepErrorsConstraint`] — items containing error/failure+//! keywords (`error`, `exception`, `failed`, `fatal`, etc.). Wraps+//! the existing [`detect_error_items_for_preservation`] function.+//! - [`KeepStructuralOutliersConstraint`] — items with rare fields or+//! rare categorical values. Wraps the existing+//! [`detect_structural_outliers`] function.+//!+//! Both are byte-equivalent to the pre-PR1 hardcoded behavior — the+//! detection logic is unchanged; the constraints are thin trait+//! adapters so that custom Enterprise constraints can be stacked+//! alongside or in place of these defaults.+//!+//! # The default factory+//!+//! [`default_oss_constraints`] returns the OSS default stack as a+//! `Vec<Box<dyn Constraint>>`. `SmartCrusher::new(config)` uses this;+//! `SmartCrusherBuilder::new(config)` does NOT (the builder gives you+//! exactly what you ask for, no surprises). To get OSS defaults plus+//! your own constraints, use `with_default_constraints()` on the+//! builder.++use serde_json::Value;++use super::outliers::{detect_error_items_for_preservation, detect_structural_outliers};+use super::traits::Constraint;++// ── KeepErrorsConstraint ──────────────────────────────────────────────────++/// OSS default: keep items that contain error keywords.+///+/// "Error keyword" is matched case-insensitively against the JSON+/// serialization of each item against the list in [`super::error_keywords::ERROR_KEYWORDS`]+/// (`error`, `exception`, `failed`, `fatal`, `critical`, `crash`, `panic`,+/// `abort`, `timeout`, `denied`, `rejected`).+#[derive(Debug, Default, Clone, Copy)]+pub struct KeepErrorsConstraint;++impl Constraint for KeepErrorsConstraint {+ fn name(&self) -> &str {+ "keep_errors"+ }++ fn must_keep(&self, items: &[Value], item_strings: Option<&[String]>) -> Vec<usize> {+ detect_error_items_for_preservation(items, item_strings)+ }+}++// ── KeepStructuralOutliersConstraint ─────────────────────────────────────++/// OSS default: keep items that are *structurally* unusual within the+/// array. Two flavors of unusual:+///+/// - **Rare fields**: items that have a key present in fewer than the+/// uniqueness threshold of items.+/// - **Rare values for common fields**: items whose value for a+/// high-cardinality field appears infrequently (the "rare-status"+/// path that fires on enums like `level: "ERROR"` among many `INFO`).+///+/// Implementation is unchanged from pre-PR1; this is a thin wrapper.+#[derive(Debug, Default, Clone, Copy)]+pub struct KeepStructuralOutliersConstraint;++impl Constraint for KeepStructuralOutliersConstraint {+ fn name(&self) -> &str {+ "keep_structural_outliers"+ }++ fn must_keep(&self, items: &[Value], _item_strings: Option<&[String]>) -> Vec<usize> {+ detect_structural_outliers(items)+ }+}++// ── Default OSS stack ─────────────────────────────────────────────────────++/// Returns the default OSS constraint stack used by+/// `SmartCrusher::new(config)`.+///+/// Stack contents (in order):+/// 1. [`KeepErrorsConstraint`]+/// 2. [`KeepStructuralOutliersConstraint`]+///+/// The order does not affect output (the must-keep set is a union),+/// but is fixed for determinism in observer-emitted strategy strings+/// and audit logs.+pub fn default_oss_constraints() -> Vec<Box<dyn Constraint>> {+ vec![+ Box::new(KeepErrorsConstraint),+ Box::new(KeepStructuralOutliersConstraint),+ ]+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn keep_errors_constraint_finds_error_items() {+ // 9 normal items + 1 with "ERROR" keyword.+ let mut items: Vec<Value> = (0..9).map(|i| json!({"id": i, "status": "ok"})).collect();+ items.push(json!({"id": 9, "status": "ERROR", "msg": "FATAL: boom"}));+ let kept = KeepErrorsConstraint.must_keep(&items, None);+ // Exact index 9 must be in the result.+ assert!(kept.contains(&9), "error item must be flagged for keep");+ }++ #[test]+ fn keep_errors_constraint_uses_item_strings_when_provided() {+ // Pre-computed strings parity with the on-the-fly path: same+ // content, same indices returned.+ let items: Vec<Value> = vec![json!({"a": 1}), json!({"a": "exception"})];+ let strings: Vec<String> = items+ .iter()+ .map(|v| serde_json::to_string(v).unwrap())+ .collect();+ let with_cache = KeepErrorsConstraint.must_keep(&items, Some(&strings));+ let without_cache = KeepErrorsConstraint.must_keep(&items, None);+ assert_eq!(with_cache, without_cache);+ assert!(with_cache.contains(&1));+ }++ #[test]+ fn keep_structural_outliers_constraint_returns_indices() {+ // Build an array where one item has a unique extra field —+ // it should be flagged as a rare-field outlier.+ let mut items: Vec<Value> = (0..20)+ .map(|i| json!({"id": i, "kind": "common"}))+ .collect();+ items.push(json!({"id": 20, "kind": "common", "rare_extra_field": "x"}));+ let kept = KeepStructuralOutliersConstraint.must_keep(&items, None);+ assert!(+ kept.contains(&20),+ "item with rare field should be a structural outlier"+ );+ }++ #[test]+ fn default_oss_constraints_returns_two() {+ let cs = default_oss_constraints();+ assert_eq!(cs.len(), 2);+ let names: Vec<&str> = cs.iter().map(|c| c.name()).collect();+ assert_eq!(names, vec!["keep_errors", "keep_structural_outliers"]);+ }++ #[test]+ fn constraints_handle_empty_array() {+ // No panics, no allocations beyond an empty Vec — the array+ // path will bypass us when items is empty, but constraints+ // must still be safe to call.+ assert!(KeepErrorsConstraint.must_keep(&[], None).is_empty());+ assert!(KeepStructuralOutliersConstraint+ .must_keep(&[], None)+ .is_empty());+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex ad7b4ce19..2bf8b5536 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -36,14 +36,16 @@use serde_json::Value;use super::analyzer::SmartAnalyzer;+use super::builder::SmartCrusherBuilder;use super::classifier::{classify_array, ArrayType};use super::config::SmartCrusherConfig;use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};use super::planning::SmartCrusherPlanner;+use super::traits::{Constraint, CrushEvent, Observer};use super::types::{CompressionPlan, CompressionStrategy, CrushResult};-use crate::relevance::{HybridScorer, RelevanceScorer};+use crate::relevance::RelevanceScorer;use crate::transforms::adaptive_sizer::compute_optimal_k;-use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};+use crate::transforms::anchor_selector::AnchorSelector;/// Return type for `crush_array` — mirrors Python's/// `(crushed_items, strategy_info, ccr_hash, dropped_summary)` tuple.@@ -59,39 +61,76 @@ pub struct CrushArrayResult {}/// Top-level SmartCrusher.+///+/// Three pluggable extensions (Stage 3c.2 PR1):+/// - `scorer` — relevance scoring (`HybridScorer` by default).+/// - `constraints` — must-keep predicates (`KeepErrorsConstraint` ++/// `KeepStructuralOutliersConstraint` by default).+/// - `observers` — decision-stream telemetry (`TracingObserver` by+/// default).+///+/// Compose via [`SmartCrusherBuilder`]; or call `SmartCrusher::new()`+/// for the OSS default composition.pub struct SmartCrusher {pub config: SmartCrusherConfig,pub anchor_selector: AnchorSelector,pub scorer: Box<dyn RelevanceScorer + Send + Sync>,pub analyzer: SmartAnalyzer,+ pub constraints: Vec<Box<dyn Constraint>>,+ pub observers: Vec<Box<dyn Observer>>,}impl SmartCrusher {- /// Construct with the default scorer (`HybridScorer`) and the- /// default `AnchorConfig`. Mirrors Python's- /// `SmartCrusher(config=...)` no-extra-args path with all- /// optional subsystems disabled.+ /// Construct with the OSS default composition: `HybridScorer` ++ /// `KeepErrorsConstraint` + `KeepStructuralOutliersConstraint` ++ /// `TracingObserver`. Drop-in for callers that don't need+ /// custom extensions — pre-PR1 behavior is preserved exactly.pub fn new(config: SmartCrusherConfig) -> Self {- let analyzer = SmartAnalyzer::new(config.clone());- SmartCrusher {- config,- anchor_selector: AnchorSelector::new(AnchorConfig::default()),- scorer: Box::new(HybridScorer::default()),- analyzer,- }+ SmartCrusherBuilder::new(config)+ .with_default_oss_setup()+ .build()}- /// Construct with a custom scorer.+ /// Begin a builder chain for custom composition. The Enterprise+ /// entry point: swap the scorer, add business-rule constraints,+ /// attach an audit observer.+ pub fn builder(config: SmartCrusherConfig) -> SmartCrusherBuilder {+ SmartCrusherBuilder::new(config)+ }++ /// Construct with a custom scorer (legacy convenience). Equivalent+ /// to `SmartCrusher::builder(config).with_scorer(scorer).with_default_oss_setup().build()`+ /// minus the default scorer override; preserved for backward+ /// compatibility with pre-PR1 callers.pub fn with_scorer(config: SmartCrusherConfig,scorer: Box<dyn RelevanceScorer + Send + Sync>,) -> Self {- let analyzer = SmartAnalyzer::new(config.clone());+ SmartCrusherBuilder::new(config)+ .with_scorer(scorer)+ .add_default_oss_constraints()+ .build()+ }++ /// Construct directly from owned parts. Used by+ /// [`SmartCrusherBuilder::build`] — not part of the public stable+ /// API. Prefer the builder.+ #[doc(hidden)]+ pub fn from_parts(+ config: SmartCrusherConfig,+ anchor_selector: AnchorSelector,+ scorer: Box<dyn RelevanceScorer + Send + Sync>,+ analyzer: SmartAnalyzer,+ constraints: Vec<Box<dyn Constraint>>,+ observers: Vec<Box<dyn Observer>>,+ ) -> Self {SmartCrusher {config,- anchor_selector: AnchorSelector::new(AnchorConfig::default()),+ anchor_selector,scorer,analyzer,+ constraints,+ observers,}}@@ -101,6 +140,7 @@ impl SmartCrusher {&self.anchor_selector,&*self.scorer,&self.analyzer,+ &self.constraints,)}@@ -137,12 +177,33 @@ impl SmartCrusher {/// - `strategy`: combined strategy info from all crushed arrays/// (or `"passthrough"`).pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {+ let start = std::time::Instant::now();let (compressed, was_modified, info) = self.smart_crush_content(content, query, bias);let strategy = if info.is_empty() {"passthrough".to_string()} else {info};++ // Fire one event per top-level crush. Cheap when no observers+ // are configured (`for o in &[]` is a single null-pointer+ // check); cheap when only `TracingObserver` is configured if+ // the subscriber filters `debug` out (the default in+ // production). Custom observers — audit logs, Loop training+ // stream, metrics — pay whatever they pay.+ if !self.observers.is_empty() {+ let event = CrushEvent {+ strategy: strategy.clone(),+ input_bytes: content.len(),+ output_bytes: compressed.len(),+ elapsed_ns: start.elapsed().as_nanos() as u64,+ was_modified,+ };+ for observer in &self.observers {+ observer.on_event(&event);+ }+ }+CrushResult {compressed,original: content.to_string(),diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rsindex 760d850b9..333af8a61 100644--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs@@ -34,29 +34,38 @@mod analyzer;mod anchors;+mod builder;mod classifier;mod config;+mod constraints;mod crusher;mod crushers;mod error_keywords;mod field_detect;mod hashing;+mod observer;mod orchestration;mod outliers;mod planning;mod statistics;mod stats_math;+mod traits;mod types;pub use analyzer::SmartAnalyzer;pub use anchors::{extract_query_anchors, item_matches_anchors};+pub use builder::SmartCrusherBuilder;pub use classifier::{classify_array, ArrayType};pub use config::SmartCrusherConfig;+pub use constraints::{+ default_oss_constraints, KeepErrorsConstraint, KeepStructuralOutliersConstraint,+};pub use crusher::{CrushArrayResult, SmartCrusher};pub use crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};pub use error_keywords::ERROR_KEYWORDS;pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};pub use hashing::hash_field_name;+pub use observer::TracingObserver;pub use orchestration::{deduplicate_indices_by_content, fill_remaining_slots, prioritize_indices};pub use outliers::{detect_error_items_for_preservation, detect_rare_status_values, detect_structural_outliers,@@ -64,6 +73,7 @@ pub use outliers::{pub use planning::{item_has_preserve_field_match, map_to_anchor_pattern, SmartCrusherPlanner};pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};pub use stats_math::{format_g, mean, median, sample_stdev, sample_variance};+pub use traits::{Constraint, CrushEvent, Observer, Scorer};pub use types::{ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,FieldStats,diff --git a/crates/headroom-core/src/transforms/smart_crusher/observer.rs b/crates/headroom-core/src/transforms/smart_crusher/observer.rsnew file mode 100644index 000000000..68fc63109--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/observer.rs@@ -0,0 +1,66 @@+//! OSS default `Observer` implementation.+//!+//! Ships [`TracingObserver`] which writes each `CrushEvent` to the+//! `tracing` crate at `debug` level. Subscribers that filter `debug`+//! out (typical production config) pay nothing — `tracing` stops+//! evaluation at the level check before constructing the event+//! fields. Subscribers that retain `debug` get a structured per-crush+//! event suitable for log analytics.+//!+//! Enterprise consumers ship richer observers — `AuditObserver` for+//! SOC2/HIPAA decision logs, `MetricsObserver` for Datadog/Atlas+//! gauges, `LoopTrainingObserver` to stream events to Headroom Loop —+//! all on the same trait.++use super::traits::{CrushEvent, Observer};++/// Writes each `CrushEvent` to the `tracing` crate at `debug` level.+/// Zero-cost when the subscriber filters `debug` out.+#[derive(Debug, Default, Clone, Copy)]+pub struct TracingObserver;++impl Observer for TracingObserver {+ fn name(&self) -> &str {+ "tracing"+ }++ fn on_event(&self, event: &CrushEvent) {+ // `tracing::debug!` is a macro; the level check happens before+ // the fields are evaluated, so this is essentially free at+ // higher log levels.+ tracing::debug!(+ target: "headroom::smart_crusher",+ strategy = %event.strategy,+ input_bytes = event.input_bytes,+ output_bytes = event.output_bytes,+ elapsed_ns = event.elapsed_ns,+ was_modified = event.was_modified,+ "smart_crusher.crush emitted",+ );+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn tracing_observer_does_not_panic_on_event() {+ // We can't easily assert what `tracing` did without a+ // subscriber set up — but the call must not panic, and the+ // event fields must be readable.+ let event = CrushEvent {+ strategy: "passthrough".to_string(),+ input_bytes: 100,+ output_bytes: 100,+ elapsed_ns: 0,+ was_modified: false,+ };+ TracingObserver.on_event(&event);+ }++ #[test]+ fn tracing_observer_name_is_stable() {+ assert_eq!(TracingObserver.name(), "tracing");+ }+}diff --git a/crates/headroom-core/src/transforms/smart_crusher/planning.rs b/crates/headroom-core/src/transforms/smart_crusher/planning.rsindex 7816d6e21..888e7daf7 100644--- a/crates/headroom-core/src/transforms/smart_crusher/planning.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/planning.rs@@ -35,8 +35,12 @@ use super::config::SmartCrusherConfig;use super::field_detect::detect_score_field_statistically;use super::hashing::hash_field_name;use super::orchestration::prioritize_indices;-use super::outliers::{detect_error_items_for_preservation, detect_structural_outliers};+use super::traits::Constraint;use super::types::{ArrayAnalysis, CompressionPlan, CompressionStrategy, FieldStats};+// Note: `detect_error_items_for_preservation` and `detect_structural_outliers`+// are still imported transitively by `constraints.rs` (via `KeepErrorsConstraint`+// and `KeepStructuralOutliersConstraint`). Planning no longer calls them+// directly; it iterates `self.constraints` via `apply_constraints`.use crate::relevance::RelevanceScorer;use crate::transforms::anchor_selector::{AnchorSelector, DataPattern};@@ -47,6 +51,12 @@ pub struct SmartCrusherPlanner<'a> {pub anchor_selector: &'a AnchorSelector,pub scorer: &'a (dyn RelevanceScorer + Send + Sync),pub analyzer: &'a SmartAnalyzer,+ /// User-configured must-keep predicates. The plan methods union+ /// the output of every constraint into the kept set; OSS default+ /// composition includes `KeepErrorsConstraint` and+ /// `KeepStructuralOutliersConstraint`, reproducing the pre-PR1+ /// hardcoded behavior byte-for-byte.+ pub constraints: &'a [Box<dyn Constraint>],}impl<'a> SmartCrusherPlanner<'a> {@@ -55,12 +65,31 @@ impl<'a> SmartCrusherPlanner<'a> {anchor_selector: &'a AnchorSelector,scorer: &'a (dyn RelevanceScorer + Send + Sync),analyzer: &'a SmartAnalyzer,+ constraints: &'a [Box<dyn Constraint>],) -> Self {SmartCrusherPlanner {config,anchor_selector,scorer,analyzer,+ constraints,+ }+ }++ /// Apply every configured `Constraint::must_keep` and union the+ /// results into `keep`. Replaces the hardcoded+ /// `detect_error_items_for_preservation` ++ /// `detect_structural_outliers` calls that lived in each plan+ /// method. With the OSS default constraint stack the output is+ /// byte-identical to pre-PR1 behavior.+ fn apply_constraints(+ &self,+ items: &[Value],+ item_strings: Option<&[String]>,+ keep: &mut BTreeSet<usize>,+ ) {+ for c in self.constraints {+ keep.extend(c.must_keep(items, item_strings));}}@@ -159,9 +188,8 @@ impl<'a> SmartCrusherPlanner<'a> {query_or_none(query_context),));- // 2. Structural outliers + error keywords.- keep.extend(detect_structural_outliers(items));- keep.extend(detect_error_items_for_preservation(items, item_strings));+ // 2. Structural outliers + error keywords (configured via Constraint trait).+ self.apply_constraints(items, item_strings, &mut keep);// 3. Numeric anomalies (>variance_threshold σ from per-field mean).for (name, stats) in &analysis.field_stats {@@ -260,9 +288,8 @@ impl<'a> SmartCrusherPlanner<'a> {keep.insert(*idx);}- // 2. Structural outliers + error keywords.- keep.extend(detect_structural_outliers(items));- keep.extend(detect_error_items_for_preservation(items, item_strings));+ // 2. Structural outliers + error keywords (configured via Constraint trait).+ self.apply_constraints(items, item_strings, &mut keep);// 3. Query-anchor matches (additive — preserved regardless of top-N).if !query_context.is_empty() {@@ -335,9 +362,8 @@ impl<'a> SmartCrusherPlanner<'a> {query_or_none(query_context),));- // 2. Structural outliers + error keywords.- keep.extend(detect_structural_outliers(items));- keep.extend(detect_error_items_for_preservation(items, item_strings));+ // 2. Structural outliers + error keywords (configured via Constraint trait).+ self.apply_constraints(items, item_strings, &mut keep);// 3. Cluster by message-like field (highest unique_ratio > 0.3).let mut message_field: Option<&str> = None;@@ -424,9 +450,8 @@ impl<'a> SmartCrusherPlanner<'a> {}}- // 3. Structural outliers + error keywords.- keep.extend(detect_structural_outliers(items));- keep.extend(detect_error_items_for_preservation(items, item_strings));+ // 3. Structural outliers + error keywords (configured via Constraint trait).+ self.apply_constraints(items, item_strings, &mut keep);// 4/5. Query signals.self.apply_query_signals(items, query_context, item_strings, &mut keep, false);@@ -613,6 +638,7 @@ mod tests {use super::*;use crate::relevance::HybridScorer;use crate::transforms::anchor_selector::AnchorConfig;+ use crate::transforms::smart_crusher::constraints::default_oss_constraints;use serde_json::json;fn fixture<'a>(@@ -620,8 +646,9 @@ mod tests {anchor_selector: &'a AnchorSelector,scorer: &'a HybridScorer,analyzer: &'a SmartAnalyzer,+ constraints: &'a [Box<dyn Constraint>],) -> SmartCrusherPlanner<'a> {- SmartCrusherPlanner::new(config, anchor_selector, scorer, analyzer)+ SmartCrusherPlanner::new(config, anchor_selector, scorer, analyzer, constraints)}fn make_planner_deps() -> (@@ -629,12 +656,14 @@ mod tests {AnchorSelector,HybridScorer,SmartAnalyzer,+ Vec<Box<dyn Constraint>>,) {let cfg = SmartCrusherConfig::default();let asel = AnchorSelector::new(AnchorConfig::default());let scorer = HybridScorer::default();let analyzer = SmartAnalyzer::new(cfg.clone());- (cfg, asel, scorer, analyzer)+ let constraints = default_oss_constraints();+ (cfg, asel, scorer, analyzer, constraints)}// ---------- map_to_anchor_pattern ----------@@ -703,8 +732,8 @@ mod tests {#[test]fn create_plan_skip_returns_all_indices() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);let analysis = ArrayAnalysis {item_count: 5,field_stats: BTreeMap::new(),@@ -721,8 +750,8 @@ mod tests {#[test]fn create_plan_routes_smart_sample_to_smart_sample() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "v": i})).collect();let analysis = analyzer.analyze_array(&items);let plan = p.create_plan(&analysis, &items, "", None, Some(15), None);@@ -736,8 +765,8 @@ mod tests {#[test]fn smart_sample_keeps_error_items() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);let mut items: Vec<Value> = (0..30).map(|i| json!({"id": i, "msg": format!("ok {}", i)})).collect();@@ -756,8 +785,8 @@ mod tests {#[test]fn smart_sample_query_anchor_pinned() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);let items: Vec<Value> = (0..30).map(|i| {json!({@@ -786,8 +815,8 @@ mod tests {#[test]fn top_n_falls_back_when_no_score_field() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);// No bounded score field — top_n falls through to smart_sample.let items: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();let analysis = analyzer.analyze_array(&items);@@ -802,8 +831,8 @@ mod tests {#[test]fn top_n_keeps_highest_scored_items() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);// 20 items with score 0.0..0.95 in 0.05 increments. Top-K// should be the highest scores.let items: Vec<Value> = (0..20)@@ -826,8 +855,8 @@ mod tests {#[test]fn cluster_sample_assigns_cluster_field() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);// Logs-shaped data: high-cardinality message + low-cardinality level.let items: Vec<Value> = (0..30).map(|i| {@@ -852,8 +881,8 @@ mod tests {#[test]fn time_series_keeps_window_around_change_points() {- let (cfg, asel, scorer, analyzer) = make_planner_deps();- let p = fixture(&cfg, &asel, &scorer, &analyzer);+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);// 30 items with a step at index 15; analyzer should detect// change points around there.let items: Vec<Value> = (0..60)diff --git a/crates/headroom-core/src/transforms/smart_crusher/traits.rs b/crates/headroom-core/src/transforms/smart_crusher/traits.rsnew file mode 100644index 000000000..620cd7b14--- /dev/null+++ b/crates/headroom-core/src/transforms/smart_crusher/traits.rs@@ -0,0 +1,203 @@+//! Public extension surface for `SmartCrusher` (Stage 3c.2 PR 1).+//!+//! Three traits — `Scorer`, `Constraint`, `Observer` — capture every+//! decision a `SmartCrusher` makes that downstream consumers might+//! want to override:+//!+//! - **`Scorer`** (already public via `crate::relevance::RelevanceScorer`):+//! how relevant is item *i* to query *q*? OSS default `HybridScorer`+//! (BM25 + fastembed). Enterprise can plug in a per-tenant Loop-trained+//! scorer.+//! - **`Constraint`** (this module): which indices must be kept+//! regardless of score? OSS defaults preserve errors and structural+//! outliers. Enterprise can add `BusinessRuleConstraint`,+//! `RegulatoryConstraint`, etc.+//! - **`Observer`** (this module): emit a structured event after each+//! `crush()` so telemetry, audit logs, and continuous-eval pipelines+//! can hook in. OSS default writes to the `tracing` crate.+//!+//! # Why three, not eight+//!+//! The 5-stage pipeline (classify → compact → score → allocate →+//! format) has more stage boundaries, but only three of them carry+//! *differentiated value* to Enterprise customers — Loop scorer,+//! business rules, audit telemetry. The other stages stay as concrete+//! Rust types; if an Enterprise customer ever needs to plug into a+//! different stage we can promote it to a trait at that point. We're+//! not designing for hypothetical futures — we're naming the seams+//! that real customers will pay for today.+//!+//! # Composition+//!+//! Use [`SmartCrusherBuilder`](super::builder::SmartCrusherBuilder) to+//! compose a custom `SmartCrusher`. The default OSS composition is+//! reachable via `SmartCrusher::new(config)` and stays byte-equivalent+//! to pre-PR1 behavior — all 17 parity fixtures pass.++use serde_json::Value;++// ── Constraint ────────────────────────────────────────────────────────────++/// A hard preservation constraint: indices the allocator must keep+/// regardless of saliency score or token budget.+///+/// Constraints stack — the must-keep set is the union of every+/// constraint's `must_keep` output. OSS ships [`KeepErrorsConstraint`]+/// and [`KeepStructuralOutliersConstraint`] (wrappers around the+/// existing detection functions); Enterprise crates can add+/// `BusinessRuleConstraint("amount > 10000")`,+/// `RegulatoryConstraint::HIPAA`, and so on.+///+/// # Contract+///+/// - **`must_keep` returns indices into `items`.** Out-of-bounds+/// indices are silently dropped by the allocator; constraints+/// should not return them.+/// - **Idempotent in the small.** Calling `must_keep` twice with the+/// same `items` returns the same set; constraints do not own+/// mutable state that drifts between calls within one+/// `SmartCrusher::crush` invocation. (Caching across crushes is+/// fine — see `LoopScorer` style stateful enrichers.)+/// - **Cheap is free.** Constraints run on every dict-array crush.+/// A constraint that does I/O or heavy regex per-item can dominate+/// the crusher's cost. Aim for O(n) on items with small constants.+///+/// # Why `item_strings: Option<&[String]>`?+///+/// Many constraints search the JSON serialization for keywords (the+/// existing `detect_error_items_for_preservation` does this). The+/// caller may already have computed `item_strings` for adaptive+/// sizing; passing them through avoids redundant `serde_json::to_string`+/// calls. Constraints that don't need the strings simply ignore the+/// argument.+pub trait Constraint: Send + Sync {+ /// Stable identifier — appears in `CrushEvent` strategy strings,+ /// audit logs, and config-validation diagnostics. Use snake_case+ /// (`"keep_errors"`, `"business_rule"`).+ fn name(&self) -> &str;++ /// Indices of items the allocator MUST keep.+ fn must_keep(&self, items: &[Value], item_strings: Option<&[String]>) -> Vec<usize>;+}++// ── Observer ──────────────────────────────────────────────────────────────++/// Telemetry event emitted at the end of each `SmartCrusher::crush`+/// call. Observers (multiple, stacked) consume these for `tracing`,+/// audit logs, Loop training data, real-time dashboards.+#[derive(Debug, Clone)]+pub struct CrushEvent {+ /// Strategy debug string returned by the crusher+ /// (e.g. `"smart_sample(30->15)"`, `"passthrough"`,+ /// `"top_n(50->15)"`).+ pub strategy: String,+ /// Length in bytes of the input content (whatever was passed to+ /// `crush()`).+ pub input_bytes: usize,+ /// Length in bytes of the compressed output.+ pub output_bytes: usize,+ /// Wall-clock duration of the `crush()` call.+ pub elapsed_ns: u64,+ /// Whether the output differs from the input.+ pub was_modified: bool,+}++/// Decision-stream hook. Called after each top-level `SmartCrusher::crush`+/// returns; observers run synchronously on the crusher's thread.+///+/// # Contract+///+/// - **Cheap is free.** Like constraints, observers run on every+/// crush. `tracing::debug!` is essentially free when the subscriber+/// filters the level out; remote network calls are not.+/// - **Don't panic.** A panicking observer aborts the calling thread.+/// If your observer does I/O, catch and log errors yourself.+/// - **Order matters.** Observers fire in the order they were added+/// to the builder. Most callers should not depend on the order+/// (telemetry is naturally idempotent), but if you have an+/// audit-then-publish chain, add them in that order.+pub trait Observer: Send + Sync {+ /// Stable identifier — useful for filtering in the rare cases+ /// where an observer wants to disable itself when another is+ /// already configured. Default: the type name.+ fn name(&self) -> &str {+ std::any::type_name::<Self>()+ }++ /// Called once per `SmartCrusher::crush` invocation, after the+ /// result is computed and before it is returned to the caller.+ fn on_event(&self, event: &CrushEvent);+}++// ── Re-exports for convenience ────────────────────────────────────────────++// Scorer is in the relevance crate, not here; re-export so callers+// can get all three traits from one path.+pub use crate::relevance::RelevanceScorer as Scorer;++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::sync::Arc;++ /// A tiny constraint that always keeps index 0 (if any items).+ /// Pins the trait shape and the additive behavior of constraint+ /// stacking.+ struct AlwaysKeepFirst;+ impl Constraint for AlwaysKeepFirst {+ fn name(&self) -> &str {+ "always_keep_first"+ }+ fn must_keep(&self, items: &[Value], _: Option<&[String]>) -> Vec<usize> {+ if items.is_empty() {+ Vec::new()+ } else {+ vec![0]+ }+ }+ }++ #[test]+ fn constraint_returns_indices_in_bounds() {+ let items = vec![json!({"a": 1}), json!({"a": 2})];+ let c = AlwaysKeepFirst;+ let kept = c.must_keep(&items, None);+ assert_eq!(kept, vec![0]);+ assert_eq!(c.name(), "always_keep_first");+ }++ #[test]+ fn constraint_handles_empty_input() {+ let kept = AlwaysKeepFirst.must_keep(&[], None);+ assert!(kept.is_empty());+ }++ /// Counts events to verify the observer trait fires correctly+ /// when wired into a SmartCrusher (integration test in+ /// `crusher.rs::tests`).+ #[derive(Default)]+ struct CountingObserver {+ count: Arc<AtomicUsize>,+ }+ impl Observer for CountingObserver {+ fn on_event(&self, _: &CrushEvent) {+ self.count.fetch_add(1, Ordering::SeqCst);+ }+ }++ #[test]+ fn observer_event_carries_strategy_and_sizes() {+ let observer = CountingObserver::default();+ let event = CrushEvent {+ strategy: "smart_sample(30->15)".to_string(),+ input_bytes: 1000,+ output_bytes: 500,+ elapsed_ns: 12_345,+ was_modified: true,+ };+ observer.on_event(&event);+ assert_eq!(observer.count.load(Ordering::SeqCst), 1);+ }+}