Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): SmartCrusher extension surface — Constraint, Observer, Builder

chopratejas57c16b9e2b10
diff --git a/crates/headroom-core/src/transforms/smart_crusher/builder.rs b/crates/headroom-core/src/transforms/smart_crusher/builder.rs
new file mode 100644
index 000000000..b4243fad3
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/builder.rs
@@ -0,0 +1,219 @@
+//! `SmartCrusherBuilder` — explicit composition of the three traits.
+//!
+//! `SmartCrusher::new(config)` returns the OSS default composition
+//! (HybridScorer + KeepErrorsConstraint + KeepStructuralOutliersConstraint
+//! + TracingObserver) — drop-in compatible with pre-PR1 callers.
+//!
+//! Builder is for callers who want to customize the composition:
+//!
+//! ```ignore
+//! use headroom_core::transforms::smart_crusher::{
+//! SmartCrusher, SmartCrusherConfig, SmartCrusherBuilder,
+//! };
+//! // Enterprise: swap the scorer, add a business-rule constraint,
+//! // attach an audit observer.
+//! let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())
+//! .with_scorer(Box::new(my_loop_scorer))
+//! .add_default_oss_constraints() // KeepErrors + KeepStructuralOutliers
+//! .add_constraint(Box::new(my_business_rule))
+//! .add_observer(Box::new(my_audit_observer))
+//! .build();
+//! ```
+//!
+//! # Defaults vs explicit
+//!
+//! `SmartCrusherBuilder::new()` starts EMPTY — no scorer, no
+//! constraints, no observers. You get exactly what you ask for. Use
+//! [`with_default_oss_setup`](SmartCrusherBuilder::with_default_oss_setup)
+//! to start from the OSS default and customize from there. This is
+//! the "no silent fallback" rule applied to composition: the builder
+//! makes your intent explicit; the `new()` factory shorthand for the
+//! OSS preset.
+
+use crate::relevance::{HybridScorer, RelevanceScorer};
+use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};
+
+use super::analyzer::SmartAnalyzer;
+use super::config::SmartCrusherConfig;
+use super::constraints::default_oss_constraints;
+use super::crusher::SmartCrusher;
+use super::observer::TracingObserver;
+use super::traits::{Constraint, Observer};
+
+/// Builder for `SmartCrusher`. See module docs.
+pub struct SmartCrusherBuilder {
+ config: SmartCrusherConfig,
+ anchor_config: Option<AnchorConfig>,
+ scorer: Option<Box<dyn RelevanceScorer + Send + Sync>>,
+ constraints: Vec<Box<dyn Constraint>>,
+ observers: Vec<Box<dyn Observer>>,
+}
+
+impl SmartCrusherBuilder {
+ /// Empty builder — no scorer, no constraints, no observers.
+ pub fn new(config: SmartCrusherConfig) -> Self {
+ SmartCrusherBuilder {
+ config,
+ anchor_config: None,
+ scorer: None,
+ constraints: Vec::new(),
+ observers: Vec::new(),
+ }
+ }
+
+ /// Override the default `AnchorConfig` (rare — most callers leave
+ /// this as the default).
+ pub fn anchor_config(mut self, cfg: AnchorConfig) -> Self {
+ self.anchor_config = Some(cfg);
+ self
+ }
+
+ /// Set the relevance scorer. The Enterprise plug-in point — pass
+ /// a `LoopScorer`, custom `HybridScorer { adaptive: false, alpha: 0.5 }`,
+ /// or any other `RelevanceScorer` impl.
+ pub fn with_scorer(mut self, scorer: Box<dyn RelevanceScorer + Send + Sync>) -> Self {
+ self.scorer = Some(scorer);
+ self
+ }
+
+ /// Append a constraint. Constraints stack — the must-keep set is
+ /// the union of every constraint's output. Order does not affect
+ /// correctness but is preserved in observer event strategy strings
+ /// for determinism.
+ pub fn add_constraint(mut self, c: Box<dyn Constraint>) -> Self {
+ self.constraints.push(c);
+ self
+ }
+
+ /// Append the OSS default constraint stack (`KeepErrorsConstraint`
+ /// plus `KeepStructuralOutliersConstraint`) to the current builder.
+ /// Composes naturally with `add_constraint`:
+ ///
+ /// ```ignore
+ /// SmartCrusherBuilder::new(cfg)
+ /// .add_default_oss_constraints()
+ /// .add_constraint(Box::new(MyBusinessRule))
+ /// ```
+ pub fn add_default_oss_constraints(mut self) -> Self {
+ self.constraints.extend(default_oss_constraints());
+ self
+ }
+
+ /// Append an observer. Observers stack — every event fires every
+ /// observer in registration order.
+ pub fn add_observer(mut self, o: Box<dyn Observer>) -> Self {
+ self.observers.push(o);
+ self
+ }
+
+ /// Apply the OSS default setup: `HybridScorer`,
+ /// default-OSS-constraints, `TracingObserver`. Equivalent to
+ /// `SmartCrusher::new(config)` if no further customization is
+ /// applied. Use this when starting from the OSS preset and
+ /// adding a few enterprise components.
+ pub fn with_default_oss_setup(self) -> Self {
+ self.with_scorer(Box::<HybridScorer>::default())
+ .add_default_oss_constraints()
+ .add_observer(Box::new(TracingObserver))
+ }
+
+ /// Construct the `SmartCrusher`. If `with_scorer` was not called,
+ /// falls back to `HybridScorer::default()` so a builder with no
+ /// other customization still produces a working crusher.
+ pub fn build(self) -> SmartCrusher {
+ let analyzer = SmartAnalyzer::new(self.config.clone());
+ let anchor_selector = AnchorSelector::new(self.anchor_config.unwrap_or_default());
+ let scorer = self
+ .scorer
+ .unwrap_or_else(|| Box::<HybridScorer>::default());
+ SmartCrusher::from_parts(
+ self.config,
+ anchor_selector,
+ scorer,
+ analyzer,
+ self.constraints,
+ self.observers,
+ )
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::transforms::smart_crusher::traits::{Constraint, CrushEvent, Observer};
+ use serde_json::Value;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+
+ struct MarkerConstraint {
+ name: &'static str,
+ }
+ impl Constraint for MarkerConstraint {
+ fn name(&self) -> &str {
+ self.name
+ }
+ fn must_keep(&self, _: &[Value], _: Option<&[String]>) -> Vec<usize> {
+ Vec::new()
+ }
+ }
+
+ struct MarkerObserver {
+ count: Arc<AtomicUsize>,
+ }
+ impl Observer for MarkerObserver {
+ fn on_event(&self, _: &CrushEvent) {
+ self.count.fetch_add(1, Ordering::SeqCst);
+ }
+ }
+
+ #[test]
+ fn empty_builder_builds_with_default_scorer() {
+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default()).build();
+ assert!(crusher.constraints.is_empty());
+ assert!(crusher.observers.is_empty());
+ }
+
+ #[test]
+ fn add_default_oss_constraints_appends_two() {
+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())
+ .add_default_oss_constraints()
+ .build();
+ assert_eq!(crusher.constraints.len(), 2);
+ let names: Vec<&str> = crusher.constraints.iter().map(|c| c.name()).collect();
+ assert_eq!(names, vec!["keep_errors", "keep_structural_outliers"]);
+ }
+
+ #[test]
+ fn add_constraint_preserves_order() {
+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())
+ .add_constraint(Box::new(MarkerConstraint { name: "first" }))
+ .add_constraint(Box::new(MarkerConstraint { name: "second" }))
+ .add_constraint(Box::new(MarkerConstraint { name: "third" }))
+ .build();
+ let names: Vec<&str> = crusher.constraints.iter().map(|c| c.name()).collect();
+ assert_eq!(names, vec!["first", "second", "third"]);
+ }
+
+ #[test]
+ fn with_default_oss_setup_yields_two_constraints_one_observer() {
+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())
+ .with_default_oss_setup()
+ .build();
+ assert_eq!(crusher.constraints.len(), 2);
+ assert_eq!(crusher.observers.len(), 1);
+ }
+
+ #[test]
+ fn builder_observer_fires_on_crush() {
+ // Wire a counting observer, run a crush, expect exactly one
+ // event. Pins the observer integration end-to-end.
+ let counter = Arc::new(AtomicUsize::new(0));
+ let crusher = SmartCrusherBuilder::new(SmartCrusherConfig::default())
+ .add_observer(Box::new(MarkerObserver {
+ count: counter.clone(),
+ }))
+ .build();
+ let _ = crusher.crush(r#"[1, 2, 3]"#, "", 1.0);
+ assert_eq!(counter.load(Ordering::SeqCst), 1);
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/constraints.rs b/crates/headroom-core/src/transforms/smart_crusher/constraints.rs
new file mode 100644
index 000000000..a246d73d8
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/constraints.rs
@@ -0,0 +1,159 @@
+//! OSS default `Constraint` implementations.
+//!
+//! Two constraints ship in the default OSS composition:
+//!
+//! - [`KeepErrorsConstraint`] — items containing error/failure
+//! keywords (`error`, `exception`, `failed`, `fatal`, etc.). Wraps
+//! the existing [`detect_error_items_for_preservation`] function.
+//! - [`KeepStructuralOutliersConstraint`] — items with rare fields or
+//! rare categorical values. Wraps the existing
+//! [`detect_structural_outliers`] function.
+//!
+//! Both are byte-equivalent to the pre-PR1 hardcoded behavior — the
+//! detection logic is unchanged; the constraints are thin trait
+//! adapters so that custom Enterprise constraints can be stacked
+//! alongside or in place of these defaults.
+//!
+//! # The default factory
+//!
+//! [`default_oss_constraints`] returns the OSS default stack as a
+//! `Vec<Box<dyn Constraint>>`. `SmartCrusher::new(config)` uses this;
+//! `SmartCrusherBuilder::new(config)` does NOT (the builder gives you
+//! exactly what you ask for, no surprises). To get OSS defaults plus
+//! your own constraints, use `with_default_constraints()` on the
+//! builder.
+
+use serde_json::Value;
+
+use super::outliers::{detect_error_items_for_preservation, detect_structural_outliers};
+use super::traits::Constraint;
+
+// ── KeepErrorsConstraint ──────────────────────────────────────────────────
+
+/// OSS default: keep items that contain error keywords.
+///
+/// "Error keyword" is matched case-insensitively against the JSON
+/// serialization of each item against the list in [`super::error_keywords::ERROR_KEYWORDS`]
+/// (`error`, `exception`, `failed`, `fatal`, `critical`, `crash`, `panic`,
+/// `abort`, `timeout`, `denied`, `rejected`).
+#[derive(Debug, Default, Clone, Copy)]
+pub struct KeepErrorsConstraint;
+
+impl Constraint for KeepErrorsConstraint {
+ fn name(&self) -> &str {
+ "keep_errors"
+ }
+
+ fn must_keep(&self, items: &[Value], item_strings: Option<&[String]>) -> Vec<usize> {
+ detect_error_items_for_preservation(items, item_strings)
+ }
+}
+
+// ── KeepStructuralOutliersConstraint ─────────────────────────────────────
+
+/// OSS default: keep items that are *structurally* unusual within the
+/// array. Two flavors of unusual:
+///
+/// - **Rare fields**: items that have a key present in fewer than the
+/// uniqueness threshold of items.
+/// - **Rare values for common fields**: items whose value for a
+/// high-cardinality field appears infrequently (the "rare-status"
+/// path that fires on enums like `level: "ERROR"` among many `INFO`).
+///
+/// Implementation is unchanged from pre-PR1; this is a thin wrapper.
+#[derive(Debug, Default, Clone, Copy)]
+pub struct KeepStructuralOutliersConstraint;
+
+impl Constraint for KeepStructuralOutliersConstraint {
+ fn name(&self) -> &str {
+ "keep_structural_outliers"
+ }
+
+ fn must_keep(&self, items: &[Value], _item_strings: Option<&[String]>) -> Vec<usize> {
+ detect_structural_outliers(items)
+ }
+}
+
+// ── Default OSS stack ─────────────────────────────────────────────────────
+
+/// Returns the default OSS constraint stack used by
+/// `SmartCrusher::new(config)`.
+///
+/// Stack contents (in order):
+/// 1. [`KeepErrorsConstraint`]
+/// 2. [`KeepStructuralOutliersConstraint`]
+///
+/// The order does not affect output (the must-keep set is a union),
+/// but is fixed for determinism in observer-emitted strategy strings
+/// and audit logs.
+pub fn default_oss_constraints() -> Vec<Box<dyn Constraint>> {
+ vec![
+ Box::new(KeepErrorsConstraint),
+ Box::new(KeepStructuralOutliersConstraint),
+ ]
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn keep_errors_constraint_finds_error_items() {
+ // 9 normal items + 1 with "ERROR" keyword.
+ let mut items: Vec<Value> = (0..9).map(|i| json!({"id": i, "status": "ok"})).collect();
+ items.push(json!({"id": 9, "status": "ERROR", "msg": "FATAL: boom"}));
+ let kept = KeepErrorsConstraint.must_keep(&items, None);
+ // Exact index 9 must be in the result.
+ assert!(kept.contains(&9), "error item must be flagged for keep");
+ }
+
+ #[test]
+ fn keep_errors_constraint_uses_item_strings_when_provided() {
+ // Pre-computed strings parity with the on-the-fly path: same
+ // content, same indices returned.
+ let items: Vec<Value> = vec![json!({"a": 1}), json!({"a": "exception"})];
+ let strings: Vec<String> = items
+ .iter()
+ .map(|v| serde_json::to_string(v).unwrap())
+ .collect();
+ let with_cache = KeepErrorsConstraint.must_keep(&items, Some(&strings));
+ let without_cache = KeepErrorsConstraint.must_keep(&items, None);
+ assert_eq!(with_cache, without_cache);
+ assert!(with_cache.contains(&1));
+ }
+
+ #[test]
+ fn keep_structural_outliers_constraint_returns_indices() {
+ // Build an array where one item has a unique extra field —
+ // it should be flagged as a rare-field outlier.
+ let mut items: Vec<Value> = (0..20)
+ .map(|i| json!({"id": i, "kind": "common"}))
+ .collect();
+ items.push(json!({"id": 20, "kind": "common", "rare_extra_field": "x"}));
+ let kept = KeepStructuralOutliersConstraint.must_keep(&items, None);
+ assert!(
+ kept.contains(&20),
+ "item with rare field should be a structural outlier"
+ );
+ }
+
+ #[test]
+ fn default_oss_constraints_returns_two() {
+ let cs = default_oss_constraints();
+ assert_eq!(cs.len(), 2);
+ let names: Vec<&str> = cs.iter().map(|c| c.name()).collect();
+ assert_eq!(names, vec!["keep_errors", "keep_structural_outliers"]);
+ }
+
+ #[test]
+ fn constraints_handle_empty_array() {
+ // No panics, no allocations beyond an empty Vec — the array
+ // path will bypass us when items is empty, but constraints
+ // must still be safe to call.
+ assert!(KeepErrorsConstraint.must_keep(&[], None).is_empty());
+ assert!(KeepStructuralOutliersConstraint
+ .must_keep(&[], None)
+ .is_empty());
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
index ad7b4ce19..2bf8b5536 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -36,14 +36,16 @@
use serde_json::Value;
use super::analyzer::SmartAnalyzer;
+use super::builder::SmartCrusherBuilder;
use super::classifier::{classify_array, ArrayType};
use super::config::SmartCrusherConfig;
use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
use super::planning::SmartCrusherPlanner;
+use super::traits::{Constraint, CrushEvent, Observer};
use super::types::{CompressionPlan, CompressionStrategy, CrushResult};
-use crate::relevance::{HybridScorer, RelevanceScorer};
+use crate::relevance::RelevanceScorer;
use crate::transforms::adaptive_sizer::compute_optimal_k;
-use crate::transforms::anchor_selector::{AnchorConfig, AnchorSelector};
+use crate::transforms::anchor_selector::AnchorSelector;
/// Return type for `crush_array` — mirrors Python's
/// `(crushed_items, strategy_info, ccr_hash, dropped_summary)` tuple.
@@ -59,39 +61,76 @@ pub struct CrushArrayResult {
}
/// Top-level SmartCrusher.
+///
+/// Three pluggable extensions (Stage 3c.2 PR1):
+/// - `scorer` — relevance scoring (`HybridScorer` by default).
+/// - `constraints` — must-keep predicates (`KeepErrorsConstraint` +
+/// `KeepStructuralOutliersConstraint` by default).
+/// - `observers` — decision-stream telemetry (`TracingObserver` by
+/// default).
+///
+/// Compose via [`SmartCrusherBuilder`]; or call `SmartCrusher::new()`
+/// for the OSS default composition.
pub struct SmartCrusher {
pub config: SmartCrusherConfig,
pub anchor_selector: AnchorSelector,
pub scorer: Box<dyn RelevanceScorer + Send + Sync>,
pub analyzer: SmartAnalyzer,
+ pub constraints: Vec<Box<dyn Constraint>>,
+ pub observers: Vec<Box<dyn Observer>>,
}
impl SmartCrusher {
- /// Construct with the default scorer (`HybridScorer`) and the
- /// default `AnchorConfig`. Mirrors Python's
- /// `SmartCrusher(config=...)` no-extra-args path with all
- /// optional subsystems disabled.
+ /// Construct with the OSS default composition: `HybridScorer` +
+ /// `KeepErrorsConstraint` + `KeepStructuralOutliersConstraint` +
+ /// `TracingObserver`. Drop-in for callers that don't need
+ /// custom extensions — pre-PR1 behavior is preserved exactly.
pub fn new(config: SmartCrusherConfig) -> Self {
- let analyzer = SmartAnalyzer::new(config.clone());
- SmartCrusher {
- config,
- anchor_selector: AnchorSelector::new(AnchorConfig::default()),
- scorer: Box::new(HybridScorer::default()),
- analyzer,
- }
+ SmartCrusherBuilder::new(config)
+ .with_default_oss_setup()
+ .build()
}
- /// Construct with a custom scorer.
+ /// Begin a builder chain for custom composition. The Enterprise
+ /// entry point: swap the scorer, add business-rule constraints,
+ /// attach an audit observer.
+ pub fn builder(config: SmartCrusherConfig) -> SmartCrusherBuilder {
+ SmartCrusherBuilder::new(config)
+ }
+
+ /// Construct with a custom scorer (legacy convenience). Equivalent
+ /// to `SmartCrusher::builder(config).with_scorer(scorer).with_default_oss_setup().build()`
+ /// minus the default scorer override; preserved for backward
+ /// compatibility with pre-PR1 callers.
pub fn with_scorer(
config: SmartCrusherConfig,
scorer: Box<dyn RelevanceScorer + Send + Sync>,
) -> Self {
- let analyzer = SmartAnalyzer::new(config.clone());
+ SmartCrusherBuilder::new(config)
+ .with_scorer(scorer)
+ .add_default_oss_constraints()
+ .build()
+ }
+
+ /// Construct directly from owned parts. Used by
+ /// [`SmartCrusherBuilder::build`] — not part of the public stable
+ /// API. Prefer the builder.
+ #[doc(hidden)]
+ pub fn from_parts(
+ config: SmartCrusherConfig,
+ anchor_selector: AnchorSelector,
+ scorer: Box<dyn RelevanceScorer + Send + Sync>,
+ analyzer: SmartAnalyzer,
+ constraints: Vec<Box<dyn Constraint>>,
+ observers: Vec<Box<dyn Observer>>,
+ ) -> Self {
SmartCrusher {
config,
- anchor_selector: AnchorSelector::new(AnchorConfig::default()),
+ anchor_selector,
scorer,
analyzer,
+ constraints,
+ observers,
}
}
@@ -101,6 +140,7 @@ impl SmartCrusher {
&self.anchor_selector,
&*self.scorer,
&self.analyzer,
+ &self.constraints,
)
}
@@ -137,12 +177,33 @@ impl SmartCrusher {
/// - `strategy`: combined strategy info from all crushed arrays
/// (or `"passthrough"`).
pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {
+ let start = std::time::Instant::now();
let (compressed, was_modified, info) = self.smart_crush_content(content, query, bias);
let strategy = if info.is_empty() {
"passthrough".to_string()
} else {
info
};
+
+ // Fire one event per top-level crush. Cheap when no observers
+ // are configured (`for o in &[]` is a single null-pointer
+ // check); cheap when only `TracingObserver` is configured if
+ // the subscriber filters `debug` out (the default in
+ // production). Custom observers — audit logs, Loop training
+ // stream, metrics — pay whatever they pay.
+ if !self.observers.is_empty() {
+ let event = CrushEvent {
+ strategy: strategy.clone(),
+ input_bytes: content.len(),
+ output_bytes: compressed.len(),
+ elapsed_ns: start.elapsed().as_nanos() as u64,
+ was_modified,
+ };
+ for observer in &self.observers {
+ observer.on_event(&event);
+ }
+ }
+
CrushResult {
compressed,
original: content.to_string(),
diff --git a/crates/headroom-core/src/transforms/smart_crusher/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
index 760d850b9..333af8a61 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/mod.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/mod.rs
@@ -34,29 +34,38 @@
mod analyzer;
mod anchors;
+mod builder;
mod classifier;
mod config;
+mod constraints;
mod crusher;
mod crushers;
mod error_keywords;
mod field_detect;
mod hashing;
+mod observer;
mod orchestration;
mod outliers;
mod planning;
mod statistics;
mod stats_math;
+mod traits;
mod types;
pub use analyzer::SmartAnalyzer;
pub use anchors::{extract_query_anchors, item_matches_anchors};
+pub use builder::SmartCrusherBuilder;
pub use classifier::{classify_array, ArrayType};
pub use config::SmartCrusherConfig;
+pub use constraints::{
+ default_oss_constraints, KeepErrorsConstraint, KeepStructuralOutliersConstraint,
+};
pub use crusher::{CrushArrayResult, SmartCrusher};
pub use crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
pub use error_keywords::ERROR_KEYWORDS;
pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};
pub use hashing::hash_field_name;
+pub use observer::TracingObserver;
pub use orchestration::{deduplicate_indices_by_content, fill_remaining_slots, prioritize_indices};
pub use outliers::{
detect_error_items_for_preservation, detect_rare_status_values, detect_structural_outliers,
@@ -64,6 +73,7 @@ pub use outliers::{
pub use planning::{item_has_preserve_field_match, map_to_anchor_pattern, SmartCrusherPlanner};
pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
pub use stats_math::{format_g, mean, median, sample_stdev, sample_variance};
+pub use traits::{Constraint, CrushEvent, Observer, Scorer};
pub use types::{
ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,
FieldStats,
diff --git a/crates/headroom-core/src/transforms/smart_crusher/observer.rs b/crates/headroom-core/src/transforms/smart_crusher/observer.rs
new file mode 100644
index 000000000..68fc63109
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/observer.rs
@@ -0,0 +1,66 @@
+//! OSS default `Observer` implementation.
+//!
+//! Ships [`TracingObserver`] which writes each `CrushEvent` to the
+//! `tracing` crate at `debug` level. Subscribers that filter `debug`
+//! out (typical production config) pay nothing — `tracing` stops
+//! evaluation at the level check before constructing the event
+//! fields. Subscribers that retain `debug` get a structured per-crush
+//! event suitable for log analytics.
+//!
+//! Enterprise consumers ship richer observers — `AuditObserver` for
+//! SOC2/HIPAA decision logs, `MetricsObserver` for Datadog/Atlas
+//! gauges, `LoopTrainingObserver` to stream events to Headroom Loop —
+//! all on the same trait.
+
+use super::traits::{CrushEvent, Observer};
+
+/// Writes each `CrushEvent` to the `tracing` crate at `debug` level.
+/// Zero-cost when the subscriber filters `debug` out.
+#[derive(Debug, Default, Clone, Copy)]
+pub struct TracingObserver;
+
+impl Observer for TracingObserver {
+ fn name(&self) -> &str {
+ "tracing"
+ }
+
+ fn on_event(&self, event: &CrushEvent) {
+ // `tracing::debug!` is a macro; the level check happens before
+ // the fields are evaluated, so this is essentially free at
+ // higher log levels.
+ tracing::debug!(
+ target: "headroom::smart_crusher",
+ strategy = %event.strategy,
+ input_bytes = event.input_bytes,
+ output_bytes = event.output_bytes,
+ elapsed_ns = event.elapsed_ns,
+ was_modified = event.was_modified,
+ "smart_crusher.crush emitted",
+ );
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn tracing_observer_does_not_panic_on_event() {
+ // We can't easily assert what `tracing` did without a
+ // subscriber set up — but the call must not panic, and the
+ // event fields must be readable.
+ let event = CrushEvent {
+ strategy: "passthrough".to_string(),
+ input_bytes: 100,
+ output_bytes: 100,
+ elapsed_ns: 0,
+ was_modified: false,
+ };
+ TracingObserver.on_event(&event);
+ }
+
+ #[test]
+ fn tracing_observer_name_is_stable() {
+ assert_eq!(TracingObserver.name(), "tracing");
+ }
+}
diff --git a/crates/headroom-core/src/transforms/smart_crusher/planning.rs b/crates/headroom-core/src/transforms/smart_crusher/planning.rs
index 7816d6e21..888e7daf7 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/planning.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/planning.rs
@@ -35,8 +35,12 @@ use super::config::SmartCrusherConfig;
use super::field_detect::detect_score_field_statistically;
use super::hashing::hash_field_name;
use super::orchestration::prioritize_indices;
-use super::outliers::{detect_error_items_for_preservation, detect_structural_outliers};
+use super::traits::Constraint;
use super::types::{ArrayAnalysis, CompressionPlan, CompressionStrategy, FieldStats};
+// Note: `detect_error_items_for_preservation` and `detect_structural_outliers`
+// are still imported transitively by `constraints.rs` (via `KeepErrorsConstraint`
+// and `KeepStructuralOutliersConstraint`). Planning no longer calls them
+// directly; it iterates `self.constraints` via `apply_constraints`.
use crate::relevance::RelevanceScorer;
use crate::transforms::anchor_selector::{AnchorSelector, DataPattern};
@@ -47,6 +51,12 @@ pub struct SmartCrusherPlanner<'a> {
pub anchor_selector: &'a AnchorSelector,
pub scorer: &'a (dyn RelevanceScorer + Send + Sync),
pub analyzer: &'a SmartAnalyzer,
+ /// User-configured must-keep predicates. The plan methods union
+ /// the output of every constraint into the kept set; OSS default
+ /// composition includes `KeepErrorsConstraint` and
+ /// `KeepStructuralOutliersConstraint`, reproducing the pre-PR1
+ /// hardcoded behavior byte-for-byte.
+ pub constraints: &'a [Box<dyn Constraint>],
}
impl<'a> SmartCrusherPlanner<'a> {
@@ -55,12 +65,31 @@ impl<'a> SmartCrusherPlanner<'a> {
anchor_selector: &'a AnchorSelector,
scorer: &'a (dyn RelevanceScorer + Send + Sync),
analyzer: &'a SmartAnalyzer,
+ constraints: &'a [Box<dyn Constraint>],
) -> Self {
SmartCrusherPlanner {
config,
anchor_selector,
scorer,
analyzer,
+ constraints,
+ }
+ }
+
+ /// Apply every configured `Constraint::must_keep` and union the
+ /// results into `keep`. Replaces the hardcoded
+ /// `detect_error_items_for_preservation` +
+ /// `detect_structural_outliers` calls that lived in each plan
+ /// method. With the OSS default constraint stack the output is
+ /// byte-identical to pre-PR1 behavior.
+ fn apply_constraints(
+ &self,
+ items: &[Value],
+ item_strings: Option<&[String]>,
+ keep: &mut BTreeSet<usize>,
+ ) {
+ for c in self.constraints {
+ keep.extend(c.must_keep(items, item_strings));
}
}
@@ -159,9 +188,8 @@ impl<'a> SmartCrusherPlanner<'a> {
query_or_none(query_context),
));
- // 2. Structural outliers + error keywords.
- keep.extend(detect_structural_outliers(items));
- keep.extend(detect_error_items_for_preservation(items, item_strings));
+ // 2. Structural outliers + error keywords (configured via Constraint trait).
+ self.apply_constraints(items, item_strings, &mut keep);
// 3. Numeric anomalies (>variance_threshold σ from per-field mean).
for (name, stats) in &analysis.field_stats {
@@ -260,9 +288,8 @@ impl<'a> SmartCrusherPlanner<'a> {
keep.insert(*idx);
}
- // 2. Structural outliers + error keywords.
- keep.extend(detect_structural_outliers(items));
- keep.extend(detect_error_items_for_preservation(items, item_strings));
+ // 2. Structural outliers + error keywords (configured via Constraint trait).
+ self.apply_constraints(items, item_strings, &mut keep);
// 3. Query-anchor matches (additive — preserved regardless of top-N).
if !query_context.is_empty() {
@@ -335,9 +362,8 @@ impl<'a> SmartCrusherPlanner<'a> {
query_or_none(query_context),
));
- // 2. Structural outliers + error keywords.
- keep.extend(detect_structural_outliers(items));
- keep.extend(detect_error_items_for_preservation(items, item_strings));
+ // 2. Structural outliers + error keywords (configured via Constraint trait).
+ self.apply_constraints(items, item_strings, &mut keep);
// 3. Cluster by message-like field (highest unique_ratio > 0.3).
let mut message_field: Option<&str> = None;
@@ -424,9 +450,8 @@ impl<'a> SmartCrusherPlanner<'a> {
}
}
- // 3. Structural outliers + error keywords.
- keep.extend(detect_structural_outliers(items));
- keep.extend(detect_error_items_for_preservation(items, item_strings));
+ // 3. Structural outliers + error keywords (configured via Constraint trait).
+ self.apply_constraints(items, item_strings, &mut keep);
// 4/5. Query signals.
self.apply_query_signals(items, query_context, item_strings, &mut keep, false);
@@ -613,6 +638,7 @@ mod tests {
use super::*;
use crate::relevance::HybridScorer;
use crate::transforms::anchor_selector::AnchorConfig;
+ use crate::transforms::smart_crusher::constraints::default_oss_constraints;
use serde_json::json;
fn fixture<'a>(
@@ -620,8 +646,9 @@ mod tests {
anchor_selector: &'a AnchorSelector,
scorer: &'a HybridScorer,
analyzer: &'a SmartAnalyzer,
+ constraints: &'a [Box<dyn Constraint>],
) -> SmartCrusherPlanner<'a> {
- SmartCrusherPlanner::new(config, anchor_selector, scorer, analyzer)
+ SmartCrusherPlanner::new(config, anchor_selector, scorer, analyzer, constraints)
}
fn make_planner_deps() -> (
@@ -629,12 +656,14 @@ mod tests {
AnchorSelector,
HybridScorer,
SmartAnalyzer,
+ Vec<Box<dyn Constraint>>,
) {
let cfg = SmartCrusherConfig::default();
let asel = AnchorSelector::new(AnchorConfig::default());
let scorer = HybridScorer::default();
let analyzer = SmartAnalyzer::new(cfg.clone());
- (cfg, asel, scorer, analyzer)
+ let constraints = default_oss_constraints();
+ (cfg, asel, scorer, analyzer, constraints)
}
// ---------- map_to_anchor_pattern ----------
@@ -703,8 +732,8 @@ mod tests {
#[test]
fn create_plan_skip_returns_all_indices() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
let analysis = ArrayAnalysis {
item_count: 5,
field_stats: BTreeMap::new(),
@@ -721,8 +750,8 @@ mod tests {
#[test]
fn create_plan_routes_smart_sample_to_smart_sample() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "v": i})).collect();
let analysis = analyzer.analyze_array(&items);
let plan = p.create_plan(&analysis, &items, "", None, Some(15), None);
@@ -736,8 +765,8 @@ mod tests {
#[test]
fn smart_sample_keeps_error_items() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
let mut items: Vec<Value> = (0..30)
.map(|i| json!({"id": i, "msg": format!("ok {}", i)}))
.collect();
@@ -756,8 +785,8 @@ mod tests {
#[test]
fn smart_sample_query_anchor_pinned() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
let items: Vec<Value> = (0..30)
.map(|i| {
json!({
@@ -786,8 +815,8 @@ mod tests {
#[test]
fn top_n_falls_back_when_no_score_field() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
// No bounded score field — top_n falls through to smart_sample.
let items: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();
let analysis = analyzer.analyze_array(&items);
@@ -802,8 +831,8 @@ mod tests {
#[test]
fn top_n_keeps_highest_scored_items() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
// 20 items with score 0.0..0.95 in 0.05 increments. Top-K
// should be the highest scores.
let items: Vec<Value> = (0..20)
@@ -826,8 +855,8 @@ mod tests {
#[test]
fn cluster_sample_assigns_cluster_field() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
// Logs-shaped data: high-cardinality message + low-cardinality level.
let items: Vec<Value> = (0..30)
.map(|i| {
@@ -852,8 +881,8 @@ mod tests {
#[test]
fn time_series_keeps_window_around_change_points() {
- let (cfg, asel, scorer, analyzer) = make_planner_deps();
- let p = fixture(&cfg, &asel, &scorer, &analyzer);
+ let (cfg, asel, scorer, analyzer, cs) = make_planner_deps();
+ let p = fixture(&cfg, &asel, &scorer, &analyzer, &cs);
// 30 items with a step at index 15; analyzer should detect
// change points around there.
let items: Vec<Value> = (0..60)
diff --git a/crates/headroom-core/src/transforms/smart_crusher/traits.rs b/crates/headroom-core/src/transforms/smart_crusher/traits.rs
new file mode 100644
index 000000000..620cd7b14
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/traits.rs
@@ -0,0 +1,203 @@
+//! Public extension surface for `SmartCrusher` (Stage 3c.2 PR 1).
+//!
+//! Three traits — `Scorer`, `Constraint`, `Observer` — capture every
+//! decision a `SmartCrusher` makes that downstream consumers might
+//! want to override:
+//!
+//! - **`Scorer`** (already public via `crate::relevance::RelevanceScorer`):
+//! how relevant is item *i* to query *q*? OSS default `HybridScorer`
+//! (BM25 + fastembed). Enterprise can plug in a per-tenant Loop-trained
+//! scorer.
+//! - **`Constraint`** (this module): which indices must be kept
+//! regardless of score? OSS defaults preserve errors and structural
+//! outliers. Enterprise can add `BusinessRuleConstraint`,
+//! `RegulatoryConstraint`, etc.
+//! - **`Observer`** (this module): emit a structured event after each
+//! `crush()` so telemetry, audit logs, and continuous-eval pipelines
+//! can hook in. OSS default writes to the `tracing` crate.
+//!
+//! # Why three, not eight
+//!
+//! The 5-stage pipeline (classify → compact → score → allocate →
+//! format) has more stage boundaries, but only three of them carry
+//! *differentiated value* to Enterprise customers — Loop scorer,
+//! business rules, audit telemetry. The other stages stay as concrete
+//! Rust types; if an Enterprise customer ever needs to plug into a
+//! different stage we can promote it to a trait at that point. We're
+//! not designing for hypothetical futures — we're naming the seams
+//! that real customers will pay for today.
+//!
+//! # Composition
+//!
+//! Use [`SmartCrusherBuilder`](super::builder::SmartCrusherBuilder) to
+//! compose a custom `SmartCrusher`. The default OSS composition is
+//! reachable via `SmartCrusher::new(config)` and stays byte-equivalent
+//! to pre-PR1 behavior — all 17 parity fixtures pass.
+
+use serde_json::Value;
+
+// ── Constraint ────────────────────────────────────────────────────────────
+
+/// A hard preservation constraint: indices the allocator must keep
+/// regardless of saliency score or token budget.
+///
+/// Constraints stack — the must-keep set is the union of every
+/// constraint's `must_keep` output. OSS ships [`KeepErrorsConstraint`]
+/// and [`KeepStructuralOutliersConstraint`] (wrappers around the
+/// existing detection functions); Enterprise crates can add
+/// `BusinessRuleConstraint("amount > 10000")`,
+/// `RegulatoryConstraint::HIPAA`, and so on.
+///
+/// # Contract
+///
+/// - **`must_keep` returns indices into `items`.** Out-of-bounds
+/// indices are silently dropped by the allocator; constraints
+/// should not return them.
+/// - **Idempotent in the small.** Calling `must_keep` twice with the
+/// same `items` returns the same set; constraints do not own
+/// mutable state that drifts between calls within one
+/// `SmartCrusher::crush` invocation. (Caching across crushes is
+/// fine — see `LoopScorer` style stateful enrichers.)
+/// - **Cheap is free.** Constraints run on every dict-array crush.
+/// A constraint that does I/O or heavy regex per-item can dominate
+/// the crusher's cost. Aim for O(n) on items with small constants.
+///
+/// # Why `item_strings: Option<&[String]>`?
+///
+/// Many constraints search the JSON serialization for keywords (the
+/// existing `detect_error_items_for_preservation` does this). The
+/// caller may already have computed `item_strings` for adaptive
+/// sizing; passing them through avoids redundant `serde_json::to_string`
+/// calls. Constraints that don't need the strings simply ignore the
+/// argument.
+pub trait Constraint: Send + Sync {
+ /// Stable identifier — appears in `CrushEvent` strategy strings,
+ /// audit logs, and config-validation diagnostics. Use snake_case
+ /// (`"keep_errors"`, `"business_rule"`).
+ fn name(&self) -> &str;
+
+ /// Indices of items the allocator MUST keep.
+ fn must_keep(&self, items: &[Value], item_strings: Option<&[String]>) -> Vec<usize>;
+}
+
+// ── Observer ──────────────────────────────────────────────────────────────
+
+/// Telemetry event emitted at the end of each `SmartCrusher::crush`
+/// call. Observers (multiple, stacked) consume these for `tracing`,
+/// audit logs, Loop training data, real-time dashboards.
+#[derive(Debug, Clone)]
+pub struct CrushEvent {
+ /// Strategy debug string returned by the crusher
+ /// (e.g. `"smart_sample(30->15)"`, `"passthrough"`,
+ /// `"top_n(50->15)"`).
+ pub strategy: String,
+ /// Length in bytes of the input content (whatever was passed to
+ /// `crush()`).
+ pub input_bytes: usize,
+ /// Length in bytes of the compressed output.
+ pub output_bytes: usize,
+ /// Wall-clock duration of the `crush()` call.
+ pub elapsed_ns: u64,
+ /// Whether the output differs from the input.
+ pub was_modified: bool,
+}
+
+/// Decision-stream hook. Called after each top-level `SmartCrusher::crush`
+/// returns; observers run synchronously on the crusher's thread.
+///
+/// # Contract
+///
+/// - **Cheap is free.** Like constraints, observers run on every
+/// crush. `tracing::debug!` is essentially free when the subscriber
+/// filters the level out; remote network calls are not.
+/// - **Don't panic.** A panicking observer aborts the calling thread.
+/// If your observer does I/O, catch and log errors yourself.
+/// - **Order matters.** Observers fire in the order they were added
+/// to the builder. Most callers should not depend on the order
+/// (telemetry is naturally idempotent), but if you have an
+/// audit-then-publish chain, add them in that order.
+pub trait Observer: Send + Sync {
+ /// Stable identifier — useful for filtering in the rare cases
+ /// where an observer wants to disable itself when another is
+ /// already configured. Default: the type name.
+ fn name(&self) -> &str {
+ std::any::type_name::<Self>()
+ }
+
+ /// Called once per `SmartCrusher::crush` invocation, after the
+ /// result is computed and before it is returned to the caller.
+ fn on_event(&self, event: &CrushEvent);
+}
+
+// ── Re-exports for convenience ────────────────────────────────────────────
+
+// Scorer is in the relevance crate, not here; re-export so callers
+// can get all three traits from one path.
+pub use crate::relevance::RelevanceScorer as Scorer;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+
+ /// A tiny constraint that always keeps index 0 (if any items).
+ /// Pins the trait shape and the additive behavior of constraint
+ /// stacking.
+ struct AlwaysKeepFirst;
+ impl Constraint for AlwaysKeepFirst {
+ fn name(&self) -> &str {
+ "always_keep_first"
+ }
+ fn must_keep(&self, items: &[Value], _: Option<&[String]>) -> Vec<usize> {
+ if items.is_empty() {
+ Vec::new()
+ } else {
+ vec![0]
+ }
+ }
+ }
+
+ #[test]
+ fn constraint_returns_indices_in_bounds() {
+ let items = vec![json!({"a": 1}), json!({"a": 2})];
+ let c = AlwaysKeepFirst;
+ let kept = c.must_keep(&items, None);
+ assert_eq!(kept, vec![0]);
+ assert_eq!(c.name(), "always_keep_first");
+ }
+
+ #[test]
+ fn constraint_handles_empty_input() {
+ let kept = AlwaysKeepFirst.must_keep(&[], None);
+ assert!(kept.is_empty());
+ }
+
+ /// Counts events to verify the observer trait fires correctly
+ /// when wired into a SmartCrusher (integration test in
+ /// `crusher.rs::tests`).
+ #[derive(Default)]
+ struct CountingObserver {
+ count: Arc<AtomicUsize>,
+ }
+ impl Observer for CountingObserver {
+ fn on_event(&self, _: &CrushEvent) {
+ self.count.fetch_add(1, Ordering::SeqCst);
+ }
+ }
+
+ #[test]
+ fn observer_event_carries_strategy_and_sizes() {
+ let observer = CountingObserver::default();
+ let event = CrushEvent {
+ strategy: "smart_sample(30->15)".to_string(),
+ input_bytes: 1000,
+ output_bytes: 500,
+ elapsed_ns: 12_345,
+ was_modified: true,
+ };
+ observer.on_event(&event);
+ assert_eq!(observer.count.load(Ordering::SeqCst), 1);
+ }
+}