Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(rust): port ContentDetector to Rust + parity harness + PyO3 bridge
chopratejas625290cfbce6
diff --git a/crates/headroom-core/src/transforms/content_detector.rs b/crates/headroom-core/src/transforms/content_detector.rsnew file mode 100644index 000000000..57b46cd14--- /dev/null+++ b/crates/headroom-core/src/transforms/content_detector.rs@@ -0,0 +1,769 @@+//! Content type detection for multi-format compression.+//!+//! Direct port of `headroom/transforms/content_detector.py`. This module+//! detects the type of tool output content so the upstream+//! `ContentRouter` can dispatch it to the right compressor:+//!+//! - **JsonArray**: Structured JSON data → `SmartCrusher`+//! - **SourceCode**: Python, JavaScript, Go, Rust, etc. → `CodeAwareCompressor`+//! - **SearchResults**: grep / ripgrep output (`file:line:content`)+//! - **BuildOutput**: Compiler / test / lint logs+//! - **GitDiff**: Unified diff format → `DiffCompressor`+//! - **Html**: Web pages (needs extraction, not compression)+//! - **PlainText**: Generic fallback+//!+//! Detection is **regex-based** — no ML, no model loading, no I/O.+//! Magika integration lives one level up in `ContentRouter`, not here.+//!+//! # Parity with Python+//!+//! Regex patterns, dispatch order, confidence formulas, and line-count+//! caps are byte-equal with the Python source. Recorded fixtures in+//! `tests/parity/fixtures/content_detector/` lock the output across+//! the bridge.++use std::sync::LazyLock;++use regex::Regex;+use serde_json::{json, Map, Value};++/// Content types recognized by the detector. String tags match Python's+/// `ContentType` enum values 1:1.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ContentType {+ JsonArray,+ SourceCode,+ SearchResults,+ BuildOutput,+ GitDiff,+ Html,+ PlainText,+}++impl ContentType {+ /// Stable string tag — matches Python's `ContentType.<NAME>.value`.+ pub fn as_str(&self) -> &'static str {+ match self {+ ContentType::JsonArray => "json_array",+ ContentType::SourceCode => "source_code",+ ContentType::SearchResults => "search",+ ContentType::BuildOutput => "build",+ ContentType::GitDiff => "diff",+ ContentType::Html => "html",+ ContentType::PlainText => "text",+ }+ }+}++/// Result of `detect_content_type`. `metadata` is per-type free-form key/+/// value data — same shape as Python's `dict[str, Any]`. We use+/// `serde_json::Map` so PyO3 can convert it to a Python dict on the+/// boundary without losing type fidelity.+#[derive(Debug, Clone)]+pub struct DetectionResult {+ pub content_type: ContentType,+ pub confidence: f64,+ pub metadata: Map<String, Value>,+}++impl DetectionResult {+ fn new(content_type: ContentType, confidence: f64, metadata: Map<String, Value>) -> Self {+ Self {+ content_type,+ confidence,+ metadata,+ }+ }++ fn plain_text(confidence: f64) -> Self {+ Self::new(ContentType::PlainText, confidence, Map::new())+ }+}++// ─── Regex patterns (compiled once, shared) ───────────────────────────++/// `file:line:` (grep -n style) — first column on a non-blank line.+static SEARCH_RESULT_PATTERN: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"^[^\s:]+:\d+:").unwrap());++/// Diff-header detection. Recognizes:+/// - `git diff` (`diff --git`, `--- a/`)+/// - merge-commit headers (`diff --combined`, `diff --cc`)+/// - regular hunk headers (`@@ -A,B +C,D @@`)+/// - combined-diff hunk headers (`@@@ ... @@@`)+///+/// Mirrors Python's bug-fix from 2026-04-25 that widened the grammar+/// to handle merge-commit diffs from `git log -p`.+static DIFF_HEADER_PATTERN: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(+ r"^(diff --git|diff --combined |diff --cc |--- a/|@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@|@@@+\s+-\d+(?:,\d+)?\s+(?:-\d+(?:,\d+)?\s+)+\+\d+(?:,\d+)?\s+@@@+)",+ )+ .unwrap()+});++/// Lines starting with `+` or `-` followed by a non-`+`/`-` char (i.e.+/// real change lines, not header lines like `+++ b/file`).+static DIFF_CHANGE_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[+-][^+-]").unwrap());++// ─── Code patterns by language ─────────────────────────────────────────++struct CodePatterns {+ name: &'static str,+ patterns: Vec<Regex>,+}++static CODE_PATTERNS: LazyLock<Vec<CodePatterns>> = LazyLock::new(|| {+ vec![+ CodePatterns {+ name: "python",+ patterns: vec![+ Regex::new(r"^\s*(def|class|import|from|async def)\s+\w+").unwrap(),+ Regex::new(r"^\s*@\w+").unwrap(),+ Regex::new(r#"^\s*""""#).unwrap(),+ Regex::new(r"^\s*if __name__\s*==").unwrap(),+ ],+ },+ CodePatterns {+ name: "javascript",+ patterns: vec![+ Regex::new(r"^\s*(function|const|let|var|class|import|export)\s+").unwrap(),+ Regex::new(r"^\s*(async\s+function|=>\s*\{)").unwrap(),+ Regex::new(r"^\s*module\.exports").unwrap(),+ ],+ },+ CodePatterns {+ name: "typescript",+ patterns: vec![+ Regex::new(r"^\s*(interface|type|enum|namespace)\s+\w+").unwrap(),+ // Python uses `pattern.match(line)` which is start-anchored,+ // so this pattern only ever fires on lines literally starting+ // with `:`. We anchor with `^` to keep parity (the `regex`+ // crate's `is_match` is unanchored by default).+ Regex::new(r"^:\s*(string|number|boolean|any|void)\b").unwrap(),+ ],+ },+ CodePatterns {+ name: "go",+ patterns: vec![+ Regex::new(r"^\s*(func|type|package|import)\s+").unwrap(),+ Regex::new(r"^\s*func\s+\([^)]+\)\s+\w+").unwrap(),+ ],+ },+ CodePatterns {+ name: "rust",+ patterns: vec![+ Regex::new(r"^\s*(fn|struct|enum|impl|mod|use|pub)\s+").unwrap(),+ Regex::new(r"^\s*#\[").unwrap(),+ ],+ },+ CodePatterns {+ name: "java",+ patterns: vec![+ Regex::new(r"^\s*(public|private|protected)\s+(class|interface|enum)").unwrap(),+ Regex::new(r"^\s*@\w+").unwrap(),+ Regex::new(r"^\s*package\s+[\w.]+;").unwrap(),+ ],+ },+ ]+});++// ─── Log / build output patterns ───────────────────────────────────────+//+// Order matters: indices 0–1 (`ERROR` and `WARN` family) are treated as+// "error" matches by `try_detect_log`, contributing extra to confidence.+// Same ordering as Python.++static LOG_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {+ vec![+ Regex::new(r"(?i)\b(ERROR|FAIL|FAILED|FATAL|CRITICAL)\b").unwrap(),+ Regex::new(r"(?i)\b(WARN|WARNING)\b").unwrap(),+ Regex::new(r"(?i)\b(INFO|DEBUG|TRACE)\b").unwrap(),+ Regex::new(r"^\s*\d{4}-\d{2}-\d{2}").unwrap(),+ Regex::new(r"^\s*\[\d{2}:\d{2}:\d{2}\]").unwrap(),+ Regex::new(r"^={3,}|^-{3,}").unwrap(),+ Regex::new(r"^\s*PASSED|^\s*FAILED|^\s*SKIPPED").unwrap(),+ Regex::new(r"^npm ERR!|^yarn error|^cargo error").unwrap(),+ Regex::new(r"Traceback \(most recent call last\)").unwrap(),+ Regex::new(r"^\s*at\s+[\w.$]+\(").unwrap(),+ ]+});++// ─── HTML patterns ─────────────────────────────────────────────────────++static HTML_DOCTYPE_PATTERN: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"(?i)^\s*<!doctype\s+html").unwrap());+static HTML_TAG_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<html[\s>]").unwrap());+static HTML_HEAD_PATTERN: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"(?i)<head[\s>]").unwrap());+static HTML_BODY_PATTERN: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"(?i)<body[\s>]").unwrap());+static HTML_STRUCTURAL_TAGS: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(+ r"(?i)<(div|span|script|style|link|meta|nav|header|footer|aside|article|section|main)[\s>]",+ )+ .unwrap()+});++// ─── Public entry point ────────────────────────────────────────────────++/// Detect the type of `content` for routing. Mirrors Python's+/// `detect_content_type`.+///+/// Dispatch order (matches Python verbatim):+/// 1. Empty / whitespace-only → `PlainText` confidence 0.0+/// 2. JSON array (highest priority for `SmartCrusher`)+/// 3. Git diff (≥ 0.7 confidence required)+/// 4. HTML (≥ 0.7 confidence required)+/// 5. Search results (≥ 0.6 confidence required)+/// 6. Build / log output (≥ 0.5 confidence required)+/// 7. Source code (≥ 0.5 confidence required)+/// 8. Fallback to `PlainText` confidence 0.5+pub fn detect_content_type(content: &str) -> DetectionResult {+ if content.is_empty() || content.trim().is_empty() {+ return DetectionResult::plain_text(0.0);+ }++ if let Some(r) = try_detect_json(content) {+ return r;+ }+ if let Some(r) = try_detect_diff(content) {+ if r.confidence >= 0.7 {+ return r;+ }+ }+ if let Some(r) = try_detect_html(content) {+ if r.confidence >= 0.7 {+ return r;+ }+ }+ if let Some(r) = try_detect_search(content) {+ if r.confidence >= 0.6 {+ return r;+ }+ }+ if let Some(r) = try_detect_log(content) {+ if r.confidence >= 0.5 {+ return r;+ }+ }+ if let Some(r) = try_detect_code(content) {+ if r.confidence >= 0.5 {+ return r;+ }+ }+ DetectionResult::plain_text(0.5)+}++/// Quick check: is `content` a JSON array of dictionaries (the format+/// `SmartCrusher` natively handles)? Convenience wrapper around+/// `detect_content_type`.+pub fn is_json_array_of_dicts(content: &str) -> bool {+ let result = detect_content_type(content);+ if result.content_type != ContentType::JsonArray {+ return false;+ }+ result+ .metadata+ .get("is_dict_array")+ .and_then(|v| v.as_bool())+ .unwrap_or(false)+}++// ─── Per-type detection helpers ────────────────────────────────────────++fn try_detect_json(content: &str) -> Option<DetectionResult> {+ let trimmed = content.trim();+ if !trimmed.starts_with('[') {+ return None;+ }+ let parsed: Value = serde_json::from_str(trimmed).ok()?;+ let arr = parsed.as_array()?;+ let item_count = arr.len();+ let is_dict_array = !arr.is_empty() && arr.iter().all(|v| v.is_object());+ let confidence = if is_dict_array { 1.0 } else { 0.8 };+ Some(DetectionResult::new(+ ContentType::JsonArray,+ confidence,+ json!({+ "item_count": item_count,+ "is_dict_array": is_dict_array,+ })+ .as_object()+ .cloned()+ .unwrap(),+ ))+}++fn try_detect_diff(content: &str) -> Option<DetectionResult> {+ // Window: 500 lines (extended from 50 in Python's 2026-04-25 fix).+ let mut header_matches: u32 = 0;+ let mut change_matches: u32 = 0;+ for line in content.split('\n').take(500) {+ if DIFF_HEADER_PATTERN.is_match(line) {+ header_matches += 1;+ }+ if DIFF_CHANGE_PATTERN.is_match(line) {+ change_matches += 1;+ }+ }+ if header_matches == 0 {+ return None;+ }+ // Same formula as Python: 0.5 + 0.2 * headers + 0.05 * changes, capped at 1.0+ let confidence =+ (0.5 + (header_matches as f64) * 0.2 + (change_matches as f64) * 0.05).min(1.0);+ Some(DetectionResult::new(+ ContentType::GitDiff,+ confidence,+ json!({+ "header_matches": header_matches,+ "change_lines": change_matches,+ })+ .as_object()+ .cloned()+ .unwrap(),+ ))+}++fn try_detect_html(content: &str) -> Option<DetectionResult> {+ // Sample first 3000 chars (byte-indexed; matches Python's str slice+ // for ASCII inputs which is the common HTML case).+ let sample: &str = if content.len() > 3000 {+ // Find the last char-boundary <= 3000 so we don't slice mid-codepoint.+ let mut cutoff = 3000;+ while !content.is_char_boundary(cutoff) {+ cutoff -= 1;+ }+ &content[..cutoff]+ } else {+ content+ };++ let has_doctype = HTML_DOCTYPE_PATTERN.is_match(sample);+ let has_html_tag = HTML_TAG_PATTERN.is_match(sample);+ let has_head = HTML_HEAD_PATTERN.is_match(sample);+ let has_body = HTML_BODY_PATTERN.is_match(sample);+ let structural_matches = HTML_STRUCTURAL_TAGS.find_iter(sample).count() as u32;++ if !has_doctype && !has_html_tag && structural_matches < 3 {+ return None;+ }++ let mut confidence = 0.0_f64;+ if has_doctype {+ confidence += 0.5;+ }+ if has_html_tag {+ confidence += 0.3;+ }+ if has_head {+ confidence += 0.1;+ }+ if has_body {+ confidence += 0.1;+ }+ confidence += (structural_matches as f64 * 0.03).min(0.3);+ confidence = confidence.min(1.0);++ if confidence < 0.5 {+ return None;+ }+ Some(DetectionResult::new(+ ContentType::Html,+ confidence,+ json!({+ "has_doctype": has_doctype,+ "has_html_tag": has_html_tag,+ "structural_tags": structural_matches,+ })+ .as_object()+ .cloned()+ .unwrap(),+ ))+}++fn try_detect_search(content: &str) -> Option<DetectionResult> {+ let lines: Vec<&str> = content.split('\n').take(100).collect();+ if lines.is_empty() {+ return None;+ }+ let mut matching_lines: u32 = 0;+ for line in &lines {+ if !line.trim().is_empty() && SEARCH_RESULT_PATTERN.is_match(line) {+ matching_lines += 1;+ }+ }+ if matching_lines == 0 {+ return None;+ }+ let non_empty_lines = lines.iter().filter(|l| !l.trim().is_empty()).count() as u32;+ if non_empty_lines == 0 {+ return None;+ }+ let ratio = matching_lines as f64 / non_empty_lines as f64;+ if ratio < 0.3 {+ return None;+ }+ let confidence = (0.4 + ratio * 0.6).min(1.0);+ Some(DetectionResult::new(+ ContentType::SearchResults,+ confidence,+ json!({+ "matching_lines": matching_lines,+ "total_lines": non_empty_lines,+ })+ .as_object()+ .cloned()+ .unwrap(),+ ))+}++fn try_detect_log(content: &str) -> Option<DetectionResult> {+ let lines: Vec<&str> = content.split('\n').take(200).collect();+ if lines.is_empty() {+ return None;+ }+ let mut pattern_matches: u32 = 0;+ let mut error_matches: u32 = 0;+ for line in &lines {+ for (i, pattern) in LOG_PATTERNS.iter().enumerate() {+ if pattern.is_match(line) {+ pattern_matches += 1;+ if i < 2 {+ error_matches += 1;+ }+ break; // one pattern per line is enough+ }+ }+ }+ if pattern_matches == 0 {+ return None;+ }+ let non_empty_lines = lines.iter().filter(|l| !l.trim().is_empty()).count() as u32;+ if non_empty_lines == 0 {+ return None;+ }+ let ratio = pattern_matches as f64 / non_empty_lines as f64;+ if ratio < 0.1 {+ return None;+ }+ let confidence = (0.3 + ratio * 0.5 + (error_matches as f64) * 0.05).min(1.0);+ Some(DetectionResult::new(+ ContentType::BuildOutput,+ confidence,+ json!({+ "pattern_matches": pattern_matches,+ "error_matches": error_matches,+ "total_lines": non_empty_lines,+ })+ .as_object()+ .cloned()+ .unwrap(),+ ))+}++fn try_detect_code(content: &str) -> Option<DetectionResult> {+ let lines: Vec<&str> = content.split('\n').take(100).collect();+ if lines.is_empty() {+ return None;+ }+ // Track scores in **first-match insertion order** to mirror Python's+ // dict semantics. Python:+ //+ // language_scores: dict[str, int] = {}+ // ...+ // best_lang = max(language_scores, key=lambda k: language_scores[k])+ //+ // - Languages are inserted into the dict the first time they match a+ // line, so the dict's iteration order is the order languages first+ // showed up — NOT registration order.+ // - `max(...)` returns the FIRST element with the maximum value when+ // multiple keys tie, per the language spec.+ //+ // We replicate both with a Vec and a manual `find(score == max)` for+ // the first-on-tie tie-break (Rust's `max_by` returns LAST on ties).+ let mut language_scores: Vec<(&'static str, u32)> = Vec::new();++ for line in &lines {+ for cp in CODE_PATTERNS.iter() {+ for pattern in &cp.patterns {+ if pattern.is_match(line) {+ if let Some(entry) = language_scores.iter_mut().find(|(n, _)| *n == cp.name) {+ entry.1 += 1;+ } else {+ language_scores.push((cp.name, 1));+ }+ break;+ }+ }+ }+ }++ if language_scores.is_empty() {+ return None;+ }+ let max_score = language_scores.iter().map(|x| x.1).max().unwrap_or(0);+ let (best_lang, best_score) = *language_scores+ .iter()+ .find(|x| x.1 == max_score)+ .expect("language_scores non-empty");+ if best_score < 3 {+ return None;+ }+ let non_empty_lines = lines.iter().filter(|l| !l.trim().is_empty()).count() as u32;+ let ratio = best_score as f64 / non_empty_lines.max(1) as f64;+ let confidence = (0.4 + ratio * 0.4 + (best_score as f64) * 0.02).min(1.0);+ Some(DetectionResult::new(+ ContentType::SourceCode,+ confidence,+ json!({+ "language": best_lang,+ "pattern_matches": best_score,+ })+ .as_object()+ .cloned()+ .unwrap(),+ ))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn empty_returns_plain_text_zero_confidence() {+ let r = detect_content_type("");+ assert_eq!(r.content_type, ContentType::PlainText);+ assert_eq!(r.confidence, 0.0);+ }++ #[test]+ fn whitespace_only_returns_plain_text_zero_confidence() {+ let r = detect_content_type(" \n\t ");+ assert_eq!(r.content_type, ContentType::PlainText);+ assert_eq!(r.confidence, 0.0);+ }++ #[test]+ fn json_array_of_dicts_high_confidence() {+ let r = detect_content_type(r#"[{"id": 1}, {"id": 2}]"#);+ assert_eq!(r.content_type, ContentType::JsonArray);+ assert_eq!(r.confidence, 1.0);+ assert_eq!(+ r.metadata.get("is_dict_array").unwrap().as_bool(),+ Some(true)+ );+ assert_eq!(r.metadata.get("item_count").unwrap().as_u64(), Some(2));+ }++ #[test]+ fn json_array_of_scalars_lower_confidence() {+ let r = detect_content_type(r#"[1, 2, 3]"#);+ assert_eq!(r.content_type, ContentType::JsonArray);+ assert_eq!(r.confidence, 0.8);+ assert_eq!(+ r.metadata.get("is_dict_array").unwrap().as_bool(),+ Some(false)+ );+ }++ #[test]+ fn empty_json_array_not_dict_array() {+ let r = detect_content_type("[]");+ assert_eq!(r.content_type, ContentType::JsonArray);+ assert_eq!(r.confidence, 0.8);+ assert_eq!(+ r.metadata.get("is_dict_array").unwrap().as_bool(),+ Some(false)+ );+ }++ #[test]+ fn json_object_falls_through_to_text() {+ // Detector only handles arrays.+ let r = detect_content_type(r#"{"id": 1}"#);+ assert_eq!(r.content_type, ContentType::PlainText);+ }++ #[test]+ fn search_results_detected() {+ let content =+ "src/main.py:42:def process():\nsrc/util.py:13: return None\nlib/x.py:7:class X:";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::SearchResults);+ assert!(r.confidence >= 0.6);+ }++ #[test]+ fn git_diff_detected() {+ let content = "\+diff --git a/foo.py b/foo.py+--- a/foo.py++++ b/foo.py+@@ -1,3 +1,4 @@+ def hello():+- print('hi')++ print('hello')++ print('world')+";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::GitDiff);+ assert!(r.confidence >= 0.7);+ }++ #[test]+ fn html_doctype_detected() {+ let content = "\+<!DOCTYPE html>+<html>+<head><title>X</title></head>+<body><div>hi</div></body>+</html>";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::Html);+ assert!(r.confidence >= 0.7);+ }++ #[test]+ fn build_output_detected() {+ let content = "\+[INFO] Starting build+[INFO] Compiling 42 sources+[ERROR] Compilation failed+[WARN] Deprecated API+FAILED test_one+PASSED test_two+";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::BuildOutput);+ assert!(r.confidence >= 0.5);+ }++ #[test]+ fn python_code_detected() {+ let content = "\+import os+from typing import Any++def process(data):+ return data++class Service:+ def __init__(self):+ pass++ @property+ def x(self):+ return 1++if __name__ == '__main__':+ process({})+";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::SourceCode);+ assert_eq!(r.metadata.get("language").unwrap().as_str(), Some("python"));+ }++ #[test]+ fn rust_code_detected() {+ let content = "\+use std::sync::Arc;++#[derive(Debug)]+pub struct Foo {+ bar: u32,+}++pub fn baz() -> u32 {+ 42+}++impl Foo {+ pub fn new() -> Self {+ Self { bar: 0 }+ }+}+";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::SourceCode);+ assert_eq!(r.metadata.get("language").unwrap().as_str(), Some("rust"));+ }++ #[test]+ fn go_code_detected() {+ let content = "\+package main++import \"fmt\"++func main() {+ fmt.Println(\"hello\")+}++type Service struct{}++func (s *Service) Do() {}++func helper() {}+";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::SourceCode);+ assert_eq!(r.metadata.get("language").unwrap().as_str(), Some("go"));+ }++ #[test]+ fn fallback_to_plain_text() {+ let content = "Just some random text without any special structure.";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::PlainText);+ assert_eq!(r.confidence, 0.5);+ }++ #[test]+ fn is_json_array_of_dicts_true_path() {+ assert!(is_json_array_of_dicts(r#"[{"a": 1}, {"a": 2}]"#));+ }++ #[test]+ fn is_json_array_of_dicts_scalars_returns_false() {+ assert!(!is_json_array_of_dicts(r#"[1, 2, 3]"#));+ }++ #[test]+ fn is_json_array_of_dicts_object_returns_false() {+ assert!(!is_json_array_of_dicts(r#"{"a": 1}"#));+ }++ #[test]+ fn is_json_array_of_dicts_empty_returns_false() {+ // Empty array is JsonArray but not is_dict_array.+ assert!(!is_json_array_of_dicts("[]"));+ }++ #[test]+ fn diff_low_confidence_does_not_short_circuit() {+ // Single header with no change lines yields 0.7 — borderline.+ // Should still register as diff (>= 0.7 threshold).+ let content = "diff --git a/x b/x\n";+ let r = detect_content_type(content);+ assert_eq!(r.content_type, ContentType::GitDiff);+ }++ #[test]+ fn html_below_threshold_falls_through() {+ // Just one structural tag — not enough.+ let r = detect_content_type("<div>hello</div>");+ assert_ne!(r.content_type, ContentType::Html);+ }++ #[test]+ fn content_type_string_tags_match_python() {+ assert_eq!(ContentType::JsonArray.as_str(), "json_array");+ assert_eq!(ContentType::SourceCode.as_str(), "source_code");+ assert_eq!(ContentType::SearchResults.as_str(), "search");+ assert_eq!(ContentType::BuildOutput.as_str(), "build");+ assert_eq!(ContentType::GitDiff.as_str(), "diff");+ assert_eq!(ContentType::Html.as_str(), "html");+ assert_eq!(ContentType::PlainText.as_str(), "text");+ }+}diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rsindex d3c7eeaf2..1596fab20 100644--- a/crates/headroom-core/src/transforms/mod.rs+++ b/crates/headroom-core/src/transforms/mod.rs@@ -17,9 +17,13 @@pub mod adaptive_sizer;pub mod anchor_selector;+pub mod content_detector;pub mod diff_compressor;pub mod smart_crusher;+pub use content_detector::{+ detect_content_type, is_json_array_of_dicts, ContentType, DetectionResult,+};pub use diff_compressor::{DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,};diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rsindex 189635b31..55a2ae12a 100644--- a/crates/headroom-parity/src/lib.rs+++ b/crates/headroom-parity/src/lib.rs@@ -45,6 +45,15 @@ pub trait TransformComparator {}/// Compare a single fixture against a comparator and return an outcome.+///+/// f64 normalization: `serde_json` (without the `arbitrary_precision`+/// feature) has an asymmetry — values constructed via `json!(f64)` keep+/// full precision (e.g. `0.9500000000000001`), but values parsed from+/// fixture JSON sometimes round to a neighboring f64 (e.g. `0.95`,+/// differing by 1 ULP). To make comparisons robust we round-trip the+/// comparator's output through `to_string` + `from_str` so it goes+/// through the same lossy parser the fixture did. Bit-identical f64s+/// from both sides then compare equal.pub fn compare_fixture(comparator: &dyn TransformComparator,fixture: &Fixture,@@ -57,12 +66,15 @@ pub fn compare_fixture(})}};- if actual == fixture.output {+ let actual_normalized: serde_json::Value =+ serde_json::from_str(&serde_json::to_string(&actual)?)+ .context("re-parsing comparator output through serde_json (f64 normalization)")?;+ if actual_normalized == fixture.output {Ok(ComparisonOutcome::Match)} else {Ok(ComparisonOutcome::Diff {expected: serde_json::to_string_pretty(&fixture.output)?,- actual: serde_json::to_string_pretty(&actual)?,+ actual: serde_json::to_string_pretty(&actual_normalized)?,})}}@@ -401,6 +413,48 @@ impl TransformComparator for SmartCrusherComparator {}}+/// Real comparator for the `content_detector` transform. Drives the Rust+/// port over the recorded fixture inputs (a single JSON string) and+/// emits the same shape Python's recorder serializes for+/// `DetectionResult`:+///+/// ```json+/// {"content_type": "json_array", "confidence": 1.0, "metadata": {...}}+/// ```+///+/// Python's recorder relies on `_json_default` to serialize the+/// `DetectionResult` dataclass and the `ContentType` enum:+/// - dataclass → `asdict(...)` produces `{content_type, confidence, metadata}`.+/// - enum → its `.value` (the lowercase tag, e.g. "json_array").+///+/// Numeric fields in metadata are recorded as JSON numbers (Python ints+/// stay ints), so we mirror that exactly with `serde_json::Number`.+pub struct ContentDetectorComparator;++impl TransformComparator for ContentDetectorComparator {+ fn name(&self) -> &str {+ "content_detector"+ }++ fn run(+ &self,+ input: &serde_json::Value,+ _config: &serde_json::Value,+ ) -> Result<serde_json::Value> {+ use headroom_core::transforms::detect_content_type;++ let content = input+ .as_str()+ .context("content_detector fixture input must be a JSON string")?;+ let result = detect_content_type(content);+ Ok(serde_json::json!({+ "content_type": result.content_type.as_str(),+ "confidence": result.confidence,+ "metadata": serde_json::Value::Object(result.metadata),+ }))+ }+}+/// Every built-in comparator, in a stable order.pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {vec![@@ -410,6 +464,7 @@ pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {Box::new(TokenizerComparator),Box::new(CcrComparator),Box::new(SmartCrusherComparator),+ Box::new(ContentDetectorComparator),]}diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex b5b0eb36a..d0f837985 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -21,10 +21,13 @@ use headroom_core::transforms::smart_crusher::{SmartCrusherConfig as RustSmartCrusherConfig,};use headroom_core::transforms::{- DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,+ detect_content_type as rust_detect_content_type,+ is_json_array_of_dicts as rust_is_json_array_of_dicts, ContentType as RustContentType,+ DetectionResult as RustDetectionResult, DiffCompressionResult, DiffCompressor,+ DiffCompressorConfig, DiffCompressorStats,};use pyo3::prelude::*;-use pyo3::types::PyDict;+use pyo3::types::{PyDict, PyString};/// Identity stub used by the Python smoke test to verify linkage.#[pyfunction]@@ -753,6 +756,109 @@ impl PySmartCrusher {}}+// ─── ContentDetector ───────────────────────────────────────────────────────++/// Mirror of `headroom.transforms.content_detector.DetectionResult`.+///+/// Field names + types match the Python dataclass exactly so the existing+/// Python `ContentRouter` (which `import`s `DetectionResult` directly) can+/// continue to read `.content_type`, `.confidence`, and `.metadata` without+/// modification.+///+/// `content_type` is exposed as the lowercase string tag (e.g.+/// `"json_array"`). The Python wrapper translates it back into the+/// `ContentType` enum so the call-site looks identical.+#[pyclass(name = "DetectionResult", module = "headroom._core")]+#[derive(Clone)]+struct PyDetectionResult {+ inner: RustDetectionResult,+}++#[pymethods]+impl PyDetectionResult {+ #[getter]+ fn content_type(&self) -> &'static str {+ self.inner.content_type.as_str()+ }++ #[getter]+ fn confidence(&self) -> f64 {+ self.inner.confidence+ }++ /// Per-type metadata bag (e.g. `{"language": "python", "pattern_matches": 5}`+ /// for code, `{"item_count": 3, "is_dict_array": true}` for JSON arrays).+ /// Returned as a fresh `dict` so callers can mutate without affecting+ /// the underlying Rust value.+ #[getter]+ fn metadata<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {+ let dict = PyDict::new_bound(py);+ for (k, v) in &self.inner.metadata {+ // Convert each JSON value into the closest Python primitive.+ // Detection metadata is always a flat dict of scalars (ints,+ // bools, strings) so we don't need to recurse.+ let py_value: PyObject = match v {+ serde_json::Value::Bool(b) => b.into_py(py),+ serde_json::Value::Number(n) => {+ if let Some(i) = n.as_u64() {+ i.into_py(py)+ } else if let Some(i) = n.as_i64() {+ i.into_py(py)+ } else if let Some(f) = n.as_f64() {+ f.into_py(py)+ } else {+ py.None()+ }+ }+ serde_json::Value::String(s) => PyString::new_bound(py, s).into_py(py),+ serde_json::Value::Null => py.None(),+ // Detection never emits arrays / objects in metadata+ // today; if it ever does, fall through to JSON-string for+ // visibility rather than silently dropping.+ other => PyString::new_bound(py, &other.to_string()).into_py(py),+ };+ dict.set_item(k, py_value)?;+ }+ Ok(dict)+ }++ fn __repr__(&self) -> String {+ format!(+ "DetectionResult(content_type={:?}, confidence={}, metadata=<{} keys>)",+ self.inner.content_type.as_str(),+ self.inner.confidence,+ self.inner.metadata.len()+ )+ }+}++/// Detect the type of `content`. Returns a `DetectionResult` with the+/// same field surface as Python's dataclass.+///+/// Releases the GIL while detecting — pattern matching can be substantial+/// on large bodies (HTML scans, 500-line diff windows), and freeing the+/// GIL lets other Python threads make progress in the meantime.+#[pyfunction]+fn detect_content_type(py: Python<'_>, content: &str) -> PyDetectionResult {+ let owned = content.to_string();+ let result = py.allow_threads(move || rust_detect_content_type(&owned));+ PyDetectionResult { inner: result }+}++/// Quick check: is `content` a JSON array of dictionaries (the format+/// `SmartCrusher` natively handles)?+#[pyfunction]+fn is_json_array_of_dicts(py: Python<'_>, content: &str) -> bool {+ let owned = content.to_string();+ py.allow_threads(move || rust_is_json_array_of_dicts(&owned))+}++// Suppress unused-import warning when ContentType isn't referenced+// directly — `as_str()` is the public surface.+const _: fn() = || {+ let _ = RustContentType::PlainText;+};+// ─── Module init ───────────────────────────────────────────────────────────#[pymodule]@@ -765,5 +871,8 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_class::<PySmartCrusherConfig>()?;m.add_class::<PyCrushResult>()?;m.add_class::<PySmartCrusher>()?;+ m.add_class::<PyDetectionResult>()?;+ m.add_function(wrap_pyfunction!(detect_content_type, m)?)?;+ m.add_function(wrap_pyfunction!(is_json_array_of_dicts, m)?)?;Ok(())}diff --git a/tests/parity/fixtures/content_detector/247811aecfdec556.json b/tests/parity/fixtures/content_detector/247811aecfdec556.jsonnew file mode 100644index 000000000..87bc23d75--- /dev/null+++ b/tests/parity/fixtures/content_detector/247811aecfdec556.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "diff --combined merged.py\nindex aaa..bbb..ccc 100644\n--- a/merged.py\n+++ b/merged.py\n@@@ -1,4 -1,4 +1,5 @@@\n unchanged\n- branch_a_only\n -branch_b_only\n++merge_added\n",+ "input_sha256": "247811aecfdec55647a37c088c611f8f093119d95335da3d10aa317455159655",+ "output": {+ "confidence": 1.0,+ "content_type": "diff",+ "metadata": {+ "change_lines": 1,+ "header_matches": 3+ }+ },+ "recorded_at": "2026-04-28T06:39:34.959992+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/37c86907c057e293.json b/tests/parity/fixtures/content_detector/37c86907c057e293.jsonnew file mode 100644index 000000000..aa635897b--- /dev/null+++ b/tests/parity/fixtures/content_detector/37c86907c057e293.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "package com.example;\n\npublic class Foo {\n @Override\n public String toString() { return \"foo\"; }\n}\n\nprivate interface Bar {}\nprotected enum Baz { A, B }\n",+ "input_sha256": "37c86907c057e293385ffaef6c4278a4b903fca82e3d52fb42df601f47010a3d",+ "output": {+ "confidence": 0.7857142857142857,+ "content_type": "source_code",+ "metadata": {+ "language": "java",+ "pattern_matches": 5+ }+ },+ "recorded_at": "2026-04-28T06:39:34.962175+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/3a0b14d2dea7c876.json b/tests/parity/fixtures/content_detector/3a0b14d2dea7c876.jsonnew file mode 100644index 000000000..27d4837d1--- /dev/null+++ b/tests/parity/fixtures/content_detector/3a0b14d2dea7c876.json@@ -0,0 +1,12 @@+{+ "config": {},+ "input": "Just some prose text without any structure or special markers.",+ "input_sha256": "3a0b14d2dea7c87688fc40482a02f006accfae1da50f0734a1e395fed13fc744",+ "output": {+ "confidence": 0.5,+ "content_type": "text",+ "metadata": {}+ },+ "recorded_at": "2026-04-28T06:39:34.962324+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/3ba74a1f57c5351e.json b/tests/parity/fixtures/content_detector/3ba74a1f57c5351e.jsonnew file mode 100644index 000000000..1698bbf0f--- /dev/null+++ b/tests/parity/fixtures/content_detector/3ba74a1f57c5351e.json@@ -0,0 +1,12 @@+{+ "config": {},+ "input": "<div>a</div>\n<span>b</span>\n<script>x()</script>\n<style>y</style>",+ "input_sha256": "3ba74a1f57c5351ecb47efcfff73080bc3f96a75589b459e20424fe0c715be39",+ "output": {+ "confidence": 0.5,+ "content_type": "text",+ "metadata": {}+ },+ "recorded_at": "2026-04-28T06:39:34.960293+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/4efe946303371867.json b/tests/parity/fixtures/content_detector/4efe946303371867.jsonnew file mode 100644index 000000000..28755b045--- /dev/null+++ b/tests/parity/fixtures/content_detector/4efe946303371867.json@@ -0,0 +1,12 @@+{+ "config": {},+ "input": "",+ "input_sha256": "4efe946303371867389009a0144a8ea235bfe9f6e87a9395b0c0453b8e3f99c0",+ "output": {+ "confidence": 0.0,+ "content_type": "text",+ "metadata": {}+ },+ "recorded_at": "2026-04-28T06:39:34.962454+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/53400da2b7dd2428.json b/tests/parity/fixtures/content_detector/53400da2b7dd2428.jsonnew file mode 100644index 000000000..8e3685e48--- /dev/null+++ b/tests/parity/fixtures/content_detector/53400da2b7dd2428.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "import x from 'y';\nexport const foo = 1;\nfunction bar() { return 42; }\nconst f = async function() {};\nmodule.exports = { foo, bar };\n",+ "input_sha256": "53400da2b7dd2428c8146088e60779a8d61c2430a2c7c95a6d2b07e83b9b726e",+ "output": {+ "confidence": 0.9,+ "content_type": "source_code",+ "metadata": {+ "language": "javascript",+ "pattern_matches": 5+ }+ },+ "recorded_at": "2026-04-28T06:39:34.961426+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/552d30c30c2a6793.json b/tests/parity/fixtures/content_detector/552d30c30c2a6793.jsonnew file mode 100644index 000000000..15b6e815f--- /dev/null+++ b/tests/parity/fixtures/content_detector/552d30c30c2a6793.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "diff --git a/foo.py b/foo.py\nindex abc..def 100644\n--- a/foo.py\n+++ b/foo.py\n@@ -1,3 +1,3 @@\n-old line\n+new line\n unchanged\n",+ "input_sha256": "552d30c30c2a67939857c22f85696406f1557476912b41d9918031a15364163a",+ "output": {+ "confidence": 1.0,+ "content_type": "diff",+ "metadata": {+ "change_lines": 2,+ "header_matches": 3+ }+ },+ "recorded_at": "2026-04-28T06:39:34.959842+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/5e8e1bc29f6c71c5.json b/tests/parity/fixtures/content_detector/5e8e1bc29f6c71c5.jsonnew file mode 100644index 000000000..0c07f0227--- /dev/null+++ b/tests/parity/fixtures/content_detector/5e8e1bc29f6c71c5.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "import os\nfrom typing import Any\n\n@dataclass\nclass Foo:\n \"\"\"Docstring.\"\"\"\n def bar(self):\n return 42\n\ndef baz():\n pass\n\nif __name__ == '__main__':\n baz()\n",+ "input_sha256": "5e8e1bc29f6c71c5b6523dc65f91a615b15e8bd27d633cbcd54293bbe43b6951",+ "output": {+ "confidence": 0.850909090909091,+ "content_type": "source_code",+ "metadata": {+ "language": "python",+ "pattern_matches": 8+ }+ },+ "recorded_at": "2026-04-28T06:39:34.961138+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/628c8985c40adb7d.json b/tests/parity/fixtures/content_detector/628c8985c40adb7d.jsonnew file mode 100644index 000000000..484bb4760--- /dev/null+++ b/tests/parity/fixtures/content_detector/628c8985c40adb7d.json@@ -0,0 +1,16 @@+{+ "config": {},+ "input": "INFO starting build\nWARN deprecated API used\nERROR compilation failed\nFAILED test_x\nPASSED test_y\n============================================\n",+ "input_sha256": "628c8985c40adb7d265eac37436ea45a549ba9ed2b10eb47a1fa7b478ce17e77",+ "output": {+ "confidence": 0.9500000000000001,+ "content_type": "build",+ "metadata": {+ "error_matches": 3,+ "pattern_matches": 6,+ "total_lines": 6+ }+ },+ "recorded_at": "2026-04-28T06:39:34.960715+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/67f4b1a1697c8aa9.json b/tests/parity/fixtures/content_detector/67f4b1a1697c8aa9.jsonnew file mode 100644index 000000000..9b6063dc9--- /dev/null+++ b/tests/parity/fixtures/content_detector/67f4b1a1697c8aa9.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "interface User { id: number; name: string; }\ntype Maybe<T> = T | null;\nenum Color { Red, Green, Blue }\nfunction f(x: number): boolean { return x > 0; }\n",+ "input_sha256": "67f4b1a1697c8aa916e4acb0920a92279bbc35efa3849edc9bc2aa8dcff73fb3",+ "output": {+ "confidence": 0.76,+ "content_type": "source_code",+ "metadata": {+ "language": "typescript",+ "pattern_matches": 3+ }+ },+ "recorded_at": "2026-04-28T06:39:34.961598+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/70fbb444a583fe3c.json b/tests/parity/fixtures/content_detector/70fbb444a583fe3c.jsonnew file mode 100644index 000000000..6bd99fe91--- /dev/null+++ b/tests/parity/fixtures/content_detector/70fbb444a583fe3c.json@@ -0,0 +1,12 @@+{+ "config": {},+ "input": " \n\t \n",+ "input_sha256": "70fbb444a583fe3ce3d9d5fd6783e64a5261afa0a0f7b25b0a48ec224d255534",+ "output": {+ "confidence": 0.0,+ "content_type": "text",+ "metadata": {}+ },+ "recorded_at": "2026-04-28T06:39:34.962581+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/71d648a39e151a72.json b/tests/parity/fixtures/content_detector/71d648a39e151a72.jsonnew file mode 100644index 000000000..206665acf--- /dev/null+++ b/tests/parity/fixtures/content_detector/71d648a39e151a72.json@@ -0,0 +1,16 @@+{+ "config": {},+ "input": "============================= test session starts ==============================\ntests/test_0.py::test_case PASSED\ntests/test_1.py::test_case PASSED\ntests/test_2.py::test_case PASSED\ntests/test_3.py::test_case PASSED\ntests/test_4.py::test_case PASSED\nFAILED tests/test_5.py::test_bad\nTraceback (most recent call last)\n",+ "input_sha256": "71d648a39e151a72411fe014b2e0820abc922609752d74798d26bf6c99fd2168",+ "output": {+ "confidence": 0.5375,+ "content_type": "build",+ "metadata": {+ "error_matches": 1,+ "pattern_matches": 3,+ "total_lines": 8+ }+ },+ "recorded_at": "2026-04-28T06:39:34.960922+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/79e6b68a05a1c639.json b/tests/parity/fixtures/content_detector/79e6b68a05a1c639.jsonnew file mode 100644index 000000000..321ac8808--- /dev/null+++ b/tests/parity/fixtures/content_detector/79e6b68a05a1c639.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "src/main.py:42:def process():\nsrc/util.py:13: return None\nlib/x.py:7:class X:\ntests/test_a.py:3: assert True",+ "input_sha256": "79e6b68a05a1c639efba2fe2f3bf8471ea00a04e9dc0a5fb8853cbc94ac6b42a",+ "output": {+ "confidence": 1.0,+ "content_type": "search",+ "metadata": {+ "matching_lines": 4,+ "total_lines": 4+ }+ },+ "recorded_at": "2026-04-28T06:39:34.960560+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/7e145153192c11a1.json b/tests/parity/fixtures/content_detector/7e145153192c11a1.jsonnew file mode 100644index 000000000..c62c9f946--- /dev/null+++ b/tests/parity/fixtures/content_detector/7e145153192c11a1.json@@ -0,0 +1,12 @@+{+ "config": {},+ "input": "<p>just one tag</p>",+ "input_sha256": "7e145153192c11a1b2ffc574059deaf3990e1cb6c83f85b6c5ed0a2bfdb93ab5",+ "output": {+ "confidence": 0.5,+ "content_type": "text",+ "metadata": {}+ },+ "recorded_at": "2026-04-28T06:39:34.960430+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/7fde660489bc47aa.json b/tests/parity/fixtures/content_detector/7fde660489bc47aa.jsonnew file mode 100644index 000000000..ba15f5c9e--- /dev/null+++ b/tests/parity/fixtures/content_detector/7fde660489bc47aa.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "[]",+ "input_sha256": "7fde660489bc47aa12bccf43567d6c386c36670ca5185e8a5f8781ac257f7c7d",+ "output": {+ "confidence": 0.8,+ "content_type": "json_array",+ "metadata": {+ "is_dict_array": false,+ "item_count": 0+ }+ },+ "recorded_at": "2026-04-28T06:39:34.959529+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/837b272f1604eaaa.json b/tests/parity/fixtures/content_detector/837b272f1604eaaa.jsonnew file mode 100644index 000000000..435369ae4--- /dev/null+++ b/tests/parity/fixtures/content_detector/837b272f1604eaaa.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "package main\n\nimport \"fmt\"\n\ntype Foo struct { ID int }\n\nfunc (f *Foo) Bar() int { return f.ID }\n\nfunc main() { fmt.Println(\"hi\") }\n",+ "input_sha256": "837b272f1604eaaadee6f9d33bff0b00d85204cc3ce8ae9c064964d8a432a611",+ "output": {+ "confidence": 0.9,+ "content_type": "source_code",+ "metadata": {+ "language": "go",+ "pattern_matches": 5+ }+ },+ "recorded_at": "2026-04-28T06:39:34.961787+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/9fb76321d31ed64c.json b/tests/parity/fixtures/content_detector/9fb76321d31ed64c.jsonnew file mode 100644index 000000000..3590bdcb3--- /dev/null+++ b/tests/parity/fixtures/content_detector/9fb76321d31ed64c.json@@ -0,0 +1,12 @@+{+ "config": {},+ "input": "{\"foo\": \"bar\"}",+ "input_sha256": "9fb76321d31ed64c9b59daa7456102070d25ea02b8a414049a9b93a1b50751f7",+ "output": {+ "confidence": 0.5,+ "content_type": "text",+ "metadata": {}+ },+ "recorded_at": "2026-04-28T06:39:34.959686+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/bc4488ce189d1c86.json b/tests/parity/fixtures/content_detector/bc4488ce189d1c86.jsonnew file mode 100644index 000000000..8d313c3e7--- /dev/null+++ b/tests/parity/fixtures/content_detector/bc4488ce189d1c86.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "[{\"id\":1,\"name\":\"a\"},{\"id\":2,\"name\":\"b\"},{\"id\":3,\"name\":\"c\"}]",+ "input_sha256": "bc4488ce189d1c867e5b600d4691354f94a3f63ecf869ad2bc89344cb2142566",+ "output": {+ "confidence": 1.0,+ "content_type": "json_array",+ "metadata": {+ "is_dict_array": true,+ "item_count": 3+ }+ },+ "recorded_at": "2026-04-28T06:39:34.959161+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/bf9b9ceb4b70dcd0.json b/tests/parity/fixtures/content_detector/bf9b9ceb4b70dcd0.jsonnew file mode 100644index 000000000..e39f1da8c--- /dev/null+++ b/tests/parity/fixtures/content_detector/bf9b9ceb4b70dcd0.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "use std::collections::HashMap;\n\npub struct Foo { id: u32 }\n\nimpl Foo {\n pub fn new() -> Self { Self { id: 0 } }\n}\n\nfn main() {}\n#[derive(Debug)]\nenum Color { Red, Green }\n",+ "input_sha256": "bf9b9ceb4b70dcd0a291c57ff0a53690e575f6d2d4273eb9d2a73b7f9beb29db",+ "output": {+ "confidence": 0.89,+ "content_type": "source_code",+ "metadata": {+ "language": "rust",+ "pattern_matches": 7+ }+ },+ "recorded_at": "2026-04-28T06:39:34.961975+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/cabc55445af63e2f.json b/tests/parity/fixtures/content_detector/cabc55445af63e2f.jsonnew file mode 100644index 000000000..fcc81d7e9--- /dev/null+++ b/tests/parity/fixtures/content_detector/cabc55445af63e2f.json@@ -0,0 +1,15 @@+{+ "config": {},+ "input": "[1, 2, 3, 4, 5]",+ "input_sha256": "cabc55445af63e2fa6667ee023240b19884644bd72ef99c7a0f232d8fecd674f",+ "output": {+ "confidence": 0.8,+ "content_type": "json_array",+ "metadata": {+ "is_dict_array": false,+ "item_count": 5+ }+ },+ "recorded_at": "2026-04-28T06:39:34.959327+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/fixtures/content_detector/d100a9ac7a3671da.json b/tests/parity/fixtures/content_detector/d100a9ac7a3671da.jsonnew file mode 100644index 000000000..aef1901bb--- /dev/null+++ b/tests/parity/fixtures/content_detector/d100a9ac7a3671da.json@@ -0,0 +1,16 @@+{+ "config": {},+ "input": "<!DOCTYPE html>\n<html>\n<head><title>x</title></head>\n<body><div><span>hi</span></div></body>\n</html>",+ "input_sha256": "d100a9ac7a3671dafbc6666243ba796c0985281868d47921bd1f75506b4d81ce",+ "output": {+ "confidence": 1.0,+ "content_type": "html",+ "metadata": {+ "has_doctype": true,+ "has_html_tag": true,+ "structural_tags": 2+ }+ },+ "recorded_at": "2026-04-28T06:39:34.960132+00:00",+ "transform": "content_detector"+}diff --git a/tests/parity/recorder.py b/tests/parity/recorder.pyindex b231bdf97..5707586a6 100644--- a/tests/parity/recorder.py+++ b/tests/parity/recorder.py@@ -264,6 +264,22 @@ def record_all(root: Path | None = None) -> dict[str, str]:except Exception as e:statuses["ccr"] = f"blocked:{e.__class__.__name__}:{e}"+ # --- content_detector --------------------------------------------------+ # `detect_content_type` is a module-level function, so we monkey-patch+ # the module attribute rather than a class method.+ try:+ from headroom.transforms import content_detector as _cd_mod++ _wrap_function(+ _cd_mod,+ "detect_content_type",+ "content_detector",+ root=root,+ )+ statuses["content_detector"] = "patched"+ except Exception as e:+ statuses["content_detector"] = f"blocked:{e.__class__.__name__}:{e}"+return statuses@@ -291,6 +307,32 @@ def _wrap_method(setattr(cls, method_name, wrapped)+def _wrap_function(+ module: Any,+ fn_name: str,+ transform_name: str,+ *,+ root: Path | None = None,+ input_arg: int = 0,+ input_kw: str | None = None,+) -> None:+ """Monkey-patch a module-level free function the same way `_wrap_method`+ handles class methods. Idempotent; safe to call repeatedly."""+ original = getattr(module, fn_name)+ if getattr(original, "_parity_recorder_wrapped", False):+ return # idempotent++ decorator = record(+ transform_name,+ root=root,+ input_arg=input_arg,+ input_kw=input_kw,+ )+ wrapped = decorator(original)+ wrapped._parity_recorder_wrapped = True # type: ignore[attr-defined]+ setattr(module, fn_name, wrapped)++# ---------------------------------------------------------------------------# Minimal workload helpers — callable independently of the scripts entry.# ---------------------------------------------------------------------------@@ -541,6 +583,147 @@ def _varied_text_inputs() -> list[str]:return out[:20]+def _varied_content_detector_inputs() -> list[str]:+ """Hit every dispatch branch in `detect_content_type`. Each entry below+ targets a specific path so parity diffs surface the right branch."""+ json_array_dicts = '[{"id":1,"name":"a"},{"id":2,"name":"b"},{"id":3,"name":"c"}]'+ json_array_scalars = "[1, 2, 3, 4, 5]"+ json_empty_array = "[]"+ json_object = '{"foo": "bar"}'++ git_diff = (+ "diff --git a/foo.py b/foo.py\n"+ "index abc..def 100644\n"+ "--- a/foo.py\n"+ "+++ b/foo.py\n"+ "@@ -1,3 +1,3 @@\n"+ "-old line\n"+ "+new line\n"+ " unchanged\n"+ )+ merge_diff = (+ "diff --combined merged.py\n"+ "index aaa..bbb..ccc 100644\n"+ "--- a/merged.py\n"+ "+++ b/merged.py\n"+ "@@@ -1,4 -1,4 +1,5 @@@\n"+ " unchanged\n"+ "- branch_a_only\n"+ " -branch_b_only\n"+ "++merge_added\n"+ )++ html_doctype = (+ "<!DOCTYPE html>\n<html>\n<head><title>x</title></head>\n"+ "<body><div><span>hi</span></div></body>\n</html>"+ )+ html_structural = "<div>a</div>\n<span>b</span>\n<script>x()</script>\n<style>y</style>"+ html_below_threshold = "<p>just one tag</p>"++ search_results = (+ "src/main.py:42:def process():\n"+ "src/util.py:13: return None\n"+ "lib/x.py:7:class X:\n"+ "tests/test_a.py:3: assert True"+ )++ build_log = (+ "INFO starting build\n"+ "WARN deprecated API used\n"+ "ERROR compilation failed\n"+ "FAILED test_x\n"+ "PASSED test_y\n"+ "============================================\n"+ )+ pytest_output = (+ "============================= test session starts ==============================\n"+ + "\n".join(f"tests/test_{i}.py::test_case PASSED" for i in range(5))+ + "\nFAILED tests/test_5.py::test_bad\n"+ + "Traceback (most recent call last)\n"+ )++ python_code = (+ "import os\n"+ "from typing import Any\n\n"+ "@dataclass\n"+ "class Foo:\n"+ ' """Docstring."""\n'+ " def bar(self):\n"+ " return 42\n\n"+ "def baz():\n"+ " pass\n\n"+ "if __name__ == '__main__':\n"+ " baz()\n"+ )+ js_code = (+ "import x from 'y';\n"+ "export const foo = 1;\n"+ "function bar() { return 42; }\n"+ "const f = async function() {};\n"+ "module.exports = { foo, bar };\n"+ )+ ts_code = (+ "interface User { id: number; name: string; }\n"+ "type Maybe<T> = T | null;\n"+ "enum Color { Red, Green, Blue }\n"+ "function f(x: number): boolean { return x > 0; }\n"+ )+ go_code = (+ "package main\n\n"+ 'import "fmt"\n\n'+ "type Foo struct { ID int }\n\n"+ "func (f *Foo) Bar() int { return f.ID }\n\n"+ 'func main() { fmt.Println("hi") }\n'+ )+ rust_code = (+ "use std::collections::HashMap;\n\n"+ "pub struct Foo { id: u32 }\n\n"+ "impl Foo {\n"+ " pub fn new() -> Self { Self { id: 0 } }\n"+ "}\n\n"+ "fn main() {}\n"+ "#[derive(Debug)]\n"+ "enum Color { Red, Green }\n"+ )+ java_code = (+ "package com.example;\n\n"+ "public class Foo {\n"+ " @Override\n"+ ' public String toString() { return "foo"; }\n'+ "}\n\n"+ "private interface Bar {}\n"+ "protected enum Baz { A, B }\n"+ )++ plain_text = "Just some prose text without any structure or special markers."+ empty = ""+ whitespace = " \n\t \n"++ return [+ json_array_dicts,+ json_array_scalars,+ json_empty_array,+ json_object,+ git_diff,+ merge_diff,+ html_doctype,+ html_structural,+ html_below_threshold,+ search_results,+ build_log,+ pytest_output,+ python_code,+ js_code,+ ts_code,+ go_code,+ rust_code,+ java_code,+ plain_text,+ empty,+ whitespace,+ ]++def _varied_message_batches() -> list[list[dict[str, Any]]]:today = _dt.date.today().isoformat()out: list[list[dict[str, Any]]] = []@@ -566,6 +749,7 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]:"tokenizer": 0,"cache_aligner": 0,"ccr": 0,+ "content_detector": 0,}# log_compressor@@ -643,6 +827,18 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]:except Exception as e:LOG.warning("ccr workload failed: %s", e)+ # content_detector — drive a wide mix of content types so every dispatch+ # branch (json_array, diff, html, search, log, code-by-language,+ # plain-text fallback) is exercised at least once.+ try:+ from headroom.transforms import content_detector as _cd_mod++ for s in _varied_content_detector_inputs():+ _cd_mod.detect_content_type(s)+ counts["content_detector"] += 1+ except Exception as e:+ LOG.warning("content_detector workload failed: %s", e)+return counts