Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore(rust): port ContentDetector to Rust + parity harness + PyO3 bridge

chopratejas625290cfbce6
diff --git a/crates/headroom-core/src/transforms/content_detector.rs b/crates/headroom-core/src/transforms/content_detector.rs
new file mode 100644
index 000000000..57b46cd14
--- /dev/null
+++ b/crates/headroom-core/src/transforms/content_detector.rs
@@ -0,0 +1,769 @@
+//! Content type detection for multi-format compression.
+//!
+//! Direct port of `headroom/transforms/content_detector.py`. This module
+//! detects the type of tool output content so the upstream
+//! `ContentRouter` can dispatch it to the right compressor:
+//!
+//! - **JsonArray**: Structured JSON data → `SmartCrusher`
+//! - **SourceCode**: Python, JavaScript, Go, Rust, etc. → `CodeAwareCompressor`
+//! - **SearchResults**: grep / ripgrep output (`file:line:content`)
+//! - **BuildOutput**: Compiler / test / lint logs
+//! - **GitDiff**: Unified diff format → `DiffCompressor`
+//! - **Html**: Web pages (needs extraction, not compression)
+//! - **PlainText**: Generic fallback
+//!
+//! Detection is **regex-based** — no ML, no model loading, no I/O.
+//! Magika integration lives one level up in `ContentRouter`, not here.
+//!
+//! # Parity with Python
+//!
+//! Regex patterns, dispatch order, confidence formulas, and line-count
+//! caps are byte-equal with the Python source. Recorded fixtures in
+//! `tests/parity/fixtures/content_detector/` lock the output across
+//! the bridge.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+use serde_json::{json, Map, Value};
+
+/// Content types recognized by the detector. String tags match Python's
+/// `ContentType` enum values 1:1.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ContentType {
+ JsonArray,
+ SourceCode,
+ SearchResults,
+ BuildOutput,
+ GitDiff,
+ Html,
+ PlainText,
+}
+
+impl ContentType {
+ /// Stable string tag — matches Python's `ContentType.<NAME>.value`.
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ ContentType::JsonArray => "json_array",
+ ContentType::SourceCode => "source_code",
+ ContentType::SearchResults => "search",
+ ContentType::BuildOutput => "build",
+ ContentType::GitDiff => "diff",
+ ContentType::Html => "html",
+ ContentType::PlainText => "text",
+ }
+ }
+}
+
+/// Result of `detect_content_type`. `metadata` is per-type free-form key/
+/// value data — same shape as Python's `dict[str, Any]`. We use
+/// `serde_json::Map` so PyO3 can convert it to a Python dict on the
+/// boundary without losing type fidelity.
+#[derive(Debug, Clone)]
+pub struct DetectionResult {
+ pub content_type: ContentType,
+ pub confidence: f64,
+ pub metadata: Map<String, Value>,
+}
+
+impl DetectionResult {
+ fn new(content_type: ContentType, confidence: f64, metadata: Map<String, Value>) -> Self {
+ Self {
+ content_type,
+ confidence,
+ metadata,
+ }
+ }
+
+ fn plain_text(confidence: f64) -> Self {
+ Self::new(ContentType::PlainText, confidence, Map::new())
+ }
+}
+
+// ─── Regex patterns (compiled once, shared) ───────────────────────────
+
+/// `file:line:` (grep -n style) — first column on a non-blank line.
+static SEARCH_RESULT_PATTERN: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"^[^\s:]+:\d+:").unwrap());
+
+/// Diff-header detection. Recognizes:
+/// - `git diff` (`diff --git`, `--- a/`)
+/// - merge-commit headers (`diff --combined`, `diff --cc`)
+/// - regular hunk headers (`@@ -A,B +C,D @@`)
+/// - combined-diff hunk headers (`@@@ ... @@@`)
+///
+/// Mirrors Python's bug-fix from 2026-04-25 that widened the grammar
+/// to handle merge-commit diffs from `git log -p`.
+static DIFF_HEADER_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(
+ r"^(diff --git|diff --combined |diff --cc |--- a/|@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@|@@@+\s+-\d+(?:,\d+)?\s+(?:-\d+(?:,\d+)?\s+)+\+\d+(?:,\d+)?\s+@@@+)",
+ )
+ .unwrap()
+});
+
+/// Lines starting with `+` or `-` followed by a non-`+`/`-` char (i.e.
+/// real change lines, not header lines like `+++ b/file`).
+static DIFF_CHANGE_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[+-][^+-]").unwrap());
+
+// ─── Code patterns by language ─────────────────────────────────────────
+
+struct CodePatterns {
+ name: &'static str,
+ patterns: Vec<Regex>,
+}
+
+static CODE_PATTERNS: LazyLock<Vec<CodePatterns>> = LazyLock::new(|| {
+ vec![
+ CodePatterns {
+ name: "python",
+ patterns: vec![
+ Regex::new(r"^\s*(def|class|import|from|async def)\s+\w+").unwrap(),
+ Regex::new(r"^\s*@\w+").unwrap(),
+ Regex::new(r#"^\s*""""#).unwrap(),
+ Regex::new(r"^\s*if __name__\s*==").unwrap(),
+ ],
+ },
+ CodePatterns {
+ name: "javascript",
+ patterns: vec![
+ Regex::new(r"^\s*(function|const|let|var|class|import|export)\s+").unwrap(),
+ Regex::new(r"^\s*(async\s+function|=>\s*\{)").unwrap(),
+ Regex::new(r"^\s*module\.exports").unwrap(),
+ ],
+ },
+ CodePatterns {
+ name: "typescript",
+ patterns: vec![
+ Regex::new(r"^\s*(interface|type|enum|namespace)\s+\w+").unwrap(),
+ // Python uses `pattern.match(line)` which is start-anchored,
+ // so this pattern only ever fires on lines literally starting
+ // with `:`. We anchor with `^` to keep parity (the `regex`
+ // crate's `is_match` is unanchored by default).
+ Regex::new(r"^:\s*(string|number|boolean|any|void)\b").unwrap(),
+ ],
+ },
+ CodePatterns {
+ name: "go",
+ patterns: vec![
+ Regex::new(r"^\s*(func|type|package|import)\s+").unwrap(),
+ Regex::new(r"^\s*func\s+\([^)]+\)\s+\w+").unwrap(),
+ ],
+ },
+ CodePatterns {
+ name: "rust",
+ patterns: vec![
+ Regex::new(r"^\s*(fn|struct|enum|impl|mod|use|pub)\s+").unwrap(),
+ Regex::new(r"^\s*#\[").unwrap(),
+ ],
+ },
+ CodePatterns {
+ name: "java",
+ patterns: vec![
+ Regex::new(r"^\s*(public|private|protected)\s+(class|interface|enum)").unwrap(),
+ Regex::new(r"^\s*@\w+").unwrap(),
+ Regex::new(r"^\s*package\s+[\w.]+;").unwrap(),
+ ],
+ },
+ ]
+});
+
+// ─── Log / build output patterns ───────────────────────────────────────
+//
+// Order matters: indices 0–1 (`ERROR` and `WARN` family) are treated as
+// "error" matches by `try_detect_log`, contributing extra to confidence.
+// Same ordering as Python.
+
+static LOG_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
+ vec![
+ Regex::new(r"(?i)\b(ERROR|FAIL|FAILED|FATAL|CRITICAL)\b").unwrap(),
+ Regex::new(r"(?i)\b(WARN|WARNING)\b").unwrap(),
+ Regex::new(r"(?i)\b(INFO|DEBUG|TRACE)\b").unwrap(),
+ Regex::new(r"^\s*\d{4}-\d{2}-\d{2}").unwrap(),
+ Regex::new(r"^\s*\[\d{2}:\d{2}:\d{2}\]").unwrap(),
+ Regex::new(r"^={3,}|^-{3,}").unwrap(),
+ Regex::new(r"^\s*PASSED|^\s*FAILED|^\s*SKIPPED").unwrap(),
+ Regex::new(r"^npm ERR!|^yarn error|^cargo error").unwrap(),
+ Regex::new(r"Traceback \(most recent call last\)").unwrap(),
+ Regex::new(r"^\s*at\s+[\w.$]+\(").unwrap(),
+ ]
+});
+
+// ─── HTML patterns ─────────────────────────────────────────────────────
+
+static HTML_DOCTYPE_PATTERN: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"(?i)^\s*<!doctype\s+html").unwrap());
+static HTML_TAG_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<html[\s>]").unwrap());
+static HTML_HEAD_PATTERN: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"(?i)<head[\s>]").unwrap());
+static HTML_BODY_PATTERN: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"(?i)<body[\s>]").unwrap());
+static HTML_STRUCTURAL_TAGS: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(
+ r"(?i)<(div|span|script|style|link|meta|nav|header|footer|aside|article|section|main)[\s>]",
+ )
+ .unwrap()
+});
+
+// ─── Public entry point ────────────────────────────────────────────────
+
+/// Detect the type of `content` for routing. Mirrors Python's
+/// `detect_content_type`.
+///
+/// Dispatch order (matches Python verbatim):
+/// 1. Empty / whitespace-only → `PlainText` confidence 0.0
+/// 2. JSON array (highest priority for `SmartCrusher`)
+/// 3. Git diff (≥ 0.7 confidence required)
+/// 4. HTML (≥ 0.7 confidence required)
+/// 5. Search results (≥ 0.6 confidence required)
+/// 6. Build / log output (≥ 0.5 confidence required)
+/// 7. Source code (≥ 0.5 confidence required)
+/// 8. Fallback to `PlainText` confidence 0.5
+pub fn detect_content_type(content: &str) -> DetectionResult {
+ if content.is_empty() || content.trim().is_empty() {
+ return DetectionResult::plain_text(0.0);
+ }
+
+ if let Some(r) = try_detect_json(content) {
+ return r;
+ }
+ if let Some(r) = try_detect_diff(content) {
+ if r.confidence >= 0.7 {
+ return r;
+ }
+ }
+ if let Some(r) = try_detect_html(content) {
+ if r.confidence >= 0.7 {
+ return r;
+ }
+ }
+ if let Some(r) = try_detect_search(content) {
+ if r.confidence >= 0.6 {
+ return r;
+ }
+ }
+ if let Some(r) = try_detect_log(content) {
+ if r.confidence >= 0.5 {
+ return r;
+ }
+ }
+ if let Some(r) = try_detect_code(content) {
+ if r.confidence >= 0.5 {
+ return r;
+ }
+ }
+ DetectionResult::plain_text(0.5)
+}
+
+/// Quick check: is `content` a JSON array of dictionaries (the format
+/// `SmartCrusher` natively handles)? Convenience wrapper around
+/// `detect_content_type`.
+pub fn is_json_array_of_dicts(content: &str) -> bool {
+ let result = detect_content_type(content);
+ if result.content_type != ContentType::JsonArray {
+ return false;
+ }
+ result
+ .metadata
+ .get("is_dict_array")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false)
+}
+
+// ─── Per-type detection helpers ────────────────────────────────────────
+
+fn try_detect_json(content: &str) -> Option<DetectionResult> {
+ let trimmed = content.trim();
+ if !trimmed.starts_with('[') {
+ return None;
+ }
+ let parsed: Value = serde_json::from_str(trimmed).ok()?;
+ let arr = parsed.as_array()?;
+ let item_count = arr.len();
+ let is_dict_array = !arr.is_empty() && arr.iter().all(|v| v.is_object());
+ let confidence = if is_dict_array { 1.0 } else { 0.8 };
+ Some(DetectionResult::new(
+ ContentType::JsonArray,
+ confidence,
+ json!({
+ "item_count": item_count,
+ "is_dict_array": is_dict_array,
+ })
+ .as_object()
+ .cloned()
+ .unwrap(),
+ ))
+}
+
+fn try_detect_diff(content: &str) -> Option<DetectionResult> {
+ // Window: 500 lines (extended from 50 in Python's 2026-04-25 fix).
+ let mut header_matches: u32 = 0;
+ let mut change_matches: u32 = 0;
+ for line in content.split('\n').take(500) {
+ if DIFF_HEADER_PATTERN.is_match(line) {
+ header_matches += 1;
+ }
+ if DIFF_CHANGE_PATTERN.is_match(line) {
+ change_matches += 1;
+ }
+ }
+ if header_matches == 0 {
+ return None;
+ }
+ // Same formula as Python: 0.5 + 0.2 * headers + 0.05 * changes, capped at 1.0
+ let confidence =
+ (0.5 + (header_matches as f64) * 0.2 + (change_matches as f64) * 0.05).min(1.0);
+ Some(DetectionResult::new(
+ ContentType::GitDiff,
+ confidence,
+ json!({
+ "header_matches": header_matches,
+ "change_lines": change_matches,
+ })
+ .as_object()
+ .cloned()
+ .unwrap(),
+ ))
+}
+
+fn try_detect_html(content: &str) -> Option<DetectionResult> {
+ // Sample first 3000 chars (byte-indexed; matches Python's str slice
+ // for ASCII inputs which is the common HTML case).
+ let sample: &str = if content.len() > 3000 {
+ // Find the last char-boundary <= 3000 so we don't slice mid-codepoint.
+ let mut cutoff = 3000;
+ while !content.is_char_boundary(cutoff) {
+ cutoff -= 1;
+ }
+ &content[..cutoff]
+ } else {
+ content
+ };
+
+ let has_doctype = HTML_DOCTYPE_PATTERN.is_match(sample);
+ let has_html_tag = HTML_TAG_PATTERN.is_match(sample);
+ let has_head = HTML_HEAD_PATTERN.is_match(sample);
+ let has_body = HTML_BODY_PATTERN.is_match(sample);
+ let structural_matches = HTML_STRUCTURAL_TAGS.find_iter(sample).count() as u32;
+
+ if !has_doctype && !has_html_tag && structural_matches < 3 {
+ return None;
+ }
+
+ let mut confidence = 0.0_f64;
+ if has_doctype {
+ confidence += 0.5;
+ }
+ if has_html_tag {
+ confidence += 0.3;
+ }
+ if has_head {
+ confidence += 0.1;
+ }
+ if has_body {
+ confidence += 0.1;
+ }
+ confidence += (structural_matches as f64 * 0.03).min(0.3);
+ confidence = confidence.min(1.0);
+
+ if confidence < 0.5 {
+ return None;
+ }
+ Some(DetectionResult::new(
+ ContentType::Html,
+ confidence,
+ json!({
+ "has_doctype": has_doctype,
+ "has_html_tag": has_html_tag,
+ "structural_tags": structural_matches,
+ })
+ .as_object()
+ .cloned()
+ .unwrap(),
+ ))
+}
+
+fn try_detect_search(content: &str) -> Option<DetectionResult> {
+ let lines: Vec<&str> = content.split('\n').take(100).collect();
+ if lines.is_empty() {
+ return None;
+ }
+ let mut matching_lines: u32 = 0;
+ for line in &lines {
+ if !line.trim().is_empty() && SEARCH_RESULT_PATTERN.is_match(line) {
+ matching_lines += 1;
+ }
+ }
+ if matching_lines == 0 {
+ return None;
+ }
+ let non_empty_lines = lines.iter().filter(|l| !l.trim().is_empty()).count() as u32;
+ if non_empty_lines == 0 {
+ return None;
+ }
+ let ratio = matching_lines as f64 / non_empty_lines as f64;
+ if ratio < 0.3 {
+ return None;
+ }
+ let confidence = (0.4 + ratio * 0.6).min(1.0);
+ Some(DetectionResult::new(
+ ContentType::SearchResults,
+ confidence,
+ json!({
+ "matching_lines": matching_lines,
+ "total_lines": non_empty_lines,
+ })
+ .as_object()
+ .cloned()
+ .unwrap(),
+ ))
+}
+
+fn try_detect_log(content: &str) -> Option<DetectionResult> {
+ let lines: Vec<&str> = content.split('\n').take(200).collect();
+ if lines.is_empty() {
+ return None;
+ }
+ let mut pattern_matches: u32 = 0;
+ let mut error_matches: u32 = 0;
+ for line in &lines {
+ for (i, pattern) in LOG_PATTERNS.iter().enumerate() {
+ if pattern.is_match(line) {
+ pattern_matches += 1;
+ if i < 2 {
+ error_matches += 1;
+ }
+ break; // one pattern per line is enough
+ }
+ }
+ }
+ if pattern_matches == 0 {
+ return None;
+ }
+ let non_empty_lines = lines.iter().filter(|l| !l.trim().is_empty()).count() as u32;
+ if non_empty_lines == 0 {
+ return None;
+ }
+ let ratio = pattern_matches as f64 / non_empty_lines as f64;
+ if ratio < 0.1 {
+ return None;
+ }
+ let confidence = (0.3 + ratio * 0.5 + (error_matches as f64) * 0.05).min(1.0);
+ Some(DetectionResult::new(
+ ContentType::BuildOutput,
+ confidence,
+ json!({
+ "pattern_matches": pattern_matches,
+ "error_matches": error_matches,
+ "total_lines": non_empty_lines,
+ })
+ .as_object()
+ .cloned()
+ .unwrap(),
+ ))
+}
+
+fn try_detect_code(content: &str) -> Option<DetectionResult> {
+ let lines: Vec<&str> = content.split('\n').take(100).collect();
+ if lines.is_empty() {
+ return None;
+ }
+ // Track scores in **first-match insertion order** to mirror Python's
+ // dict semantics. Python:
+ //
+ // language_scores: dict[str, int] = {}
+ // ...
+ // best_lang = max(language_scores, key=lambda k: language_scores[k])
+ //
+ // - Languages are inserted into the dict the first time they match a
+ // line, so the dict's iteration order is the order languages first
+ // showed up — NOT registration order.
+ // - `max(...)` returns the FIRST element with the maximum value when
+ // multiple keys tie, per the language spec.
+ //
+ // We replicate both with a Vec and a manual `find(score == max)` for
+ // the first-on-tie tie-break (Rust's `max_by` returns LAST on ties).
+ let mut language_scores: Vec<(&'static str, u32)> = Vec::new();
+
+ for line in &lines {
+ for cp in CODE_PATTERNS.iter() {
+ for pattern in &cp.patterns {
+ if pattern.is_match(line) {
+ if let Some(entry) = language_scores.iter_mut().find(|(n, _)| *n == cp.name) {
+ entry.1 += 1;
+ } else {
+ language_scores.push((cp.name, 1));
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ if language_scores.is_empty() {
+ return None;
+ }
+ let max_score = language_scores.iter().map(|x| x.1).max().unwrap_or(0);
+ let (best_lang, best_score) = *language_scores
+ .iter()
+ .find(|x| x.1 == max_score)
+ .expect("language_scores non-empty");
+ if best_score < 3 {
+ return None;
+ }
+ let non_empty_lines = lines.iter().filter(|l| !l.trim().is_empty()).count() as u32;
+ let ratio = best_score as f64 / non_empty_lines.max(1) as f64;
+ let confidence = (0.4 + ratio * 0.4 + (best_score as f64) * 0.02).min(1.0);
+ Some(DetectionResult::new(
+ ContentType::SourceCode,
+ confidence,
+ json!({
+ "language": best_lang,
+ "pattern_matches": best_score,
+ })
+ .as_object()
+ .cloned()
+ .unwrap(),
+ ))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn empty_returns_plain_text_zero_confidence() {
+ let r = detect_content_type("");
+ assert_eq!(r.content_type, ContentType::PlainText);
+ assert_eq!(r.confidence, 0.0);
+ }
+
+ #[test]
+ fn whitespace_only_returns_plain_text_zero_confidence() {
+ let r = detect_content_type(" \n\t ");
+ assert_eq!(r.content_type, ContentType::PlainText);
+ assert_eq!(r.confidence, 0.0);
+ }
+
+ #[test]
+ fn json_array_of_dicts_high_confidence() {
+ let r = detect_content_type(r#"[{"id": 1}, {"id": 2}]"#);
+ assert_eq!(r.content_type, ContentType::JsonArray);
+ assert_eq!(r.confidence, 1.0);
+ assert_eq!(
+ r.metadata.get("is_dict_array").unwrap().as_bool(),
+ Some(true)
+ );
+ assert_eq!(r.metadata.get("item_count").unwrap().as_u64(), Some(2));
+ }
+
+ #[test]
+ fn json_array_of_scalars_lower_confidence() {
+ let r = detect_content_type(r#"[1, 2, 3]"#);
+ assert_eq!(r.content_type, ContentType::JsonArray);
+ assert_eq!(r.confidence, 0.8);
+ assert_eq!(
+ r.metadata.get("is_dict_array").unwrap().as_bool(),
+ Some(false)
+ );
+ }
+
+ #[test]
+ fn empty_json_array_not_dict_array() {
+ let r = detect_content_type("[]");
+ assert_eq!(r.content_type, ContentType::JsonArray);
+ assert_eq!(r.confidence, 0.8);
+ assert_eq!(
+ r.metadata.get("is_dict_array").unwrap().as_bool(),
+ Some(false)
+ );
+ }
+
+ #[test]
+ fn json_object_falls_through_to_text() {
+ // Detector only handles arrays.
+ let r = detect_content_type(r#"{"id": 1}"#);
+ assert_eq!(r.content_type, ContentType::PlainText);
+ }
+
+ #[test]
+ fn search_results_detected() {
+ let content =
+ "src/main.py:42:def process():\nsrc/util.py:13: return None\nlib/x.py:7:class X:";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::SearchResults);
+ assert!(r.confidence >= 0.6);
+ }
+
+ #[test]
+ fn git_diff_detected() {
+ let content = "\
+diff --git a/foo.py b/foo.py
+--- a/foo.py
++++ b/foo.py
+@@ -1,3 +1,4 @@
+ def hello():
+- print('hi')
++ print('hello')
++ print('world')
+";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::GitDiff);
+ assert!(r.confidence >= 0.7);
+ }
+
+ #[test]
+ fn html_doctype_detected() {
+ let content = "\
+<!DOCTYPE html>
+<html>
+<head><title>X</title></head>
+<body><div>hi</div></body>
+</html>";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::Html);
+ assert!(r.confidence >= 0.7);
+ }
+
+ #[test]
+ fn build_output_detected() {
+ let content = "\
+[INFO] Starting build
+[INFO] Compiling 42 sources
+[ERROR] Compilation failed
+[WARN] Deprecated API
+FAILED test_one
+PASSED test_two
+";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::BuildOutput);
+ assert!(r.confidence >= 0.5);
+ }
+
+ #[test]
+ fn python_code_detected() {
+ let content = "\
+import os
+from typing import Any
+
+def process(data):
+ return data
+
+class Service:
+ def __init__(self):
+ pass
+
+ @property
+ def x(self):
+ return 1
+
+if __name__ == '__main__':
+ process({})
+";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::SourceCode);
+ assert_eq!(r.metadata.get("language").unwrap().as_str(), Some("python"));
+ }
+
+ #[test]
+ fn rust_code_detected() {
+ let content = "\
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub struct Foo {
+ bar: u32,
+}
+
+pub fn baz() -> u32 {
+ 42
+}
+
+impl Foo {
+ pub fn new() -> Self {
+ Self { bar: 0 }
+ }
+}
+";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::SourceCode);
+ assert_eq!(r.metadata.get("language").unwrap().as_str(), Some("rust"));
+ }
+
+ #[test]
+ fn go_code_detected() {
+ let content = "\
+package main
+
+import \"fmt\"
+
+func main() {
+ fmt.Println(\"hello\")
+}
+
+type Service struct{}
+
+func (s *Service) Do() {}
+
+func helper() {}
+";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::SourceCode);
+ assert_eq!(r.metadata.get("language").unwrap().as_str(), Some("go"));
+ }
+
+ #[test]
+ fn fallback_to_plain_text() {
+ let content = "Just some random text without any special structure.";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::PlainText);
+ assert_eq!(r.confidence, 0.5);
+ }
+
+ #[test]
+ fn is_json_array_of_dicts_true_path() {
+ assert!(is_json_array_of_dicts(r#"[{"a": 1}, {"a": 2}]"#));
+ }
+
+ #[test]
+ fn is_json_array_of_dicts_scalars_returns_false() {
+ assert!(!is_json_array_of_dicts(r#"[1, 2, 3]"#));
+ }
+
+ #[test]
+ fn is_json_array_of_dicts_object_returns_false() {
+ assert!(!is_json_array_of_dicts(r#"{"a": 1}"#));
+ }
+
+ #[test]
+ fn is_json_array_of_dicts_empty_returns_false() {
+ // Empty array is JsonArray but not is_dict_array.
+ assert!(!is_json_array_of_dicts("[]"));
+ }
+
+ #[test]
+ fn diff_low_confidence_does_not_short_circuit() {
+ // Single header with no change lines yields 0.7 — borderline.
+ // Should still register as diff (>= 0.7 threshold).
+ let content = "diff --git a/x b/x\n";
+ let r = detect_content_type(content);
+ assert_eq!(r.content_type, ContentType::GitDiff);
+ }
+
+ #[test]
+ fn html_below_threshold_falls_through() {
+ // Just one structural tag — not enough.
+ let r = detect_content_type("<div>hello</div>");
+ assert_ne!(r.content_type, ContentType::Html);
+ }
+
+ #[test]
+ fn content_type_string_tags_match_python() {
+ assert_eq!(ContentType::JsonArray.as_str(), "json_array");
+ assert_eq!(ContentType::SourceCode.as_str(), "source_code");
+ assert_eq!(ContentType::SearchResults.as_str(), "search");
+ assert_eq!(ContentType::BuildOutput.as_str(), "build");
+ assert_eq!(ContentType::GitDiff.as_str(), "diff");
+ assert_eq!(ContentType::Html.as_str(), "html");
+ assert_eq!(ContentType::PlainText.as_str(), "text");
+ }
+}
diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rs
index d3c7eeaf2..1596fab20 100644
--- a/crates/headroom-core/src/transforms/mod.rs
+++ b/crates/headroom-core/src/transforms/mod.rs
@@ -17,9 +17,13 @@
pub mod adaptive_sizer;
pub mod anchor_selector;
+pub mod content_detector;
pub mod diff_compressor;
pub mod smart_crusher;
+pub use content_detector::{
+ detect_content_type, is_json_array_of_dicts, ContentType, DetectionResult,
+};
pub use diff_compressor::{
DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
};
diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs
index 189635b31..55a2ae12a 100644
--- a/crates/headroom-parity/src/lib.rs
+++ b/crates/headroom-parity/src/lib.rs
@@ -45,6 +45,15 @@ pub trait TransformComparator {
}
/// Compare a single fixture against a comparator and return an outcome.
+///
+/// f64 normalization: `serde_json` (without the `arbitrary_precision`
+/// feature) has an asymmetry — values constructed via `json!(f64)` keep
+/// full precision (e.g. `0.9500000000000001`), but values parsed from
+/// fixture JSON sometimes round to a neighboring f64 (e.g. `0.95`,
+/// differing by 1 ULP). To make comparisons robust we round-trip the
+/// comparator's output through `to_string` + `from_str` so it goes
+/// through the same lossy parser the fixture did. Bit-identical f64s
+/// from both sides then compare equal.
pub fn compare_fixture(
comparator: &dyn TransformComparator,
fixture: &Fixture,
@@ -57,12 +66,15 @@ pub fn compare_fixture(
})
}
};
- if actual == fixture.output {
+ let actual_normalized: serde_json::Value =
+ serde_json::from_str(&serde_json::to_string(&actual)?)
+ .context("re-parsing comparator output through serde_json (f64 normalization)")?;
+ if actual_normalized == fixture.output {
Ok(ComparisonOutcome::Match)
} else {
Ok(ComparisonOutcome::Diff {
expected: serde_json::to_string_pretty(&fixture.output)?,
- actual: serde_json::to_string_pretty(&actual)?,
+ actual: serde_json::to_string_pretty(&actual_normalized)?,
})
}
}
@@ -401,6 +413,48 @@ impl TransformComparator for SmartCrusherComparator {
}
}
+/// Real comparator for the `content_detector` transform. Drives the Rust
+/// port over the recorded fixture inputs (a single JSON string) and
+/// emits the same shape Python's recorder serializes for
+/// `DetectionResult`:
+///
+/// ```json
+/// {"content_type": "json_array", "confidence": 1.0, "metadata": {...}}
+/// ```
+///
+/// Python's recorder relies on `_json_default` to serialize the
+/// `DetectionResult` dataclass and the `ContentType` enum:
+/// - dataclass → `asdict(...)` produces `{content_type, confidence, metadata}`.
+/// - enum → its `.value` (the lowercase tag, e.g. "json_array").
+///
+/// Numeric fields in metadata are recorded as JSON numbers (Python ints
+/// stay ints), so we mirror that exactly with `serde_json::Number`.
+pub struct ContentDetectorComparator;
+
+impl TransformComparator for ContentDetectorComparator {
+ fn name(&self) -> &str {
+ "content_detector"
+ }
+
+ fn run(
+ &self,
+ input: &serde_json::Value,
+ _config: &serde_json::Value,
+ ) -> Result<serde_json::Value> {
+ use headroom_core::transforms::detect_content_type;
+
+ let content = input
+ .as_str()
+ .context("content_detector fixture input must be a JSON string")?;
+ let result = detect_content_type(content);
+ Ok(serde_json::json!({
+ "content_type": result.content_type.as_str(),
+ "confidence": result.confidence,
+ "metadata": serde_json::Value::Object(result.metadata),
+ }))
+ }
+}
+
/// Every built-in comparator, in a stable order.
pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {
vec![
@@ -410,6 +464,7 @@ pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {
Box::new(TokenizerComparator),
Box::new(CcrComparator),
Box::new(SmartCrusherComparator),
+ Box::new(ContentDetectorComparator),
]
}
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index b5b0eb36a..d0f837985 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -21,10 +21,13 @@ use headroom_core::transforms::smart_crusher::{
SmartCrusherConfig as RustSmartCrusherConfig,
};
use headroom_core::transforms::{
- DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
+ detect_content_type as rust_detect_content_type,
+ is_json_array_of_dicts as rust_is_json_array_of_dicts, ContentType as RustContentType,
+ DetectionResult as RustDetectionResult, DiffCompressionResult, DiffCompressor,
+ DiffCompressorConfig, DiffCompressorStats,
};
use pyo3::prelude::*;
-use pyo3::types::PyDict;
+use pyo3::types::{PyDict, PyString};
/// Identity stub used by the Python smoke test to verify linkage.
#[pyfunction]
@@ -753,6 +756,109 @@ impl PySmartCrusher {
}
}
+// ─── ContentDetector ───────────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.content_detector.DetectionResult`.
+///
+/// Field names + types match the Python dataclass exactly so the existing
+/// Python `ContentRouter` (which `import`s `DetectionResult` directly) can
+/// continue to read `.content_type`, `.confidence`, and `.metadata` without
+/// modification.
+///
+/// `content_type` is exposed as the lowercase string tag (e.g.
+/// `"json_array"`). The Python wrapper translates it back into the
+/// `ContentType` enum so the call-site looks identical.
+#[pyclass(name = "DetectionResult", module = "headroom._core")]
+#[derive(Clone)]
+struct PyDetectionResult {
+ inner: RustDetectionResult,
+}
+
+#[pymethods]
+impl PyDetectionResult {
+ #[getter]
+ fn content_type(&self) -> &'static str {
+ self.inner.content_type.as_str()
+ }
+
+ #[getter]
+ fn confidence(&self) -> f64 {
+ self.inner.confidence
+ }
+
+ /// Per-type metadata bag (e.g. `{"language": "python", "pattern_matches": 5}`
+ /// for code, `{"item_count": 3, "is_dict_array": true}` for JSON arrays).
+ /// Returned as a fresh `dict` so callers can mutate without affecting
+ /// the underlying Rust value.
+ #[getter]
+ fn metadata<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
+ let dict = PyDict::new_bound(py);
+ for (k, v) in &self.inner.metadata {
+ // Convert each JSON value into the closest Python primitive.
+ // Detection metadata is always a flat dict of scalars (ints,
+ // bools, strings) so we don't need to recurse.
+ let py_value: PyObject = match v {
+ serde_json::Value::Bool(b) => b.into_py(py),
+ serde_json::Value::Number(n) => {
+ if let Some(i) = n.as_u64() {
+ i.into_py(py)
+ } else if let Some(i) = n.as_i64() {
+ i.into_py(py)
+ } else if let Some(f) = n.as_f64() {
+ f.into_py(py)
+ } else {
+ py.None()
+ }
+ }
+ serde_json::Value::String(s) => PyString::new_bound(py, s).into_py(py),
+ serde_json::Value::Null => py.None(),
+ // Detection never emits arrays / objects in metadata
+ // today; if it ever does, fall through to JSON-string for
+ // visibility rather than silently dropping.
+ other => PyString::new_bound(py, &other.to_string()).into_py(py),
+ };
+ dict.set_item(k, py_value)?;
+ }
+ Ok(dict)
+ }
+
+ fn __repr__(&self) -> String {
+ format!(
+ "DetectionResult(content_type={:?}, confidence={}, metadata=<{} keys>)",
+ self.inner.content_type.as_str(),
+ self.inner.confidence,
+ self.inner.metadata.len()
+ )
+ }
+}
+
+/// Detect the type of `content`. Returns a `DetectionResult` with the
+/// same field surface as Python's dataclass.
+///
+/// Releases the GIL while detecting — pattern matching can be substantial
+/// on large bodies (HTML scans, 500-line diff windows), and freeing the
+/// GIL lets other Python threads make progress in the meantime.
+#[pyfunction]
+fn detect_content_type(py: Python<'_>, content: &str) -> PyDetectionResult {
+ let owned = content.to_string();
+ let result = py.allow_threads(move || rust_detect_content_type(&owned));
+ PyDetectionResult { inner: result }
+}
+
+/// Quick check: is `content` a JSON array of dictionaries (the format
+/// `SmartCrusher` natively handles)?
+#[pyfunction]
+fn is_json_array_of_dicts(py: Python<'_>, content: &str) -> bool {
+ let owned = content.to_string();
+ py.allow_threads(move || rust_is_json_array_of_dicts(&owned))
+}
+
+// Suppress unused-import warning when ContentType isn't referenced
+// directly — `as_str()` is the public surface.
+const _: fn() = || {
+ let _ = RustContentType::PlainText;
+};
+
// ─── Module init ───────────────────────────────────────────────────────────
#[pymodule]
@@ -765,5 +871,8 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PySmartCrusherConfig>()?;
m.add_class::<PyCrushResult>()?;
m.add_class::<PySmartCrusher>()?;
+ m.add_class::<PyDetectionResult>()?;
+ m.add_function(wrap_pyfunction!(detect_content_type, m)?)?;
+ m.add_function(wrap_pyfunction!(is_json_array_of_dicts, m)?)?;
Ok(())
}
diff --git a/tests/parity/fixtures/content_detector/247811aecfdec556.json b/tests/parity/fixtures/content_detector/247811aecfdec556.json
new file mode 100644
index 000000000..87bc23d75
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/247811aecfdec556.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "diff --combined merged.py\nindex aaa..bbb..ccc 100644\n--- a/merged.py\n+++ b/merged.py\n@@@ -1,4 -1,4 +1,5 @@@\n unchanged\n- branch_a_only\n -branch_b_only\n++merge_added\n",
+ "input_sha256": "247811aecfdec55647a37c088c611f8f093119d95335da3d10aa317455159655",
+ "output": {
+ "confidence": 1.0,
+ "content_type": "diff",
+ "metadata": {
+ "change_lines": 1,
+ "header_matches": 3
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.959992+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/37c86907c057e293.json b/tests/parity/fixtures/content_detector/37c86907c057e293.json
new file mode 100644
index 000000000..aa635897b
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/37c86907c057e293.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "package com.example;\n\npublic class Foo {\n @Override\n public String toString() { return \"foo\"; }\n}\n\nprivate interface Bar {}\nprotected enum Baz { A, B }\n",
+ "input_sha256": "37c86907c057e293385ffaef6c4278a4b903fca82e3d52fb42df601f47010a3d",
+ "output": {
+ "confidence": 0.7857142857142857,
+ "content_type": "source_code",
+ "metadata": {
+ "language": "java",
+ "pattern_matches": 5
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.962175+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/3a0b14d2dea7c876.json b/tests/parity/fixtures/content_detector/3a0b14d2dea7c876.json
new file mode 100644
index 000000000..27d4837d1
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/3a0b14d2dea7c876.json
@@ -0,0 +1,12 @@
+{
+ "config": {},
+ "input": "Just some prose text without any structure or special markers.",
+ "input_sha256": "3a0b14d2dea7c87688fc40482a02f006accfae1da50f0734a1e395fed13fc744",
+ "output": {
+ "confidence": 0.5,
+ "content_type": "text",
+ "metadata": {}
+ },
+ "recorded_at": "2026-04-28T06:39:34.962324+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/3ba74a1f57c5351e.json b/tests/parity/fixtures/content_detector/3ba74a1f57c5351e.json
new file mode 100644
index 000000000..1698bbf0f
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/3ba74a1f57c5351e.json
@@ -0,0 +1,12 @@
+{
+ "config": {},
+ "input": "<div>a</div>\n<span>b</span>\n<script>x()</script>\n<style>y</style>",
+ "input_sha256": "3ba74a1f57c5351ecb47efcfff73080bc3f96a75589b459e20424fe0c715be39",
+ "output": {
+ "confidence": 0.5,
+ "content_type": "text",
+ "metadata": {}
+ },
+ "recorded_at": "2026-04-28T06:39:34.960293+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/4efe946303371867.json b/tests/parity/fixtures/content_detector/4efe946303371867.json
new file mode 100644
index 000000000..28755b045
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/4efe946303371867.json
@@ -0,0 +1,12 @@
+{
+ "config": {},
+ "input": "",
+ "input_sha256": "4efe946303371867389009a0144a8ea235bfe9f6e87a9395b0c0453b8e3f99c0",
+ "output": {
+ "confidence": 0.0,
+ "content_type": "text",
+ "metadata": {}
+ },
+ "recorded_at": "2026-04-28T06:39:34.962454+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/53400da2b7dd2428.json b/tests/parity/fixtures/content_detector/53400da2b7dd2428.json
new file mode 100644
index 000000000..8e3685e48
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/53400da2b7dd2428.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "import x from 'y';\nexport const foo = 1;\nfunction bar() { return 42; }\nconst f = async function() {};\nmodule.exports = { foo, bar };\n",
+ "input_sha256": "53400da2b7dd2428c8146088e60779a8d61c2430a2c7c95a6d2b07e83b9b726e",
+ "output": {
+ "confidence": 0.9,
+ "content_type": "source_code",
+ "metadata": {
+ "language": "javascript",
+ "pattern_matches": 5
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.961426+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/552d30c30c2a6793.json b/tests/parity/fixtures/content_detector/552d30c30c2a6793.json
new file mode 100644
index 000000000..15b6e815f
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/552d30c30c2a6793.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "diff --git a/foo.py b/foo.py\nindex abc..def 100644\n--- a/foo.py\n+++ b/foo.py\n@@ -1,3 +1,3 @@\n-old line\n+new line\n unchanged\n",
+ "input_sha256": "552d30c30c2a67939857c22f85696406f1557476912b41d9918031a15364163a",
+ "output": {
+ "confidence": 1.0,
+ "content_type": "diff",
+ "metadata": {
+ "change_lines": 2,
+ "header_matches": 3
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.959842+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/5e8e1bc29f6c71c5.json b/tests/parity/fixtures/content_detector/5e8e1bc29f6c71c5.json
new file mode 100644
index 000000000..0c07f0227
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/5e8e1bc29f6c71c5.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "import os\nfrom typing import Any\n\n@dataclass\nclass Foo:\n \"\"\"Docstring.\"\"\"\n def bar(self):\n return 42\n\ndef baz():\n pass\n\nif __name__ == '__main__':\n baz()\n",
+ "input_sha256": "5e8e1bc29f6c71c5b6523dc65f91a615b15e8bd27d633cbcd54293bbe43b6951",
+ "output": {
+ "confidence": 0.850909090909091,
+ "content_type": "source_code",
+ "metadata": {
+ "language": "python",
+ "pattern_matches": 8
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.961138+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/628c8985c40adb7d.json b/tests/parity/fixtures/content_detector/628c8985c40adb7d.json
new file mode 100644
index 000000000..484bb4760
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/628c8985c40adb7d.json
@@ -0,0 +1,16 @@
+{
+ "config": {},
+ "input": "INFO starting build\nWARN deprecated API used\nERROR compilation failed\nFAILED test_x\nPASSED test_y\n============================================\n",
+ "input_sha256": "628c8985c40adb7d265eac37436ea45a549ba9ed2b10eb47a1fa7b478ce17e77",
+ "output": {
+ "confidence": 0.9500000000000001,
+ "content_type": "build",
+ "metadata": {
+ "error_matches": 3,
+ "pattern_matches": 6,
+ "total_lines": 6
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.960715+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/67f4b1a1697c8aa9.json b/tests/parity/fixtures/content_detector/67f4b1a1697c8aa9.json
new file mode 100644
index 000000000..9b6063dc9
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/67f4b1a1697c8aa9.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "interface User { id: number; name: string; }\ntype Maybe<T> = T | null;\nenum Color { Red, Green, Blue }\nfunction f(x: number): boolean { return x > 0; }\n",
+ "input_sha256": "67f4b1a1697c8aa916e4acb0920a92279bbc35efa3849edc9bc2aa8dcff73fb3",
+ "output": {
+ "confidence": 0.76,
+ "content_type": "source_code",
+ "metadata": {
+ "language": "typescript",
+ "pattern_matches": 3
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.961598+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/70fbb444a583fe3c.json b/tests/parity/fixtures/content_detector/70fbb444a583fe3c.json
new file mode 100644
index 000000000..6bd99fe91
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/70fbb444a583fe3c.json
@@ -0,0 +1,12 @@
+{
+ "config": {},
+ "input": " \n\t \n",
+ "input_sha256": "70fbb444a583fe3ce3d9d5fd6783e64a5261afa0a0f7b25b0a48ec224d255534",
+ "output": {
+ "confidence": 0.0,
+ "content_type": "text",
+ "metadata": {}
+ },
+ "recorded_at": "2026-04-28T06:39:34.962581+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/71d648a39e151a72.json b/tests/parity/fixtures/content_detector/71d648a39e151a72.json
new file mode 100644
index 000000000..206665acf
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/71d648a39e151a72.json
@@ -0,0 +1,16 @@
+{
+ "config": {},
+ "input": "============================= test session starts ==============================\ntests/test_0.py::test_case PASSED\ntests/test_1.py::test_case PASSED\ntests/test_2.py::test_case PASSED\ntests/test_3.py::test_case PASSED\ntests/test_4.py::test_case PASSED\nFAILED tests/test_5.py::test_bad\nTraceback (most recent call last)\n",
+ "input_sha256": "71d648a39e151a72411fe014b2e0820abc922609752d74798d26bf6c99fd2168",
+ "output": {
+ "confidence": 0.5375,
+ "content_type": "build",
+ "metadata": {
+ "error_matches": 1,
+ "pattern_matches": 3,
+ "total_lines": 8
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.960922+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/79e6b68a05a1c639.json b/tests/parity/fixtures/content_detector/79e6b68a05a1c639.json
new file mode 100644
index 000000000..321ac8808
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/79e6b68a05a1c639.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "src/main.py:42:def process():\nsrc/util.py:13: return None\nlib/x.py:7:class X:\ntests/test_a.py:3: assert True",
+ "input_sha256": "79e6b68a05a1c639efba2fe2f3bf8471ea00a04e9dc0a5fb8853cbc94ac6b42a",
+ "output": {
+ "confidence": 1.0,
+ "content_type": "search",
+ "metadata": {
+ "matching_lines": 4,
+ "total_lines": 4
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.960560+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/7e145153192c11a1.json b/tests/parity/fixtures/content_detector/7e145153192c11a1.json
new file mode 100644
index 000000000..c62c9f946
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/7e145153192c11a1.json
@@ -0,0 +1,12 @@
+{
+ "config": {},
+ "input": "<p>just one tag</p>",
+ "input_sha256": "7e145153192c11a1b2ffc574059deaf3990e1cb6c83f85b6c5ed0a2bfdb93ab5",
+ "output": {
+ "confidence": 0.5,
+ "content_type": "text",
+ "metadata": {}
+ },
+ "recorded_at": "2026-04-28T06:39:34.960430+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/7fde660489bc47aa.json b/tests/parity/fixtures/content_detector/7fde660489bc47aa.json
new file mode 100644
index 000000000..ba15f5c9e
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/7fde660489bc47aa.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "[]",
+ "input_sha256": "7fde660489bc47aa12bccf43567d6c386c36670ca5185e8a5f8781ac257f7c7d",
+ "output": {
+ "confidence": 0.8,
+ "content_type": "json_array",
+ "metadata": {
+ "is_dict_array": false,
+ "item_count": 0
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.959529+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/837b272f1604eaaa.json b/tests/parity/fixtures/content_detector/837b272f1604eaaa.json
new file mode 100644
index 000000000..435369ae4
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/837b272f1604eaaa.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "package main\n\nimport \"fmt\"\n\ntype Foo struct { ID int }\n\nfunc (f *Foo) Bar() int { return f.ID }\n\nfunc main() { fmt.Println(\"hi\") }\n",
+ "input_sha256": "837b272f1604eaaadee6f9d33bff0b00d85204cc3ce8ae9c064964d8a432a611",
+ "output": {
+ "confidence": 0.9,
+ "content_type": "source_code",
+ "metadata": {
+ "language": "go",
+ "pattern_matches": 5
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.961787+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/9fb76321d31ed64c.json b/tests/parity/fixtures/content_detector/9fb76321d31ed64c.json
new file mode 100644
index 000000000..3590bdcb3
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/9fb76321d31ed64c.json
@@ -0,0 +1,12 @@
+{
+ "config": {},
+ "input": "{\"foo\": \"bar\"}",
+ "input_sha256": "9fb76321d31ed64c9b59daa7456102070d25ea02b8a414049a9b93a1b50751f7",
+ "output": {
+ "confidence": 0.5,
+ "content_type": "text",
+ "metadata": {}
+ },
+ "recorded_at": "2026-04-28T06:39:34.959686+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/bc4488ce189d1c86.json b/tests/parity/fixtures/content_detector/bc4488ce189d1c86.json
new file mode 100644
index 000000000..8d313c3e7
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/bc4488ce189d1c86.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "[{\"id\":1,\"name\":\"a\"},{\"id\":2,\"name\":\"b\"},{\"id\":3,\"name\":\"c\"}]",
+ "input_sha256": "bc4488ce189d1c867e5b600d4691354f94a3f63ecf869ad2bc89344cb2142566",
+ "output": {
+ "confidence": 1.0,
+ "content_type": "json_array",
+ "metadata": {
+ "is_dict_array": true,
+ "item_count": 3
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.959161+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/bf9b9ceb4b70dcd0.json b/tests/parity/fixtures/content_detector/bf9b9ceb4b70dcd0.json
new file mode 100644
index 000000000..e39f1da8c
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/bf9b9ceb4b70dcd0.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "use std::collections::HashMap;\n\npub struct Foo { id: u32 }\n\nimpl Foo {\n pub fn new() -> Self { Self { id: 0 } }\n}\n\nfn main() {}\n#[derive(Debug)]\nenum Color { Red, Green }\n",
+ "input_sha256": "bf9b9ceb4b70dcd0a291c57ff0a53690e575f6d2d4273eb9d2a73b7f9beb29db",
+ "output": {
+ "confidence": 0.89,
+ "content_type": "source_code",
+ "metadata": {
+ "language": "rust",
+ "pattern_matches": 7
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.961975+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/cabc55445af63e2f.json b/tests/parity/fixtures/content_detector/cabc55445af63e2f.json
new file mode 100644
index 000000000..fcc81d7e9
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/cabc55445af63e2f.json
@@ -0,0 +1,15 @@
+{
+ "config": {},
+ "input": "[1, 2, 3, 4, 5]",
+ "input_sha256": "cabc55445af63e2fa6667ee023240b19884644bd72ef99c7a0f232d8fecd674f",
+ "output": {
+ "confidence": 0.8,
+ "content_type": "json_array",
+ "metadata": {
+ "is_dict_array": false,
+ "item_count": 5
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.959327+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/fixtures/content_detector/d100a9ac7a3671da.json b/tests/parity/fixtures/content_detector/d100a9ac7a3671da.json
new file mode 100644
index 000000000..aef1901bb
--- /dev/null
+++ b/tests/parity/fixtures/content_detector/d100a9ac7a3671da.json
@@ -0,0 +1,16 @@
+{
+ "config": {},
+ "input": "<!DOCTYPE html>\n<html>\n<head><title>x</title></head>\n<body><div><span>hi</span></div></body>\n</html>",
+ "input_sha256": "d100a9ac7a3671dafbc6666243ba796c0985281868d47921bd1f75506b4d81ce",
+ "output": {
+ "confidence": 1.0,
+ "content_type": "html",
+ "metadata": {
+ "has_doctype": true,
+ "has_html_tag": true,
+ "structural_tags": 2
+ }
+ },
+ "recorded_at": "2026-04-28T06:39:34.960132+00:00",
+ "transform": "content_detector"
+}
diff --git a/tests/parity/recorder.py b/tests/parity/recorder.py
index b231bdf97..5707586a6 100644
--- a/tests/parity/recorder.py
+++ b/tests/parity/recorder.py
@@ -264,6 +264,22 @@ def record_all(root: Path | None = None) -> dict[str, str]:
except Exception as e:
statuses["ccr"] = f"blocked:{e.__class__.__name__}:{e}"
+ # --- content_detector --------------------------------------------------
+ # `detect_content_type` is a module-level function, so we monkey-patch
+ # the module attribute rather than a class method.
+ try:
+ from headroom.transforms import content_detector as _cd_mod
+
+ _wrap_function(
+ _cd_mod,
+ "detect_content_type",
+ "content_detector",
+ root=root,
+ )
+ statuses["content_detector"] = "patched"
+ except Exception as e:
+ statuses["content_detector"] = f"blocked:{e.__class__.__name__}:{e}"
+
return statuses
@@ -291,6 +307,32 @@ def _wrap_method(
setattr(cls, method_name, wrapped)
+def _wrap_function(
+ module: Any,
+ fn_name: str,
+ transform_name: str,
+ *,
+ root: Path | None = None,
+ input_arg: int = 0,
+ input_kw: str | None = None,
+) -> None:
+ """Monkey-patch a module-level free function the same way `_wrap_method`
+ handles class methods. Idempotent; safe to call repeatedly."""
+ original = getattr(module, fn_name)
+ if getattr(original, "_parity_recorder_wrapped", False):
+ return # idempotent
+
+ decorator = record(
+ transform_name,
+ root=root,
+ input_arg=input_arg,
+ input_kw=input_kw,
+ )
+ wrapped = decorator(original)
+ wrapped._parity_recorder_wrapped = True # type: ignore[attr-defined]
+ setattr(module, fn_name, wrapped)
+
+
# ---------------------------------------------------------------------------
# Minimal workload helpers — callable independently of the scripts entry.
# ---------------------------------------------------------------------------
@@ -541,6 +583,147 @@ def _varied_text_inputs() -> list[str]:
return out[:20]
+def _varied_content_detector_inputs() -> list[str]:
+ """Hit every dispatch branch in `detect_content_type`. Each entry below
+ targets a specific path so parity diffs surface the right branch."""
+ json_array_dicts = '[{"id":1,"name":"a"},{"id":2,"name":"b"},{"id":3,"name":"c"}]'
+ json_array_scalars = "[1, 2, 3, 4, 5]"
+ json_empty_array = "[]"
+ json_object = '{"foo": "bar"}'
+
+ git_diff = (
+ "diff --git a/foo.py b/foo.py\n"
+ "index abc..def 100644\n"
+ "--- a/foo.py\n"
+ "+++ b/foo.py\n"
+ "@@ -1,3 +1,3 @@\n"
+ "-old line\n"
+ "+new line\n"
+ " unchanged\n"
+ )
+ merge_diff = (
+ "diff --combined merged.py\n"
+ "index aaa..bbb..ccc 100644\n"
+ "--- a/merged.py\n"
+ "+++ b/merged.py\n"
+ "@@@ -1,4 -1,4 +1,5 @@@\n"
+ " unchanged\n"
+ "- branch_a_only\n"
+ " -branch_b_only\n"
+ "++merge_added\n"
+ )
+
+ html_doctype = (
+ "<!DOCTYPE html>\n<html>\n<head><title>x</title></head>\n"
+ "<body><div><span>hi</span></div></body>\n</html>"
+ )
+ html_structural = "<div>a</div>\n<span>b</span>\n<script>x()</script>\n<style>y</style>"
+ html_below_threshold = "<p>just one tag</p>"
+
+ search_results = (
+ "src/main.py:42:def process():\n"
+ "src/util.py:13: return None\n"
+ "lib/x.py:7:class X:\n"
+ "tests/test_a.py:3: assert True"
+ )
+
+ build_log = (
+ "INFO starting build\n"
+ "WARN deprecated API used\n"
+ "ERROR compilation failed\n"
+ "FAILED test_x\n"
+ "PASSED test_y\n"
+ "============================================\n"
+ )
+ pytest_output = (
+ "============================= test session starts ==============================\n"
+ + "\n".join(f"tests/test_{i}.py::test_case PASSED" for i in range(5))
+ + "\nFAILED tests/test_5.py::test_bad\n"
+ + "Traceback (most recent call last)\n"
+ )
+
+ python_code = (
+ "import os\n"
+ "from typing import Any\n\n"
+ "@dataclass\n"
+ "class Foo:\n"
+ ' """Docstring."""\n'
+ " def bar(self):\n"
+ " return 42\n\n"
+ "def baz():\n"
+ " pass\n\n"
+ "if __name__ == '__main__':\n"
+ " baz()\n"
+ )
+ js_code = (
+ "import x from 'y';\n"
+ "export const foo = 1;\n"
+ "function bar() { return 42; }\n"
+ "const f = async function() {};\n"
+ "module.exports = { foo, bar };\n"
+ )
+ ts_code = (
+ "interface User { id: number; name: string; }\n"
+ "type Maybe<T> = T | null;\n"
+ "enum Color { Red, Green, Blue }\n"
+ "function f(x: number): boolean { return x > 0; }\n"
+ )
+ go_code = (
+ "package main\n\n"
+ 'import "fmt"\n\n'
+ "type Foo struct { ID int }\n\n"
+ "func (f *Foo) Bar() int { return f.ID }\n\n"
+ 'func main() { fmt.Println("hi") }\n'
+ )
+ rust_code = (
+ "use std::collections::HashMap;\n\n"
+ "pub struct Foo { id: u32 }\n\n"
+ "impl Foo {\n"
+ " pub fn new() -> Self { Self { id: 0 } }\n"
+ "}\n\n"
+ "fn main() {}\n"
+ "#[derive(Debug)]\n"
+ "enum Color { Red, Green }\n"
+ )
+ java_code = (
+ "package com.example;\n\n"
+ "public class Foo {\n"
+ " @Override\n"
+ ' public String toString() { return "foo"; }\n'
+ "}\n\n"
+ "private interface Bar {}\n"
+ "protected enum Baz { A, B }\n"
+ )
+
+ plain_text = "Just some prose text without any structure or special markers."
+ empty = ""
+ whitespace = " \n\t \n"
+
+ return [
+ json_array_dicts,
+ json_array_scalars,
+ json_empty_array,
+ json_object,
+ git_diff,
+ merge_diff,
+ html_doctype,
+ html_structural,
+ html_below_threshold,
+ search_results,
+ build_log,
+ pytest_output,
+ python_code,
+ js_code,
+ ts_code,
+ go_code,
+ rust_code,
+ java_code,
+ plain_text,
+ empty,
+ whitespace,
+ ]
+
+
def _varied_message_batches() -> list[list[dict[str, Any]]]:
today = _dt.date.today().isoformat()
out: list[list[dict[str, Any]]] = []
@@ -566,6 +749,7 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]:
"tokenizer": 0,
"cache_aligner": 0,
"ccr": 0,
+ "content_detector": 0,
}
# log_compressor
@@ -643,6 +827,18 @@ def run_default_workload(root: Path | None = None) -> dict[str, int]:
except Exception as e:
LOG.warning("ccr workload failed: %s", e)
+ # content_detector — drive a wide mix of content types so every dispatch
+ # branch (json_array, diff, html, search, log, code-by-language,
+ # plain-text fallback) is exercised at least once.
+ try:
+ from headroom.transforms import content_detector as _cd_mod
+
+ for s in _varied_content_detector_inputs():
+ _cd_mod.detect_content_type(s)
+ counts["content_detector"] += 1
+ except Exception as e:
+ LOG.warning("content_detector workload failed: %s", e)
+
return counts