AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): first-class email (.eml/MIME) extraction with attachment indexing
xerj-org644261711d05
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 3e9bdc937..531f82b77 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -132,7 +132,7 @@ version = "1.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"dependencies = [- "windows-sys 0.60.2",+ "windows-sys 0.61.2",][[package]]@@ -143,7 +143,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"dependencies = ["anstyle","once_cell_polyfill",- "windows-sys 0.60.2",+ "windows-sys 0.61.2",][[package]]@@ -428,6 +428,12 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "base64"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"+[[package]]name = "base64urlsafedata"version = "0.5.5"@@ -1313,7 +1319,7 @@ dependencies = ["libc","option-ext","redox_users",- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -1432,7 +1438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"dependencies = ["libc",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -2047,6 +2053,18 @@ version = "0.17.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"+[[package]]+name = "hashify"+version = "0.2.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26"+dependencies = [+ "indexmap 2.14.0",+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "hashlink"version = "0.9.1"@@ -2246,7 +2264,7 @@ dependencies = ["libc","percent-encoding","pin-project-lite",- "socket2 0.5.10",+ "socket2 0.6.3","system-configuration","tokio","tower-service",@@ -2494,7 +2512,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"dependencies = ["hermit-abi","libc",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -2764,6 +2782,15 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"+[[package]]+name = "mail-parser"+version = "0.11.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86"+dependencies = [+ "hashify",+]+[[package]]name = "mappings"version = "0.7.2"@@ -3006,7 +3033,7 @@ version = "0.50.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"dependencies = [- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -3868,7 +3895,7 @@ dependencies = ["quinn-udp","rustc-hash","rustls",- "socket2 0.5.10",+ "socket2 0.6.3","thiserror 2.0.18","tokio","tracing",@@ -3906,7 +3933,7 @@ dependencies = ["cfg_aliases","libc","once_cell",- "socket2 0.5.10",+ "socket2 0.6.3","tracing","windows-sys 0.60.2",]@@ -4296,7 +4323,7 @@ dependencies = ["errno","libc","linux-raw-sys",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -4662,7 +4689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"dependencies = ["libc",- "windows-sys 0.60.2",+ "windows-sys 0.61.2",][[package]]@@ -4867,7 +4894,7 @@ dependencies = ["getrandom 0.4.2","once_cell","rustix",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -6240,7 +6267,7 @@ version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"dependencies = [- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -6674,6 +6701,7 @@ name = "xerj-autoindex"version = "1.0.0-rc.74"dependencies = ["anyhow",+ "base64 0.23.1","chrono","csv","encoding_rs",@@ -6681,6 +6709,7 @@ dependencies = ["fs2","ignore","libc",+ "mail-parser","memchr","pdf_oxide","quick-xml",diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex 72f86a652..5a1cde700 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -103,5 +103,7 @@ zip = { version = "2", default-features = false, features = ["deflate"] } # DOCserde_yaml = "0.9" # YAML multi-doc# The one C dependency (bundled SQLite), isolated to this crate.rusqlite = { version = "0.32", features = ["bundled"] }+mail-parser = "0.11.9"[dev-dependencies]+base64 = "0.23.1"diff --git a/engine/crates/xerj-autoindex/src/estimate.rs b/engine/crates/xerj-autoindex/src/estimate.rsindex a580be54b..ae44c9433 100644--- a/engine/crates/xerj-autoindex/src/estimate.rs+++ b/engine/crates/xerj-autoindex/src/estimate.rs@@ -120,6 +120,7 @@ pub fn exact_scan_bytes(| Family::TxtProse| Family::Code| Family::Docx+ | Family::Eml| Family::Pdf => Some(size),// Streaming, byte-capped and record-capped.Family::Jsonl | Family::Csv | Family::Logs | Family::TxtLines => {diff --git a/engine/crates/xerj-autoindex/src/extract/eml.rs b/engine/crates/xerj-autoindex/src/extract/eml.rsnew file mode 100644index 000000000..56f9b5d0a--- /dev/null+++ b/engine/crates/xerj-autoindex/src/extract/eml.rs@@ -0,0 +1,441 @@+//! Email (RFC 5322 / MIME) extraction.+//!+//! An `.eml` (or any `message/rfc822`) file used to fall through to the prose+//! extractor: its headers were dumped as body text, its MIME structure was left+//! raw, and a base64 attachment was indexed verbatim as gibberish — one message+//! exploding into many noise records. This parses the message properly and emits:+//!+//! - ONE (or, for a long body, a few section) record(s) for the MESSAGE itself,+//! carrying decoded header fields (`email_from`/`email_to`/`email_subject`/+//! `email_date`/`email_message_id`/…) and the decoded plain-text body.+//! - ONE OR MORE records PER ATTACHMENT, indexed SEPARATELY and linked back to+//! the parent by `email_message_id`/`email_subject`/`email_from`. A PDF+//! attachment is routed through the real PDF extractor (per-page records); a+//! text attachment is sectioned as a document; any other file gets a+//! name/type/size card so it is still findable by name.+//!+//! Header/body field names are the extractor's own vocabulary+//! (`FieldOrigin::Extractor`): present on every email, so they never move a file+//! between datasets.++use super::{+ for_each_section, read_whole, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE,+};+use anyhow::Result;+use mail_parser::{MessageParser, MimeHeaders, PartType};+use serde_json::{Map, Value};+use std::path::Path;++/// Whole-message read cap. Mailbox exports of a single message rarely approach+/// this; a message larger than it is treated as junk rather than parsed.+const MAX_EML: u64 = 64 << 20;++/// A message with more parts than this has its tail dropped (reported via+/// `ExtractStats.truncated`) — a defensive bound on a pathological MIME bomb.+const MAX_ATTACHMENTS: usize = 256;++/// Bytes of an attachment we are willing to route/section. Larger attachments+/// get a name card only (their bytes are not indexed).+const MAX_ATTACH_BYTES: usize = 24 << 20;++pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let Some(bytes) = read_whole(path, gzip, MAX_EML)? else {+ stats.junk += 1;+ return Ok(stats);+ };+ let Some(msg) = MessageParser::default().parse(&bytes) else {+ // Unparseable as a message — fall back to indexing it as prose rather+ // than dropping it, matching the never-fatal contract.+ return super::extract_as_document(path, gzip, sink);+ };++ // ── message header fields (decoded: encoded-words, charset) ──+ let subject = msg.subject().unwrap_or("").trim().to_string();+ let mut headers = Map::new();+ put(&mut headers, "email_subject", subject.clone());+ put(&mut headers, "email_from", addr_string(msg.from()));+ put(&mut headers, "email_to", addr_string(msg.to()));+ put(&mut headers, "email_cc", addr_string(msg.cc()));+ if let Some(d) = msg.date() {+ put(&mut headers, "email_date", d.to_rfc3339());+ }+ if let Some(id) = msg.message_id() {+ put(&mut headers, "email_message_id", id.to_string());+ }+ if let Some(irt) = msg.in_reply_to().as_text() {+ put(&mut headers, "email_in_reply_to", irt.to_string());+ }++ let title = if subject.is_empty() {+ "(no subject)".to_string()+ } else {+ subject.clone()+ };++ // ── message body: prefer decoded text/plain, else strip the HTML part ──+ let body = msg+ .body_text(0)+ .map(|c| c.to_string())+ .or_else(|| msg.body_html(0).map(|h| strip_html(&h)))+ .unwrap_or_default();++ // Emit the message itself as document section(s) carrying the headers.+ emit_email_doc(&headers, &title, body.trim(), "msg", sink, &mut stats);++ // ── attachments — each indexed as its own document(s) ──+ for (n, part) in msg.attachments().enumerate() {+ if n >= MAX_ATTACHMENTS {+ stats.truncated = true;+ break;+ }+ let name = part+ .attachment_name()+ .map(str::to_string)+ .unwrap_or_else(|| format!("attachment-{n}"));+ let ctype = part+ .content_type()+ .map(|c| {+ let mut s = c.ctype().to_string();+ if let Some(sub) = c.subtype() {+ s.push('/');+ s.push_str(sub);+ }+ s+ })+ .unwrap_or_default();+ let data: &[u8] = match &part.body {+ PartType::Binary(b) | PartType::InlineBinary(b) => b.as_ref(),+ PartType::Text(t) | PartType::Html(t) => t.as_bytes(),+ _ => &[],+ };+ let link = attach_link(&headers, &name, &ctype, data.len());++ if is_pdf(&name, data) && data.len() <= MAX_ATTACH_BYTES {+ route_pdf(data, n, &link, sink, &mut stats);+ } else if is_texty(&ctype, data) && data.len() <= MAX_ATTACH_BYTES {+ let text = String::from_utf8_lossy(data);+ emit_email_doc(+ &link,+ &name,+ text.trim(),+ &format!("att{n}"),+ sink,+ &mut stats,+ );+ } else {+ // Unindexable body — a name/type/size card keeps it discoverable.+ stats.records += 1;+ sink(RawRecord {+ fields: link,+ locator: format!("att{n}-card"),+ group: None,+ origin: FieldOrigin::Extractor,+ });+ }+ }++ Ok(stats)+}++/// Emit `body` as one or more section records, each stamped with `base_fields`+/// (message headers or attachment-link fields) plus `title`/`body`/`section`.+fn emit_email_doc(+ base_fields: &Map<String, Value>,+ title: &str,+ body: &str,+ loc_prefix: &str,+ sink: Sink,+ stats: &mut ExtractStats,+) {+ // Collect sections first so we know whether to stamp a `section` field.+ let mut secs: Vec<String> = Vec::new();+ for_each_section(body, &mut |s| {+ secs.push(s);+ secs.len() < MAX_RECORDS_PER_FILE+ });+ if secs.is_empty() {+ secs.push(String::new());+ }+ let multi = secs.len() > 1;+ for (i, sec) in secs.into_iter().enumerate() {+ let mut fields = base_fields.clone();+ fields.insert("title".into(), Value::String(title.to_string()));+ fields.insert("body".into(), Value::String(sec));+ if multi {+ fields.insert("section".into(), Value::Number((i as u64).into()));+ }+ stats.records += 1;+ if !sink(RawRecord {+ fields,+ locator: format!("{loc_prefix}-s{i}"),+ group: None,+ origin: FieldOrigin::Extractor,+ }) {+ return;+ }+ }+}++/// Route a PDF attachment's bytes through the real PDF extractor, re-tagging+/// each emitted page record with the parent-email link fields and a locator+/// namespaced to this attachment.+fn route_pdf(+ data: &[u8],+ n: usize,+ link: &Map<String, Value>,+ sink: Sink,+ stats: &mut ExtractStats,+) {+ use std::io::Write;+ let Ok(mut tmp) = tempfile::Builder::new().suffix(".pdf").tempfile() else {+ stats.junk += 1;+ return;+ };+ if tmp.write_all(data).is_err() {+ stats.junk += 1;+ return;+ }+ let prefix = format!("att{n}-");+ let mut inner = |mut rec: RawRecord| -> bool {+ for (k, v) in link.iter() {+ rec.fields.entry(k.clone()).or_insert_with(|| v.clone());+ }+ rec.locator = format!("{prefix}{}", rec.locator);+ rec.origin = FieldOrigin::Extractor;+ stats.records += 1;+ sink(rec)+ };+ // The PDF extractor increments its own record count; we count via `inner`+ // instead, so start from what it reports and keep only the delta semantics+ // simple: ignore its stats.records, trust `inner`.+ let _ = super::pdf::extract(tmp.path(), &mut inner);+}++// ── field helpers ──────────────────────────────────────────────────────────++fn put(m: &mut Map<String, Value>, k: &str, v: String) {+ if !v.is_empty() {+ m.insert(k.into(), Value::String(v));+ }+}++/// The link fields copied onto every attachment record so a search can find,+/// e.g., "the term sheet that came from Dana Klein".+fn attach_link(+ headers: &Map<String, Value>,+ name: &str,+ ctype: &str,+ size: usize,+) -> Map<String, Value> {+ let mut m = Map::new();+ m.insert("attachment_name".into(), Value::String(name.to_string()));+ if !ctype.is_empty() {+ m.insert(+ "attachment_content_type".into(),+ Value::String(ctype.to_string()),+ );+ }+ m.insert(+ "attachment_bytes".into(),+ Value::Number((size as u64).into()),+ );+ for k in [+ "email_subject",+ "email_from",+ "email_date",+ "email_message_id",+ ] {+ if let Some(v) = headers.get(k) {+ m.insert(k.into(), v.clone());+ }+ }+ m+}++fn addr_string(a: Option<&mail_parser::Address>) -> String {+ let Some(a) = a else { return String::new() };+ let mut out: Vec<String> = Vec::new();+ for addr in a.iter() {+ let email = addr.address().unwrap_or("");+ match addr.name() {+ Some(name) if !name.is_empty() && !email.is_empty() => {+ out.push(format!("{name} <{email}>"))+ }+ _ if !email.is_empty() => out.push(email.to_string()),+ Some(name) if !name.is_empty() => out.push(name.to_string()),+ _ => {}+ }+ }+ out.join(", ")+}++fn is_pdf(name: &str, data: &[u8]) -> bool {+ name.to_ascii_lowercase().ends_with(".pdf") || data.starts_with(b"%PDF-")+}++fn is_texty(ctype: &str, data: &[u8]) -> bool {+ if ctype.starts_with("text/")+ || ctype.contains("json")+ || ctype.contains("xml")+ || ctype.contains("csv")+ {+ return true;+ }+ // Fallback: mostly-printable UTF-8 sample.+ let sample = &data[..data.len().min(2048)];+ !sample.is_empty()+ && std::str::from_utf8(sample).is_ok()+ && sample+ .iter()+ .filter(|b| **b < 9 || (**b > 13 && **b < 32))+ .count()+ * 20+ < sample.len().max(1)+}++/// Cheap HTML → text for messages that only carry an HTML part: drop tags and+/// collapse whitespace. Not a full renderer — enough to index the words.+fn strip_html(html: &str) -> String {+ let mut out = String::with_capacity(html.len());+ let mut in_tag = false;+ for c in html.chars() {+ match c {+ '<' => in_tag = true,+ '>' => {+ in_tag = false;+ out.push(' ');+ }+ _ if !in_tag => out.push(c),+ _ => {}+ }+ }+ out.split_whitespace().collect::<Vec<_>>().join(" ")+}++#[cfg(test)]+mod tests {+ use super::*;+ use base64::Engine;+ use std::io::Write;++ fn run(eml: &[u8]) -> Vec<RawRecord> {+ let mut tmp = tempfile::Builder::new().suffix(".eml").tempfile().unwrap();+ tmp.write_all(eml).unwrap();+ let mut out = Vec::new();+ let mut sink = |r: RawRecord| {+ out.push(r);+ true+ };+ extract(tmp.path(), false, &mut sink).unwrap();+ out+ }++ fn field<'a>(r: &'a RawRecord, k: &str) -> Option<&'a str> {+ r.fields.get(k).and_then(|v| v.as_str())+ }++ /// Headers become fields, the body is decoded (quoted-printable + charset),+ /// and NONE of the raw MIME/base64 leaks into the message body.+ #[test]+ fn parses_headers_and_decodes_body() {+ let b64 = base64::engine::general_purpose::STANDARD.encode("secret plan attached");+ let eml = format!(+ "From: Dana Klein <[email protected]>\r\n\+ To: Alex Rivard <[email protected]>\r\n\+ Subject: =?utf-8?q?Acquisition_=E2=80=94_next_steps?=\r\n\+ Date: Tue, 15 Jul 2026 09:12:00 -0700\r\n\+ Message-ID: <[email protected]>\r\n\+ MIME-Version: 1.0\r\n\+ Content-Type: multipart/mixed; boundary=\"b1\"\r\n\+ \r\n\+ --b1\r\n\+ Content-Type: text/plain; charset=utf-8\r\n\+ Content-Transfer-Encoding: quoted-printable\r\n\+ \r\n\+ Alex =E2=80=94 let's move to a term sheet at $420M.\r\n\+ --b1\r\n\+ Content-Type: application/octet-stream; name=\"notes.txt\"\r\n\+ Content-Transfer-Encoding: base64\r\n\+ Content-Disposition: attachment; filename=\"notes.txt\"\r\n\+ \r\n\+ {b64}\r\n\+ --b1--\r\n"+ );+ let recs = run(eml.as_bytes());+ let msg = recs.iter().find(|r| r.locator.starts_with("msg-")).unwrap();+ assert_eq!(+ field(msg, "email_from"),+ Some("Dana Klein <[email protected]>")+ );+ assert_eq!(+ field(msg, "email_message_id"),+ Some("[email protected]").or(Some("<[email protected]>"))+ );+ // Encoded-word subject decoded; em-dash present, no `=?utf-8?` left.+ let subj = field(msg, "email_subject").unwrap();+ assert!(+ subj.contains("Acquisition") && subj.contains('\u{2014}'),+ "subj: {subj}"+ );+ // Body decoded (QP `=E2=80=94` -> em-dash), no raw MIME.+ let body = field(msg, "body").unwrap();+ assert!(body.contains("term sheet at $420M"), "body: {body}");+ assert!(+ !body.contains("Content-Transfer-Encoding") && !body.contains("--b1"),+ "MIME leaked: {body}"+ );++ // The attachment is its OWN record, decoded (not base64), linked to the email.+ let att = recs+ .iter()+ .find(|r| field(r, "attachment_name") == Some("notes.txt"))+ .unwrap();+ assert_ne!(+ att.locator, msg.locator,+ "attachment must be a separate document"+ );+ assert_eq!(+ field(att, "email_subject"),+ field(msg, "email_subject"),+ "attachment links to parent"+ );+ let att_body = field(att, "body").unwrap();+ assert!(+ att_body.contains("secret plan attached"),+ "attachment body not decoded: {att_body}"+ );+ }++ /// A non-text, non-PDF attachment gets a name/type/size card so it stays+ /// discoverable, without indexing its bytes as noise.+ #[test]+ fn binary_attachment_gets_a_card() {+ let bytes =+ base64::engine::general_purpose::STANDARD.encode([0u8, 1, 2, 255, 254, 3, 9, 200]);+ let eml = format!(+ "From: [email protected]\r\nTo: [email protected]\r\nSubject: logo\r\nDate: Wed, 16 Jul 2026 10:00:00 -0700\r\n\+ Message-ID: <[email protected]>\r\nMIME-Version: 1.0\r\n\+ Content-Type: multipart/mixed; boundary=\"z\"\r\n\r\n\+ --z\r\nContent-Type: text/plain\r\n\r\nsee logo\r\n\+ --z\r\nContent-Type: image/png; name=\"logo.png\"\r\n\+ Content-Transfer-Encoding: base64\r\n\+ Content-Disposition: attachment; filename=\"logo.png\"\r\n\r\n{bytes}\r\n--z--\r\n"+ );+ let recs = run(eml.as_bytes());+ let card = recs+ .iter()+ .find(|r| field(r, "attachment_name") == Some("logo.png"))+ .unwrap();+ assert!(+ card.locator.contains("card"),+ "png should be a name card, got {}",+ card.locator+ );+ assert!(+ card.fields.get("body").is_none(),+ "binary body must not be indexed"+ );+ assert_eq!(field(card, "attachment_content_type"), Some("image/png"));+ }+}diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex d754c734a..53a46f53d 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -6,6 +6,7 @@ pub mod bvh;pub mod code;pub mod csv_x;pub mod docx;+pub mod eml;pub mod html;pub mod json;pub mod jsonl;@@ -201,6 +202,7 @@ pub fn extract(Family::TxtProse => txt::extract_prose(path, sn.gzip, sink),Family::TxtLines => txt::extract_lines(path, sn.gzip, limit_bytes, sink),Family::Pdf => pdf::extract(path, sink),+ Family::Eml => eml::extract(path, sn.gzip, sink),Family::Docx => docx::extract(path, sink),Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),diff --git a/engine/crates/xerj-autoindex/src/order.rs b/engine/crates/xerj-autoindex/src/order.rsindex 19b537c90..b05e888ce 100644--- a/engine/crates/xerj-autoindex/src/order.rs+++ b/engine/crates/xerj-autoindex/src/order.rs@@ -181,9 +181,12 @@ pub fn band(rel: &str, family: Family) -> Band {return Band::Vendored;}match family {- Family::Code | Family::TxtProse | Family::Html | Family::Pdf | Family::Docx => {- Band::SourceAndDocs- }+ Family::Code+ | Family::TxtProse+ | Family::Html+ | Family::Pdf+ | Family::Docx+ | Family::Eml => Band::SourceAndDocs,Family::Yaml | Family::Json | Family::Xml => Band::Config,Family::Csv | Family::Jsonl | Family::Sqlite | Family::SqlDump => Band::Data,Family::Logs | Family::TxtLines => Band::Bulk,diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex 240d5a6a1..a8ade8be0 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -19,6 +19,10 @@ pub enum Family {TxtProse,TxtLines,Pdf,+ /// RFC 5322 / MIME email message (`.eml`, `message/rfc822`): headers parsed+ /// into fields, MIME decoded, and each attachment indexed as its own+ /// document (PDFs routed to the PDF extractor).+ Eml,Docx,Sqlite,SqlDump,@@ -55,6 +59,7 @@ impl Family {Family::TxtProse => "txt-prose",Family::TxtLines => "txt-lines",Family::Pdf => "pdf",+ Family::Eml => "eml",Family::Docx => "docx",Family::Sqlite => "sqlite",Family::SqlDump => "sqldump",@@ -68,7 +73,10 @@ impl Family {}/// Document-family formats produce one record per document/section.pub fn is_document(&self) -> bool {- matches!(self, Family::Pdf | Family::Docx | Family::TxtProse)+ matches!(+ self,+ Family::Pdf | Family::Docx | Family::TxtProse | Family::Eml+ )}}@@ -1033,6 +1041,14 @@ pub fn decode_text(bytes: &[u8]) -> (String, &'static str) {}fn classify_text(text: &str, nonblank: &[&str]) -> Family {+ // Email wins first: an RFC 5322 message opens with a header block whose lines+ // (`From:`/`Date:`/`Message-ID:`/…) would otherwise be misread as prose or a+ // key:value log. Gated on a STRONG header plus >=3 canonical headers so an+ // ordinary `key: value` config or YAML never trips it.+ if looks_like_email(nonblank) {+ return Family::Eml;+ }+// Structured families (JSON/HTML/XML/logs/SQL/CSV) win first, for the whole// file and — via the #551 frontmatter lookahead below — for a frontmatter// body too.@@ -1095,6 +1111,88 @@ fn classify_text(text: &str, nonblank: &[&str]) -> Family {txt_kind(nonblank)}+/// Header name of an RFC 5322 header line (`From`, `Content-Type`, `X-Mailer`),+/// or `None` if the line is not `Name: …` with an RFC 822 field name.+fn header_name(line: &str) -> Option<&str> {+ let colon = line.find(':')?;+ let name = &line[..colon];+ if name.is_empty()+ || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')+ || !name.as_bytes()[0].is_ascii_alphabetic()+ {+ return None;+ }+ Some(name)+}++/// Does this text open like an email? Requires the first line to be a header, a+/// STRONG header (`From`/`Date`/`Message-ID`/`Received`/…) somewhere in the+/// opening block, and at least three canonical email headers total — enough to+/// separate a real message from a `key: value` config that merely uses colons.+fn looks_like_email(nonblank: &[&str]) -> bool {+ const STRONG: &[&str] = &[+ "from",+ "date",+ "message-id",+ "received",+ "return-path",+ "delivered-to",+ "dkim-signature",+ ];+ const KNOWN: &[&str] = &[+ "from",+ "to",+ "cc",+ "bcc",+ "subject",+ "date",+ "message-id",+ "received",+ "return-path",+ "reply-to",+ "in-reply-to",+ "references",+ "mime-version",+ "content-type",+ "content-transfer-encoding",+ "content-disposition",+ "delivered-to",+ "dkim-signature",+ "sender",+ "list-id",+ "authentication-results",+ "x-mailer",+ ];+ if !nonblank+ .first()+ .map(|l| header_name(l).is_some())+ .unwrap_or(false)+ {+ return false;+ }+ let mut strong = false;+ let mut known = 0usize;+ for line in nonblank.iter().take(50) {+ // Folded continuation lines (leading whitespace) are part of the header+ // block; the first line that is neither a header nor a fold is the body.+ let is_fold = line.starts_with(' ') || line.starts_with('\t');+ match header_name(line) {+ Some(name) => {+ let n = name.to_ascii_lowercase();+ if KNOWN.contains(&n.as_str()) {+ known += 1;+ }+ if STRONG.contains(&n.as_str()) {+ strong = true;+ }+ }+ None if is_fold => {}+ None => break,+ }+ }+ strong && known >= 3+}+/// The structured-family branches of [`classify_text`] — JSON/JSONL, HTML/XML,/// logs, SQL dump, CSV — returning the first family that matches, or `None` to/// continue to the YAML/prose fallbacks.@@ -1480,6 +1578,30 @@ mod unity_sniff_tests {.family}+ /// A real RFC 5322 message classifies as `Eml`; a `key: value` config or a+ /// YAML file that merely uses colons must NOT (the detector is gated on a+ /// strong header plus >=3 canonical email headers).+ #[test]+ fn email_is_detected_but_config_is_not() {+ let email = "From: Dana Klein <[email protected]>\n\+ To: Alex Rivard <[email protected]>\n\+ Subject: Acquisition next steps\n\+ Date: Tue, 15 Jul 2026 09:12:00 -0700\n\+ Message-ID: <[email protected]>\n\+ MIME-Version: 1.0\n\n\+ Alex, let's move to a term sheet.\n";+ assert_eq!(sniff_str(email, "m.eml"), Family::Eml);++ // A config with colons — one weak match ("subject"? no), no strong header.+ let config = "name: myapp\nversion: 1.2.3\nport: 8080\ndebug: true\n";+ assert_ne!(sniff_str(config, "app.yml"), Family::Eml);++ // Even a file that happens to have `to:`/`from:` keys but no strong+ // header and <3 canonical headers stays out.+ let sneaky = "to: the moon\nsubject: rockets\ncargo: fuel\n";+ assert_ne!(sniff_str(sneaky, "notes.txt"), Family::Eml);+ }+#[test]fn unity_tagged_yaml_is_detected_by_header_not_extension() {let scene =