Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): first-class email (.eml/MIME) extraction with attachment indexing

xerj-org644261711d05
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 3e9bdc937..531f82b77 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -132,7 +132,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -143,7 +143,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -428,6 +428,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
[[package]]
name = "base64urlsafedata"
version = "0.5.5"
@@ -1313,7 +1319,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -1432,7 +1438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2047,6 +2053,18 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
+[[package]]
+name = "hashify"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26"
+dependencies = [
+ "indexmap 2.14.0",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "hashlink"
version = "0.9.1"
@@ -2246,7 +2264,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2 0.5.10",
+ "socket2 0.6.3",
"system-configuration",
"tokio",
"tower-service",
@@ -2494,7 +2512,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2764,6 +2782,15 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
+[[package]]
+name = "mail-parser"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86"
+dependencies = [
+ "hashify",
+]
+
[[package]]
name = "mappings"
version = "0.7.2"
@@ -3006,7 +3033,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3868,7 +3895,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls",
- "socket2 0.5.10",
+ "socket2 0.6.3",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -3906,7 +3933,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2 0.5.10",
+ "socket2 0.6.3",
"tracing",
"windows-sys 0.60.2",
]
@@ -4296,7 +4323,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4662,7 +4689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4867,7 +4894,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -6240,7 +6267,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -6674,6 +6701,7 @@ name = "xerj-autoindex"
version = "1.0.0-rc.74"
dependencies = [
"anyhow",
+ "base64 0.23.1",
"chrono",
"csv",
"encoding_rs",
@@ -6681,6 +6709,7 @@ dependencies = [
"fs2",
"ignore",
"libc",
+ "mail-parser",
"memchr",
"pdf_oxide",
"quick-xml",
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index 72f86a652..5a1cde700 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -103,5 +103,7 @@ zip = { version = "2", default-features = false, features = ["deflate"] } # DOC
serde_yaml = "0.9" # YAML multi-doc
# The one C dependency (bundled SQLite), isolated to this crate.
rusqlite = { version = "0.32", features = ["bundled"] }
+mail-parser = "0.11.9"
[dev-dependencies]
+base64 = "0.23.1"
diff --git a/engine/crates/xerj-autoindex/src/estimate.rs b/engine/crates/xerj-autoindex/src/estimate.rs
index a580be54b..ae44c9433 100644
--- a/engine/crates/xerj-autoindex/src/estimate.rs
+++ b/engine/crates/xerj-autoindex/src/estimate.rs
@@ -120,6 +120,7 @@ pub fn exact_scan_bytes(
| Family::TxtProse
| Family::Code
| Family::Docx
+ | Family::Eml
| Family::Pdf => Some(size),
// Streaming, byte-capped and record-capped.
Family::Jsonl | Family::Csv | Family::Logs | Family::TxtLines => {
diff --git a/engine/crates/xerj-autoindex/src/extract/eml.rs b/engine/crates/xerj-autoindex/src/extract/eml.rs
new file mode 100644
index 000000000..56f9b5d0a
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/extract/eml.rs
@@ -0,0 +1,441 @@
+//! Email (RFC 5322 / MIME) extraction.
+//!
+//! An `.eml` (or any `message/rfc822`) file used to fall through to the prose
+//! extractor: its headers were dumped as body text, its MIME structure was left
+//! raw, and a base64 attachment was indexed verbatim as gibberish — one message
+//! exploding into many noise records. This parses the message properly and emits:
+//!
+//! - ONE (or, for a long body, a few section) record(s) for the MESSAGE itself,
+//! carrying decoded header fields (`email_from`/`email_to`/`email_subject`/
+//! `email_date`/`email_message_id`/…) and the decoded plain-text body.
+//! - ONE OR MORE records PER ATTACHMENT, indexed SEPARATELY and linked back to
+//! the parent by `email_message_id`/`email_subject`/`email_from`. A PDF
+//! attachment is routed through the real PDF extractor (per-page records); a
+//! text attachment is sectioned as a document; any other file gets a
+//! name/type/size card so it is still findable by name.
+//!
+//! Header/body field names are the extractor's own vocabulary
+//! (`FieldOrigin::Extractor`): present on every email, so they never move a file
+//! between datasets.
+
+use super::{
+ for_each_section, read_whole, ExtractStats, FieldOrigin, RawRecord, Sink, MAX_RECORDS_PER_FILE,
+};
+use anyhow::Result;
+use mail_parser::{MessageParser, MimeHeaders, PartType};
+use serde_json::{Map, Value};
+use std::path::Path;
+
+/// Whole-message read cap. Mailbox exports of a single message rarely approach
+/// this; a message larger than it is treated as junk rather than parsed.
+const MAX_EML: u64 = 64 << 20;
+
+/// A message with more parts than this has its tail dropped (reported via
+/// `ExtractStats.truncated`) — a defensive bound on a pathological MIME bomb.
+const MAX_ATTACHMENTS: usize = 256;
+
+/// Bytes of an attachment we are willing to route/section. Larger attachments
+/// get a name card only (their bytes are not indexed).
+const MAX_ATTACH_BYTES: usize = 24 << 20;
+
+pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let Some(bytes) = read_whole(path, gzip, MAX_EML)? else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+ let Some(msg) = MessageParser::default().parse(&bytes) else {
+ // Unparseable as a message — fall back to indexing it as prose rather
+ // than dropping it, matching the never-fatal contract.
+ return super::extract_as_document(path, gzip, sink);
+ };
+
+ // ── message header fields (decoded: encoded-words, charset) ──
+ let subject = msg.subject().unwrap_or("").trim().to_string();
+ let mut headers = Map::new();
+ put(&mut headers, "email_subject", subject.clone());
+ put(&mut headers, "email_from", addr_string(msg.from()));
+ put(&mut headers, "email_to", addr_string(msg.to()));
+ put(&mut headers, "email_cc", addr_string(msg.cc()));
+ if let Some(d) = msg.date() {
+ put(&mut headers, "email_date", d.to_rfc3339());
+ }
+ if let Some(id) = msg.message_id() {
+ put(&mut headers, "email_message_id", id.to_string());
+ }
+ if let Some(irt) = msg.in_reply_to().as_text() {
+ put(&mut headers, "email_in_reply_to", irt.to_string());
+ }
+
+ let title = if subject.is_empty() {
+ "(no subject)".to_string()
+ } else {
+ subject.clone()
+ };
+
+ // ── message body: prefer decoded text/plain, else strip the HTML part ──
+ let body = msg
+ .body_text(0)
+ .map(|c| c.to_string())
+ .or_else(|| msg.body_html(0).map(|h| strip_html(&h)))
+ .unwrap_or_default();
+
+ // Emit the message itself as document section(s) carrying the headers.
+ emit_email_doc(&headers, &title, body.trim(), "msg", sink, &mut stats);
+
+ // ── attachments — each indexed as its own document(s) ──
+ for (n, part) in msg.attachments().enumerate() {
+ if n >= MAX_ATTACHMENTS {
+ stats.truncated = true;
+ break;
+ }
+ let name = part
+ .attachment_name()
+ .map(str::to_string)
+ .unwrap_or_else(|| format!("attachment-{n}"));
+ let ctype = part
+ .content_type()
+ .map(|c| {
+ let mut s = c.ctype().to_string();
+ if let Some(sub) = c.subtype() {
+ s.push('/');
+ s.push_str(sub);
+ }
+ s
+ })
+ .unwrap_or_default();
+ let data: &[u8] = match &part.body {
+ PartType::Binary(b) | PartType::InlineBinary(b) => b.as_ref(),
+ PartType::Text(t) | PartType::Html(t) => t.as_bytes(),
+ _ => &[],
+ };
+ let link = attach_link(&headers, &name, &ctype, data.len());
+
+ if is_pdf(&name, data) && data.len() <= MAX_ATTACH_BYTES {
+ route_pdf(data, n, &link, sink, &mut stats);
+ } else if is_texty(&ctype, data) && data.len() <= MAX_ATTACH_BYTES {
+ let text = String::from_utf8_lossy(data);
+ emit_email_doc(
+ &link,
+ &name,
+ text.trim(),
+ &format!("att{n}"),
+ sink,
+ &mut stats,
+ );
+ } else {
+ // Unindexable body — a name/type/size card keeps it discoverable.
+ stats.records += 1;
+ sink(RawRecord {
+ fields: link,
+ locator: format!("att{n}-card"),
+ group: None,
+ origin: FieldOrigin::Extractor,
+ });
+ }
+ }
+
+ Ok(stats)
+}
+
+/// Emit `body` as one or more section records, each stamped with `base_fields`
+/// (message headers or attachment-link fields) plus `title`/`body`/`section`.
+fn emit_email_doc(
+ base_fields: &Map<String, Value>,
+ title: &str,
+ body: &str,
+ loc_prefix: &str,
+ sink: Sink,
+ stats: &mut ExtractStats,
+) {
+ // Collect sections first so we know whether to stamp a `section` field.
+ let mut secs: Vec<String> = Vec::new();
+ for_each_section(body, &mut |s| {
+ secs.push(s);
+ secs.len() < MAX_RECORDS_PER_FILE
+ });
+ if secs.is_empty() {
+ secs.push(String::new());
+ }
+ let multi = secs.len() > 1;
+ for (i, sec) in secs.into_iter().enumerate() {
+ let mut fields = base_fields.clone();
+ fields.insert("title".into(), Value::String(title.to_string()));
+ fields.insert("body".into(), Value::String(sec));
+ if multi {
+ fields.insert("section".into(), Value::Number((i as u64).into()));
+ }
+ stats.records += 1;
+ if !sink(RawRecord {
+ fields,
+ locator: format!("{loc_prefix}-s{i}"),
+ group: None,
+ origin: FieldOrigin::Extractor,
+ }) {
+ return;
+ }
+ }
+}
+
+/// Route a PDF attachment's bytes through the real PDF extractor, re-tagging
+/// each emitted page record with the parent-email link fields and a locator
+/// namespaced to this attachment.
+fn route_pdf(
+ data: &[u8],
+ n: usize,
+ link: &Map<String, Value>,
+ sink: Sink,
+ stats: &mut ExtractStats,
+) {
+ use std::io::Write;
+ let Ok(mut tmp) = tempfile::Builder::new().suffix(".pdf").tempfile() else {
+ stats.junk += 1;
+ return;
+ };
+ if tmp.write_all(data).is_err() {
+ stats.junk += 1;
+ return;
+ }
+ let prefix = format!("att{n}-");
+ let mut inner = |mut rec: RawRecord| -> bool {
+ for (k, v) in link.iter() {
+ rec.fields.entry(k.clone()).or_insert_with(|| v.clone());
+ }
+ rec.locator = format!("{prefix}{}", rec.locator);
+ rec.origin = FieldOrigin::Extractor;
+ stats.records += 1;
+ sink(rec)
+ };
+ // The PDF extractor increments its own record count; we count via `inner`
+ // instead, so start from what it reports and keep only the delta semantics
+ // simple: ignore its stats.records, trust `inner`.
+ let _ = super::pdf::extract(tmp.path(), &mut inner);
+}
+
+// ── field helpers ──────────────────────────────────────────────────────────
+
+fn put(m: &mut Map<String, Value>, k: &str, v: String) {
+ if !v.is_empty() {
+ m.insert(k.into(), Value::String(v));
+ }
+}
+
+/// The link fields copied onto every attachment record so a search can find,
+/// e.g., "the term sheet that came from Dana Klein".
+fn attach_link(
+ headers: &Map<String, Value>,
+ name: &str,
+ ctype: &str,
+ size: usize,
+) -> Map<String, Value> {
+ let mut m = Map::new();
+ m.insert("attachment_name".into(), Value::String(name.to_string()));
+ if !ctype.is_empty() {
+ m.insert(
+ "attachment_content_type".into(),
+ Value::String(ctype.to_string()),
+ );
+ }
+ m.insert(
+ "attachment_bytes".into(),
+ Value::Number((size as u64).into()),
+ );
+ for k in [
+ "email_subject",
+ "email_from",
+ "email_date",
+ "email_message_id",
+ ] {
+ if let Some(v) = headers.get(k) {
+ m.insert(k.into(), v.clone());
+ }
+ }
+ m
+}
+
+fn addr_string(a: Option<&mail_parser::Address>) -> String {
+ let Some(a) = a else { return String::new() };
+ let mut out: Vec<String> = Vec::new();
+ for addr in a.iter() {
+ let email = addr.address().unwrap_or("");
+ match addr.name() {
+ Some(name) if !name.is_empty() && !email.is_empty() => {
+ out.push(format!("{name} <{email}>"))
+ }
+ _ if !email.is_empty() => out.push(email.to_string()),
+ Some(name) if !name.is_empty() => out.push(name.to_string()),
+ _ => {}
+ }
+ }
+ out.join(", ")
+}
+
+fn is_pdf(name: &str, data: &[u8]) -> bool {
+ name.to_ascii_lowercase().ends_with(".pdf") || data.starts_with(b"%PDF-")
+}
+
+fn is_texty(ctype: &str, data: &[u8]) -> bool {
+ if ctype.starts_with("text/")
+ || ctype.contains("json")
+ || ctype.contains("xml")
+ || ctype.contains("csv")
+ {
+ return true;
+ }
+ // Fallback: mostly-printable UTF-8 sample.
+ let sample = &data[..data.len().min(2048)];
+ !sample.is_empty()
+ && std::str::from_utf8(sample).is_ok()
+ && sample
+ .iter()
+ .filter(|b| **b < 9 || (**b > 13 && **b < 32))
+ .count()
+ * 20
+ < sample.len().max(1)
+}
+
+/// Cheap HTML → text for messages that only carry an HTML part: drop tags and
+/// collapse whitespace. Not a full renderer — enough to index the words.
+fn strip_html(html: &str) -> String {
+ let mut out = String::with_capacity(html.len());
+ let mut in_tag = false;
+ for c in html.chars() {
+ match c {
+ '<' => in_tag = true,
+ '>' => {
+ in_tag = false;
+ out.push(' ');
+ }
+ _ if !in_tag => out.push(c),
+ _ => {}
+ }
+ }
+ out.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use base64::Engine;
+ use std::io::Write;
+
+ fn run(eml: &[u8]) -> Vec<RawRecord> {
+ let mut tmp = tempfile::Builder::new().suffix(".eml").tempfile().unwrap();
+ tmp.write_all(eml).unwrap();
+ let mut out = Vec::new();
+ let mut sink = |r: RawRecord| {
+ out.push(r);
+ true
+ };
+ extract(tmp.path(), false, &mut sink).unwrap();
+ out
+ }
+
+ fn field<'a>(r: &'a RawRecord, k: &str) -> Option<&'a str> {
+ r.fields.get(k).and_then(|v| v.as_str())
+ }
+
+ /// Headers become fields, the body is decoded (quoted-printable + charset),
+ /// and NONE of the raw MIME/base64 leaks into the message body.
+ #[test]
+ fn parses_headers_and_decodes_body() {
+ let b64 = base64::engine::general_purpose::STANDARD.encode("secret plan attached");
+ let eml = format!(
+ "From: Dana Klein <[email protected]>\r\n\
+ To: Alex Rivard <[email protected]>\r\n\
+ Subject: =?utf-8?q?Acquisition_=E2=80=94_next_steps?=\r\n\
+ Date: Tue, 15 Jul 2026 09:12:00 -0700\r\n\
+ Message-ID: <[email protected]>\r\n\
+ MIME-Version: 1.0\r\n\
+ Content-Type: multipart/mixed; boundary=\"b1\"\r\n\
+ \r\n\
+ --b1\r\n\
+ Content-Type: text/plain; charset=utf-8\r\n\
+ Content-Transfer-Encoding: quoted-printable\r\n\
+ \r\n\
+ Alex =E2=80=94 let's move to a term sheet at $420M.\r\n\
+ --b1\r\n\
+ Content-Type: application/octet-stream; name=\"notes.txt\"\r\n\
+ Content-Transfer-Encoding: base64\r\n\
+ Content-Disposition: attachment; filename=\"notes.txt\"\r\n\
+ \r\n\
+ {b64}\r\n\
+ --b1--\r\n"
+ );
+ let recs = run(eml.as_bytes());
+ let msg = recs.iter().find(|r| r.locator.starts_with("msg-")).unwrap();
+ assert_eq!(
+ field(msg, "email_from"),
+ Some("Dana Klein <[email protected]>")
+ );
+ assert_eq!(
+ field(msg, "email_message_id"),
+ Some("[email protected]").or(Some("<[email protected]>"))
+ );
+ // Encoded-word subject decoded; em-dash present, no `=?utf-8?` left.
+ let subj = field(msg, "email_subject").unwrap();
+ assert!(
+ subj.contains("Acquisition") && subj.contains('\u{2014}'),
+ "subj: {subj}"
+ );
+ // Body decoded (QP `=E2=80=94` -> em-dash), no raw MIME.
+ let body = field(msg, "body").unwrap();
+ assert!(body.contains("term sheet at $420M"), "body: {body}");
+ assert!(
+ !body.contains("Content-Transfer-Encoding") && !body.contains("--b1"),
+ "MIME leaked: {body}"
+ );
+
+ // The attachment is its OWN record, decoded (not base64), linked to the email.
+ let att = recs
+ .iter()
+ .find(|r| field(r, "attachment_name") == Some("notes.txt"))
+ .unwrap();
+ assert_ne!(
+ att.locator, msg.locator,
+ "attachment must be a separate document"
+ );
+ assert_eq!(
+ field(att, "email_subject"),
+ field(msg, "email_subject"),
+ "attachment links to parent"
+ );
+ let att_body = field(att, "body").unwrap();
+ assert!(
+ att_body.contains("secret plan attached"),
+ "attachment body not decoded: {att_body}"
+ );
+ }
+
+ /// A non-text, non-PDF attachment gets a name/type/size card so it stays
+ /// discoverable, without indexing its bytes as noise.
+ #[test]
+ fn binary_attachment_gets_a_card() {
+ let bytes =
+ base64::engine::general_purpose::STANDARD.encode([0u8, 1, 2, 255, 254, 3, 9, 200]);
+ let eml = format!(
+ "From: [email protected]\r\nTo: [email protected]\r\nSubject: logo\r\nDate: Wed, 16 Jul 2026 10:00:00 -0700\r\n\
+ Message-ID: <[email protected]>\r\nMIME-Version: 1.0\r\n\
+ Content-Type: multipart/mixed; boundary=\"z\"\r\n\r\n\
+ --z\r\nContent-Type: text/plain\r\n\r\nsee logo\r\n\
+ --z\r\nContent-Type: image/png; name=\"logo.png\"\r\n\
+ Content-Transfer-Encoding: base64\r\n\
+ Content-Disposition: attachment; filename=\"logo.png\"\r\n\r\n{bytes}\r\n--z--\r\n"
+ );
+ let recs = run(eml.as_bytes());
+ let card = recs
+ .iter()
+ .find(|r| field(r, "attachment_name") == Some("logo.png"))
+ .unwrap();
+ assert!(
+ card.locator.contains("card"),
+ "png should be a name card, got {}",
+ card.locator
+ );
+ assert!(
+ card.fields.get("body").is_none(),
+ "binary body must not be indexed"
+ );
+ assert_eq!(field(card, "attachment_content_type"), Some("image/png"));
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index d754c734a..53a46f53d 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -6,6 +6,7 @@ pub mod bvh;
pub mod code;
pub mod csv_x;
pub mod docx;
+pub mod eml;
pub mod html;
pub mod json;
pub mod jsonl;
@@ -201,6 +202,7 @@ pub fn extract(
Family::TxtProse => txt::extract_prose(path, sn.gzip, sink),
Family::TxtLines => txt::extract_lines(path, sn.gzip, limit_bytes, sink),
Family::Pdf => pdf::extract(path, sink),
+ Family::Eml => eml::extract(path, sn.gzip, sink),
Family::Docx => docx::extract(path, sink),
Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),
Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),
diff --git a/engine/crates/xerj-autoindex/src/order.rs b/engine/crates/xerj-autoindex/src/order.rs
index 19b537c90..b05e888ce 100644
--- a/engine/crates/xerj-autoindex/src/order.rs
+++ b/engine/crates/xerj-autoindex/src/order.rs
@@ -181,9 +181,12 @@ pub fn band(rel: &str, family: Family) -> Band {
return Band::Vendored;
}
match family {
- Family::Code | Family::TxtProse | Family::Html | Family::Pdf | Family::Docx => {
- Band::SourceAndDocs
- }
+ Family::Code
+ | Family::TxtProse
+ | Family::Html
+ | Family::Pdf
+ | Family::Docx
+ | Family::Eml => Band::SourceAndDocs,
Family::Yaml | Family::Json | Family::Xml => Band::Config,
Family::Csv | Family::Jsonl | Family::Sqlite | Family::SqlDump => Band::Data,
Family::Logs | Family::TxtLines => Band::Bulk,
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index 240d5a6a1..a8ade8be0 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -19,6 +19,10 @@ pub enum Family {
TxtProse,
TxtLines,
Pdf,
+ /// RFC 5322 / MIME email message (`.eml`, `message/rfc822`): headers parsed
+ /// into fields, MIME decoded, and each attachment indexed as its own
+ /// document (PDFs routed to the PDF extractor).
+ Eml,
Docx,
Sqlite,
SqlDump,
@@ -55,6 +59,7 @@ impl Family {
Family::TxtProse => "txt-prose",
Family::TxtLines => "txt-lines",
Family::Pdf => "pdf",
+ Family::Eml => "eml",
Family::Docx => "docx",
Family::Sqlite => "sqlite",
Family::SqlDump => "sqldump",
@@ -68,7 +73,10 @@ impl Family {
}
/// Document-family formats produce one record per document/section.
pub fn is_document(&self) -> bool {
- matches!(self, Family::Pdf | Family::Docx | Family::TxtProse)
+ matches!(
+ self,
+ Family::Pdf | Family::Docx | Family::TxtProse | Family::Eml
+ )
}
}
@@ -1033,6 +1041,14 @@ pub fn decode_text(bytes: &[u8]) -> (String, &'static str) {
}
fn classify_text(text: &str, nonblank: &[&str]) -> Family {
+ // Email wins first: an RFC 5322 message opens with a header block whose lines
+ // (`From:`/`Date:`/`Message-ID:`/…) would otherwise be misread as prose or a
+ // key:value log. Gated on a STRONG header plus >=3 canonical headers so an
+ // ordinary `key: value` config or YAML never trips it.
+ if looks_like_email(nonblank) {
+ return Family::Eml;
+ }
+
// Structured families (JSON/HTML/XML/logs/SQL/CSV) win first, for the whole
// file and — via the #551 frontmatter lookahead below — for a frontmatter
// body too.
@@ -1095,6 +1111,88 @@ fn classify_text(text: &str, nonblank: &[&str]) -> Family {
txt_kind(nonblank)
}
+/// Header name of an RFC 5322 header line (`From`, `Content-Type`, `X-Mailer`),
+/// or `None` if the line is not `Name: …` with an RFC 822 field name.
+fn header_name(line: &str) -> Option<&str> {
+ let colon = line.find(':')?;
+ let name = &line[..colon];
+ if name.is_empty()
+ || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')
+ || !name.as_bytes()[0].is_ascii_alphabetic()
+ {
+ return None;
+ }
+ Some(name)
+}
+
+/// Does this text open like an email? Requires the first line to be a header, a
+/// STRONG header (`From`/`Date`/`Message-ID`/`Received`/…) somewhere in the
+/// opening block, and at least three canonical email headers total — enough to
+/// separate a real message from a `key: value` config that merely uses colons.
+fn looks_like_email(nonblank: &[&str]) -> bool {
+ const STRONG: &[&str] = &[
+ "from",
+ "date",
+ "message-id",
+ "received",
+ "return-path",
+ "delivered-to",
+ "dkim-signature",
+ ];
+ const KNOWN: &[&str] = &[
+ "from",
+ "to",
+ "cc",
+ "bcc",
+ "subject",
+ "date",
+ "message-id",
+ "received",
+ "return-path",
+ "reply-to",
+ "in-reply-to",
+ "references",
+ "mime-version",
+ "content-type",
+ "content-transfer-encoding",
+ "content-disposition",
+ "delivered-to",
+ "dkim-signature",
+ "sender",
+ "list-id",
+ "authentication-results",
+ "x-mailer",
+ ];
+ if !nonblank
+ .first()
+ .map(|l| header_name(l).is_some())
+ .unwrap_or(false)
+ {
+ return false;
+ }
+ let mut strong = false;
+ let mut known = 0usize;
+ for line in nonblank.iter().take(50) {
+ // Folded continuation lines (leading whitespace) are part of the header
+ // block; the first line that is neither a header nor a fold is the body.
+ let is_fold = line.starts_with(' ') || line.starts_with('\t');
+ match header_name(line) {
+ Some(name) => {
+ let n = name.to_ascii_lowercase();
+ if KNOWN.contains(&n.as_str()) {
+ known += 1;
+ }
+ if STRONG.contains(&n.as_str()) {
+ strong = true;
+ }
+ }
+ None if is_fold => {}
+ None => break,
+ }
+ }
+ strong && known >= 3
+}
+
/// The structured-family branches of [`classify_text`] — JSON/JSONL, HTML/XML,
/// logs, SQL dump, CSV — returning the first family that matches, or `None` to
/// continue to the YAML/prose fallbacks.
@@ -1480,6 +1578,30 @@ mod unity_sniff_tests {
.family
}
+ /// A real RFC 5322 message classifies as `Eml`; a `key: value` config or a
+ /// YAML file that merely uses colons must NOT (the detector is gated on a
+ /// strong header plus >=3 canonical email headers).
+ #[test]
+ fn email_is_detected_but_config_is_not() {
+ let email = "From: Dana Klein <[email protected]>\n\
+ To: Alex Rivard <[email protected]>\n\
+ Subject: Acquisition next steps\n\
+ Date: Tue, 15 Jul 2026 09:12:00 -0700\n\
+ Message-ID: <[email protected]>\n\
+ MIME-Version: 1.0\n\n\
+ Alex, let's move to a term sheet.\n";
+ assert_eq!(sniff_str(email, "m.eml"), Family::Eml);
+
+ // A config with colons — one weak match ("subject"? no), no strong header.
+ let config = "name: myapp\nversion: 1.2.3\nport: 8080\ndebug: true\n";
+ assert_ne!(sniff_str(config, "app.yml"), Family::Eml);
+
+ // Even a file that happens to have `to:`/`from:` keys but no strong
+ // header and <3 canonical headers stays out.
+ let sneaky = "to: the moon\nsubject: rockets\ncargo: fuel\n";
+ assert_ne!(sniff_str(sneaky, "notes.txt"), Family::Eml);
+ }
+
#[test]
fn unity_tagged_yaml_is_detected_by_header_not_extension() {
let scene =