Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): OLE2 Office support — legacy .doc and .xls

xerj-org5235d9b25dbb
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 62b38b6c1..b1c209aed 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -963,6 +963,22 @@ dependencies = [
"either",
]
+[[package]]
+name = "calamine"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138646b9af2c5d7f1804ea4bf93afc597737d2bd4f7341d67c48b03316976eb1"
+dependencies = [
+ "byteorder",
+ "chrono",
+ "codepage",
+ "encoding_rs",
+ "log",
+ "quick-xml 0.31.0",
+ "serde",
+ "zip 2.4.2",
+]
+
[[package]]
name = "candle-core"
version = "0.9.2"
@@ -1057,6 +1073,17 @@ dependencies = [
"shlex",
]
+[[package]]
+name = "cfb"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f13298d9178622b7916ec5d01d470425c04273c8ed07e87b1c38b34c4f4612b"
+dependencies = [
+ "fnv",
+ "uuid",
+ "web-time",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -1199,6 +1226,15 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
+[[package]]
+name = "codepage"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26"
+dependencies = [
+ "encoding_rs",
+]
+
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -3675,7 +3711,7 @@ dependencies = [
"fast-float2",
"libc",
"log",
- "quick-xml",
+ "quick-xml 0.41.0",
"serde",
"serde_json",
"thiserror 2.0.18",
@@ -3881,7 +3917,7 @@ dependencies = [
"nom 8.0.0",
"office_oxide",
"phf 0.14.0",
- "quick-xml",
+ "quick-xml 0.41.0",
"regex",
"serde",
"serde_json",
@@ -4418,6 +4454,16 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "encoding_rs",
+ "memchr",
+]
+
[[package]]
name = "quick-xml"
version = "0.41.0"
@@ -7320,6 +7366,8 @@ dependencies = [
"aws-sdk-s3",
"aws-smithy-http-client",
"base64 0.23.1",
+ "calamine",
+ "cfb",
"chrono",
"csv",
"encoding_rs",
@@ -7332,7 +7380,7 @@ dependencies = [
"memchr",
"notify",
"pdf_oxide",
- "quick-xml",
+ "quick-xml 0.41.0",
"rayon",
"regex",
"reqwest",
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index 964a564da..8be565134 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -133,6 +133,22 @@ pdf_oxide = { version = "=0.3.75", default-features = false, features = ["legacy
csv = "1" # streaming CSV with sniffed dialect
quick-xml = "0.41" # XML pull parser + DOCX internals
zip = { version = "2", default-features = false, features = ["deflate"] } # DOCX container
+# --- Legacy Microsoft Office binary formats (OLE2 / Compound File Binary) ---
+# calamine reads the legacy BIFF `.xls` worksheet stream (it ships its own CFB
+# reader internally); pinned to 0.26.1, the last line that depends on `zip ^2`
+# so it reuses this crate's existing `zip 2` rather than pulling a second major
+# of zip into the tree. Licence: MIT. Its transitive adds are codepage (MIT,
+# for the legacy 8-bit cell codepage) and log (MIT/Apache-2.0); byteorder and
+# encoding_rs are already here.
+calamine = { version = "=0.26.1", features = ["dates"] }
+# `cfb` reads the OLE2/Compound File Binary container directly: used by the
+# content sniffer to route a `.doc`/`.xls`/other OLE2 file by its stream names,
+# and by the `.doc` extractor to pull the WordDocument + Table streams and the
+# SummaryInformation title. `open()` returns `io::Result` and treats malformed
+# or truncated input as an error (never a panic), which the fuzz-minded sniffer
+# relies on. Licence: MIT. Runtime deps (fnv, uuid) are already in the lock
+# file; web-time is a tiny wasm-time shim. Pinned exact.
+cfb = "=0.15.0"
serde_yaml = "0.9" # YAML multi-doc
# The one C dependency (bundled SQLite), isolated to this crate.
rusqlite = { version = "0.32", features = ["bundled"] }
diff --git a/engine/crates/xerj-autoindex/src/estimate.rs b/engine/crates/xerj-autoindex/src/estimate.rs
index 6874996fa..6fe5d8573 100644
--- a/engine/crates/xerj-autoindex/src/estimate.rs
+++ b/engine/crates/xerj-autoindex/src/estimate.rs
@@ -120,6 +120,7 @@ pub fn exact_scan_bytes(
| Family::TxtProse
| Family::Code
| Family::Docx
+ | Family::Doc
| Family::Pptx
| Family::Epub
| Family::Ipynb
@@ -144,7 +145,7 @@ pub fn exact_scan_bytes(
Family::UnityMeta => (size <= UNITY_META_CAP).then_some(size),
// Row-capped per table (per sheet); bytes read bear no fixed relation
// to `size`.
- Family::Sqlite | Family::Xlsx => None,
+ Family::Sqlite | Family::Xlsx | Family::Xls => None,
// Deliberately partial: `bvh::extract` stops at `Frame Time:` and
// never pulls the motion block off disk, so `size` is not the number
// of bytes this machine demonstrated it can chew through.
diff --git a/engine/crates/xerj-autoindex/src/extract/doc.rs b/engine/crates/xerj-autoindex/src/extract/doc.rs
new file mode 100644
index 000000000..2333587d4
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/extract/doc.rs
@@ -0,0 +1,500 @@
+//! DOC — legacy Word 97-2003 documents (`.doc`, an OLE2 compound file).
+//!
+//! Word 97 and later store the document's characters in the `WordDocument`
+//! stream and describe where they live with a *piece table* (the CLX) kept in
+//! the `0Table` or `1Table` stream. We read the File Information Block (FIB) at
+//! the head of `WordDocument` to locate the CLX, walk the piece table, and
+//! concatenate each piece — decoding it as little-endian UTF-16 or, when the
+//! piece is flagged "compressed", as the legacy 8-bit Windows-1252 code page
+//! (MS-DOC §2.3.1, §2.8.35, §2.9.177). The recovered text is emitted as section
+//! records, with the title taken from the `\x05SummaryInformation` property set
+//! when present.
+//!
+//! LIMITS — read honestly. This recovers the main document text of Word 97+
+//! files, which is the overwhelming majority of surviving `.doc` attachments.
+//! It does NOT recover:
+//! * formatting, styles, or heading structure — the output is plain prose, so
+//! unlike the DOCX extractor there is no `headings` list;
+//! * tables as tables — cell and row marks become line breaks, so a table
+//! reads as text, not as rows;
+//! * headers/footers, footnotes, endnotes, comments and textboxes, which live
+//! in separate ranges past the main `fcMac` and are not followed here;
+//! * Word 6.0/95 (and earlier) documents, which predate this piece-table
+//! layout — those fall back to reading the raw main stream as Windows-1252,
+//! which is lossy, and if even that yields nothing the file is counted as
+//! junk rather than mis-indexed.
+//!
+//! Field instruction codes (hidden `{ }` field source between the field-begin
+//! and field-separator marks) are dropped; the field RESULT text is kept.
+
+use super::{emit_document, ExtractStats, Sink, MAX_RECORDS_PER_FILE};
+use anyhow::Result;
+use std::io::Read;
+use std::path::Path;
+
+/// Largest `WordDocument` / table stream we read. A `.doc`'s text stream is
+/// far below this; the cap only bounds what a crafted compound file can cost.
+const MAX_STREAM_BYTES: u64 = 256 << 20;
+
+/// Largest property-set stream (`SummaryInformation`) we read for the title.
+const MAX_SUMMARY_BYTES: u64 = 1 << 20;
+
+/// Body text cap: past this the document is already far larger than any real
+/// `.doc`, and section emission is bounded by [`MAX_RECORDS_PER_FILE`] anyway.
+const MAX_BODY_CHARS: usize = 64 << 20;
+
+/// Defensive cap on piece-table entries, against a crafted `lcbClx`.
+const MAX_PIECES: usize = 1 << 20;
+
+// --- little-endian field reads, all bounds-checked (never panic on content) --
+
+fn u16_at(b: &[u8], off: usize) -> Option<u16> {
+ b.get(off..off + 2)
+ .map(|s| u16::from_le_bytes([s[0], s[1]]))
+}
+
+fn u32_at(b: &[u8], off: usize) -> Option<u32> {
+ b.get(off..off + 4)
+ .map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]))
+}
+
+pub fn extract(path: &Path, name: &Path, sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let f = std::fs::File::open(path)?;
+ // A malformed or truncated compound file is reported by `cfb` as an error,
+ // never a panic — so every failure below degrades to "junk", and a single
+ // unreadable legacy file never aborts a folder scan.
+ let mut comp = match cfb::CompoundFile::open(f) {
+ Ok(c) => c,
+ Err(_) => {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+ };
+
+ let Some(word) = read_stream(&mut comp, "WordDocument", MAX_STREAM_BYTES) else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+
+ let body = recover_text(&mut comp, &word).unwrap_or_default();
+ let body = body.trim();
+ if body.is_empty() {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+
+ let title = summary_title(&mut comp)
+ .filter(|t| !t.trim().is_empty())
+ .or_else(|| {
+ body.lines()
+ .find(|l| !l.trim().is_empty())
+ .map(|l| l.trim().chars().take(200).collect())
+ })
+ .unwrap_or_else(|| {
+ name.file_stem()
+ .map(|s| s.to_string_lossy().to_string())
+ .unwrap_or_else(|| "untitled".into())
+ });
+
+ emit_document(&title, &[], body, MAX_RECORDS_PER_FILE, sink, &mut stats);
+ Ok(stats)
+}
+
+/// Read a named stream (bounded) into memory, or `None` if it is absent or
+/// unreadable.
+fn read_stream<F: Read + std::io::Seek>(
+ comp: &mut cfb::CompoundFile<F>,
+ name: &str,
+ cap: u64,
+) -> Option<Vec<u8>> {
+ if !comp.is_stream(name) {
+ return None;
+ }
+ let stream = comp.open_stream(name).ok()?;
+ let mut buf = Vec::new();
+ stream.take(cap).read_to_end(&mut buf).ok()?;
+ Some(buf)
+}
+
+/// Recover the main document text via the FIB piece table. Returns `None` only
+/// when the FIB is too short to parse; a FIB with no usable CLX falls back to
+/// the raw-range reader (`fallback_text`).
+fn recover_text<F: Read + std::io::Seek>(
+ comp: &mut cfb::CompoundFile<F>,
+ word: &[u8],
+) -> Option<String> {
+ // FIB.base.flags1 (offset 0x0A): bit 0x0200 (fWhichTblStm) selects the
+ // table stream. fcClx / lcbClx sit at the fixed Word97 offsets 0x01A2 /
+ // 0x01A6 within the FIB (MS-DOC §2.5.6 FibRgFcLcb97 — the 67th/68th FC/LCB
+ // slot).
+ let flags1 = u16_at(word, 0x0A)?;
+ let table_name = if flags1 & 0x0200 != 0 {
+ "1Table"
+ } else {
+ "0Table"
+ };
+ let fc_clx = u32_at(word, 0x01A2)? as usize;
+ let lcb_clx = u32_at(word, 0x01A6)? as usize;
+
+ if lcb_clx > 0 {
+ // The table stream named by the flag, falling back to the other (some
+ // writers disagree with the flag).
+ let table = read_stream(comp, table_name, MAX_STREAM_BYTES)
+ .or_else(|| {
+ let other = if table_name == "1Table" {
+ "0Table"
+ } else {
+ "1Table"
+ };
+ read_stream(comp, other, MAX_STREAM_BYTES)
+ })
+ .unwrap_or_default();
+ if let Some(text) = text_from_piece_table(word, &table, fc_clx, lcb_clx) {
+ if !text.trim().is_empty() {
+ return Some(text);
+ }
+ }
+ }
+
+ // No piece table (older Word, or an unreadable CLX): read the raw main
+ // range as Windows-1252. fcMin/fcMac are at FIB offsets 0x0018 / 0x001C.
+ Some(fallback_text(word))
+}
+
+/// Walk the CLX in `table[fc_clx .. fc_clx+lcb_clx]`, then materialise each
+/// piece's characters out of `word`.
+fn text_from_piece_table(
+ word: &[u8],
+ table: &[u8],
+ fc_clx: usize,
+ lcb_clx: usize,
+) -> Option<String> {
+ let clx = table.get(fc_clx..fc_clx.checked_add(lcb_clx)?)?;
+
+ // The CLX is zero or more Prc blocks (clxt 0x01: a u16 byte count then that
+ // many bytes) followed by exactly one Pcdt (clxt 0x02: a u32 byte count then
+ // the PlcPcd). Walk to the Pcdt.
+ let mut i = 0usize;
+ let plc = loop {
+ match clx.get(i)? {
+ 0x01 => {
+ let cb = u16_at(clx, i + 1)? as usize;
+ i = i.checked_add(3)?.checked_add(cb)?;
+ }
+ 0x02 => {
+ let lcb = u32_at(clx, i + 1)? as usize;
+ let start = i.checked_add(5)?;
+ break clx.get(start..start.checked_add(lcb)?)?;
+ }
+ _ => return None,
+ }
+ };
+
+ // PlcPcd: (n+1) CPs (u32, 4 bytes) then n PCDs (8 bytes), so the total is
+ // 4·(n+1) + 8·n = 4 + 12·n bytes, i.e. n = (len - 4) / 12.
+ if plc.len() < 4 {
+ return None;
+ }
+ let n = (plc.len() - 4) / 12;
+ if n == 0 || n > MAX_PIECES {
+ return None;
+ }
+ let pcd_base = (n + 1) * 4;
+
+ let mut out = String::new();
+ for p in 0..n {
+ let cp_start = u32_at(plc, p * 4)? as usize;
+ let cp_end = u32_at(plc, (p + 1) * 4)? as usize;
+ let char_count = cp_end.checked_sub(cp_start)?;
+ if char_count == 0 {
+ continue;
+ }
+ // PCD: [0..2] flags, [2..6] FcCompressed, [6..8] prm.
+ let fc_field = u32_at(plc, pcd_base + p * 8 + 2)?;
+ let compressed = fc_field & 0x4000_0000 != 0;
+ let fc = (fc_field & 0x3FFF_FFFF) as usize;
+
+ if compressed {
+ // 8-bit: the real byte offset is fc/2, one byte per character.
+ let start = fc / 2;
+ let bytes = word.get(start..start.checked_add(char_count)?)?;
+ let (decoded, _, _) = encoding_rs::WINDOWS_1252.decode(bytes);
+ out.push_str(&decoded);
+ } else {
+ // 16-bit little-endian UTF-16, two bytes per character.
+ let start = fc;
+ let byte_len = char_count.checked_mul(2)?;
+ let bytes = word.get(start..start.checked_add(byte_len)?)?;
+ let units: Vec<u16> = bytes
+ .chunks_exact(2)
+ .map(|c| u16::from_le_bytes([c[0], c[1]]))
+ .collect();
+ out.push_str(&String::from_utf16_lossy(&units));
+ }
+ if out.len() >= MAX_BODY_CHARS {
+ break;
+ }
+ }
+ Some(clean_text(&out))
+}
+
+/// Last-resort reader for pre-piece-table documents: the main text range
+/// `[fcMin, fcMac)` of `WordDocument`, decoded as Windows-1252. Lossy by
+/// nature; returns an empty string when the range is unusable.
+fn fallback_text(word: &[u8]) -> String {
+ let (Some(fc_min), Some(fc_mac)) = (u32_at(word, 0x0018), u32_at(word, 0x001C)) else {
+ return String::new();
+ };
+ let (min, mac) = (fc_min as usize, fc_mac as usize);
+ if mac <= min {
+ return String::new();
+ }
+ let end = mac.min(min.saturating_add(MAX_BODY_CHARS));
+ let Some(bytes) = word.get(min..end.min(word.len())) else {
+ return String::new();
+ };
+ let (decoded, _, _) = encoding_rs::WINDOWS_1252.decode(bytes);
+ clean_text(&decoded)
+}
+
+/// Turn Word's in-stream control characters into clean prose: paragraph, line,
+/// page, cell and row marks become line breaks; field INSTRUCTION text (between
+/// field-begin 0x13 and field-separator 0x14) is dropped while the field RESULT
+/// is kept; and the remaining sub-0x20 control codes are discarded.
+fn clean_text(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ // Depth of nested field instruction regions we are currently inside.
+ let mut field_instr = 0u32;
+ for c in s.chars() {
+ match c {
+ '\u{13}' => field_instr += 1, // field begin → enter instruction
+ '\u{14}' => field_instr = field_instr.saturating_sub(1), // separator → result
+ '\u{15}' => {} // field end
+ _ if field_instr > 0 => {} // inside instruction: drop
+ // \r (0x0D) is the paragraph mark; 0x0B line break, 0x0C page
+ // break, 0x07 cell/row mark — all become line breaks.
+ '\r' | '\u{0B}' | '\u{0C}' | '\u{07}' => out.push('\n'),
+ '\u{1E}' => out.push('-'), // non-breaking hyphen
+ '\u{A0}' => out.push(' '), // non-breaking space
+ '\t' | '\n' => out.push(c),
+ _ if (c as u32) < 0x20 => {} // other control codes: drop
+ _ => out.push(c),
+ }
+ }
+ // Collapse the runs of blank lines the mark-to-newline mapping produces
+ // into paragraph breaks, and trim trailing whitespace on each line.
+ let mut collapsed = String::with_capacity(out.len());
+ let mut blank_run = 0u32;
+ for line in out.lines() {
+ let line = line.trim_end();
+ if line.is_empty() {
+ blank_run += 1;
+ continue;
+ }
+ if !collapsed.is_empty() {
+ collapsed.push_str(if blank_run > 0 { "\n\n" } else { "\n" });
+ }
+ blank_run = 0;
+ collapsed.push_str(line);
+ }
+ collapsed
+}
+
+/// Best-effort document title from the `\x05SummaryInformation` property set
+/// (MS-OLEPS): property id 2 (PIDSI_TITLE). Returns `None` on any shortfall.
+fn summary_title<F: Read + std::io::Seek>(comp: &mut cfb::CompoundFile<F>) -> Option<String> {
+ let s = read_stream(comp, "\u{5}SummaryInformation", MAX_SUMMARY_BYTES)?;
+ property_string(&s, 2)
+}
+
+/// Read a single string property (`pid`) out of the first section of a
+/// property-set stream. Handles VT_LPSTR (code-page bytes) and VT_LPWSTR
+/// (UTF-16). Everything is bounds-checked; `None` on any malformation.
+fn property_string(s: &[u8], pid: u32) -> Option<String> {
+ // PropertySetStream header: byteOrder(2) version(2) sysId(4) clsid(16)
+ // cSections(4) = 28 bytes, then the first FMTID(16) + section offset(4).
+ let section_off = u32_at(s, 44)? as usize;
+ let cb_section = u32_at(s, section_off)? as usize;
+ let num_props = u32_at(s, section_off + 4)? as usize;
+ // The section must fit, and the property index table must fit inside it.
+ let section_end = section_off.checked_add(cb_section)?;
+ if section_end > s.len() || num_props > 1 << 16 {
+ return None;
+ }
+
+ // Find the code page (pid 1, VT_I2) for VT_LPSTR decoding; default 1252.
+ let mut code_page: u16 = 1252;
+ let mut target: Option<usize> = None;
+ for i in 0..num_props {
+ let entry = section_off + 8 + i * 8;
+ let this_pid = u32_at(s, entry)?;
+ let value_off = section_off + u32_at(s, entry + 4)? as usize;
+ if this_pid == 1 {
+ // VT_I2 code page: type(4) then i16.
+ if u32_at(s, value_off) == Some(2) {
+ if let Some(cp) = u16_at(s, value_off + 4) {
+ code_page = cp;
+ }
+ }
+ }
+ if this_pid == pid {
+ target = Some(value_off);
+ }
+ }
+
+ let value_off = target?;
+ let vt = u32_at(s, value_off)?;
+ let len = u32_at(s, value_off + 4)? as usize;
+ let data_off = value_off + 8;
+ match vt {
+ 0x1E => {
+ // VT_LPSTR: `len` bytes incl terminating NUL, code-page encoded.
+ let bytes = s.get(data_off..data_off.checked_add(len)?)?;
+ let bytes = bytes.split(|&b| b == 0).next().unwrap_or(bytes);
+ let enc = encoding_rs::Encoding::for_label(format!("windows-{code_page}").as_bytes())
+ .unwrap_or(encoding_rs::WINDOWS_1252);
+ let (decoded, _, _) = enc.decode(bytes);
+ Some(decoded.trim().to_string())
+ }
+ 0x1F => {
+ // VT_LPWSTR: `len` UTF-16 code units incl NUL.
+ let byte_len = len.checked_mul(2)?;
+ let bytes = s.get(data_off..data_off.checked_add(byte_len)?)?;
+ let mut units: Vec<u16> = bytes
+ .chunks_exact(2)
+ .map(|c| u16::from_le_bytes([c[0], c[1]]))
+ .collect();
+ if let Some(pos) = units.iter().position(|&u| u == 0) {
+ units.truncate(pos);
+ }
+ Some(String::from_utf16_lossy(&units).trim().to_string())
+ }
+ _ => None,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn clean_text_maps_marks_and_drops_field_instructions() {
+ // A paragraph mark, then an inline field `{ HYPERLINK "x" }result`: the
+ // field RESULT ("click here") is kept inline where it sits and the
+ // instruction source is dropped.
+ let raw = "Hello\u{0D}See \u{13}HYPERLINK \"http://x\"\u{14}click here\u{15}\u{0D}End";
+ let cleaned = clean_text(raw);
+ assert_eq!(cleaned, "Hello\nSee click here\nEnd");
+ assert!(
+ !cleaned.contains("HYPERLINK"),
+ "field source must be dropped"
+ );
+ }
+
+ #[test]
+ fn clean_text_collapses_blank_runs_and_trims() {
+ let raw = "A\u{0D}\u{0D}\u{0D}B \u{0D}C";
+ assert_eq!(clean_text(raw), "A\n\nB\nC");
+ }
+
+ #[test]
+ fn field_reads_are_bounds_checked() {
+ assert_eq!(u16_at(&[1], 0), None);
+ assert_eq!(u32_at(&[1, 2, 3], 0), None);
+ assert_eq!(u32_at(&[1, 2, 3, 4], 0), Some(0x04030201));
+ }
+
+ /// A piece table with one compressed (Windows-1252) and one uncompressed
+ /// (UTF-16) piece, assembled by hand, round-trips to the expected text.
+ /// This is the core of the extractor and is exercised without needing a
+ /// real BIFF file on disk.
+ #[test]
+ fn piece_table_decodes_compressed_and_uncompressed_pieces() {
+ // WordDocument: FIB header padded to 0x0200, then the text region.
+ // Piece 0 (compressed, 5 chars "Hello") at byte 0x200.
+ // Piece 1 (uncompressed UTF-16, " wörld") right after.
+ let mut word = vec![0u8; 0x200];
+ let p0_bytes = b"Hello"; // CP1252, 1 byte/char
+ let p0_off = 0x200usize;
+ word.extend_from_slice(p0_bytes);
+ let p1_text = " wörld";
+ let p1_off = word.len();
+ for u in p1_text.encode_utf16() {
+ word.extend_from_slice(&u.to_le_bytes());
+ }
+
+ // Build the table stream: CLX = Pcdt (0x02) + lcb + PlcPcd.
+ // CPs: [0, 5, 5+6] ; PCDs: piece0 fc=p0_off*2 | compressed, piece1 fc=p1_off.
+ let p0_chars = p0_bytes.len() as u32; // 5
+ let p1_chars = p1_text.chars().count() as u32; // 6
+ let mut plc = Vec::new();
+ for cp in [0u32, p0_chars, p0_chars + p1_chars] {
+ plc.extend_from_slice(&cp.to_le_bytes());
+ }
+ // PCD 0: flags(2)=0, fc(4)= (p0_off*2) | 0x40000000, prm(2)=0
+ let fc0 = (p0_off as u32 * 2) | 0x4000_0000;
+ plc.extend_from_slice(&[0, 0]);
+ plc.extend_from_slice(&fc0.to_le_bytes());
+ plc.extend_from_slice(&[0, 0]);
+ // PCD 1: fc = p1_off (uncompressed)
+ plc.extend_from_slice(&[0, 0]);
+ plc.extend_from_slice(&(p1_off as u32).to_le_bytes());
+ plc.extend_from_slice(&[0, 0]);
+
+ let mut clx = vec![0x02u8];
+ clx.extend_from_slice(&(plc.len() as u32).to_le_bytes());
+ clx.extend_from_slice(&plc);
+
+ // Table stream: put the CLX at offset 0 so fc_clx = 0.
+ let table = clx.clone();
+ let text = text_from_piece_table(&word, &table, 0, clx.len()).unwrap();
+ assert_eq!(text, "Hello wörld");
+ }
+
+ #[test]
+ fn property_string_reads_a_vt_lpstr_title() {
+ // Minimal SummaryInformation with one section holding pid 2 = "Budget".
+ // Layout: 28-byte header, FMTID(16)+sectionOffset(4) at 28..48.
+ let title = b"Budget\0";
+ let mut s = vec![0u8; 48];
+ // byteOrder 0xFFFE, version 0, cSections 1
+ s[0] = 0xFE;
+ s[1] = 0xFF;
+ let section_off = 48u32;
+ s[44..48].copy_from_slice(&section_off.to_le_bytes());
+ // Section: cbSection, cProperties=1, then (pid=2, off=16), then value.
+ // Property table entry sits at section_off+8; value at section_off+16.
+ let value_rel = 16u32;
+ let mut section = Vec::new();
+ // cbSection filled in after; cProperties
+ section.extend_from_slice(&[0, 0, 0, 0]); // placeholder cbSection
+ section.extend_from_slice(&1u32.to_le_bytes()); // cProperties
+ section.extend_from_slice(&2u32.to_le_bytes()); // pid 2
+ section.extend_from_slice(&value_rel.to_le_bytes()); // value offset
+ // value: VT_LPSTR (0x1E), len, bytes
+ section.extend_from_slice(&0x1Eu32.to_le_bytes());
+ section.extend_from_slice(&(title.len() as u32).to_le_bytes());
+ section.extend_from_slice(title);
+ let cb = section.len() as u32;
+ section[0..4].copy_from_slice(&cb.to_le_bytes());
+ s.extend_from_slice(&section);
+
+ assert_eq!(property_string(&s, 2).as_deref(), Some("Budget"));
+ assert_eq!(property_string(&s, 99), None, "absent pid yields None");
+ }
+
+ #[test]
+ fn malformed_clx_never_panics() {
+ // Truncated / nonsense CLX bytes must return None, not panic.
+ let word = vec![0u8; 0x400];
+ for clx in [
+ vec![0x02, 0xFF, 0xFF, 0xFF, 0xFF], // lcb far past end
+ vec![0x01, 0x10, 0x00], // Prc with no body
+ vec![0x09], // unknown clxt
+ vec![], // empty
+ ] {
+ let _ = text_from_piece_table(&word, &clx, 0, clx.len());
+ }
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index 774878ec9..6c17ac7ff 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -5,6 +5,7 @@
pub mod bvh;
pub mod code;
pub mod csv_x;
+pub mod doc;
pub mod docx;
pub mod eml;
pub mod epub;
@@ -22,6 +23,7 @@ pub mod sqldump;
pub mod sqlite_x;
pub mod txt;
pub mod unity;
+pub mod xls;
pub mod xlsx;
pub mod xml_x;
pub mod yaml_x;
@@ -214,6 +216,7 @@ pub fn extract(
Family::Eml => eml::extract(path, sn.gzip, sink),
Family::Mbox => mbox::extract(path, sn.gzip, limit_bytes, sink),
Family::Docx => docx::extract(path, sink),
+ Family::Doc => doc::extract(path, sn.logical_name.as_deref().unwrap_or(path), sink),
Family::Pptx => pptx::extract(path, sn.logical_name.as_deref().unwrap_or(path), sink),
Family::Epub => epub::extract(path, sn.logical_name.as_deref().unwrap_or(path), sink),
Family::Ipynb => ipynb::extract(
@@ -234,6 +237,12 @@ pub fn extract(
limit_bytes.map(|_| 500),
sink,
),
+ Family::Xls => xls::extract(
+ path,
+ sn.logical_name.as_deref().unwrap_or(path),
+ limit_bytes.map(|_| 500),
+ sink,
+ ),
Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),
Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),
Family::Code => code::extract(path, sn, sink),
diff --git a/engine/crates/xerj-autoindex/src/extract/xls.rs b/engine/crates/xerj-autoindex/src/extract/xls.rs
new file mode 100644
index 000000000..337c8ebf2
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/extract/xls.rs
@@ -0,0 +1,590 @@
+//! XLS — legacy Excel 97-2003 workbooks (BIFF8 inside an OLE2 compound file),
+//! read as tables on the same record model as [`super::xlsx`].
+//!
+//! Each worksheet is its own dataset (`RawRecord.group` = sheet name) and each
+//! row below the sheet's header row is one record keyed by the header cells,
+//! with values carrying the type the cell stored: numbers stay numbers,
+//! booleans stay booleans, and a date/time cell becomes an ISO-8601 string so
+//! type inference elects `date` rather than seeing a serial number. The header
+//! is the first row (among the first [`HEADER_SCAN_ROWS`] non-empty rows) with
+//! at least two cells, all text or date and at least one text, that is not
+//! immediately followed by a wider row of the same kind; rows above it are not
+//! indexed. A sheet with no such header is indexed as a document, one line per
+//! row with cells joined by ` | `.
+//!
+//! Reading is delegated to calamine, which ships its own OLE2/BIFF reader. The
+//! heavy lifting of BIFF record parsing, the shared-string table and the
+//! legacy 8-bit cell codepage all live there; this module only maps calamine's
+//! `Data` cells onto the extractor's typed record model.
+//!
+//! Differences from the XLSX extractor, by design: calamine materialises a
+//! whole worksheet as a `Range` (the BIFF format caps a sheet at 65,536 rows ×
+//! 256 columns, and calamine reads it in one pass), so there is no streaming
+//! row cap of our own — the per-sheet sample limit bounds how many rows we
+//! EMIT, and [`MAX_ROWS_PER_SHEET`] bounds how many we even look at against a
+//! crafted dimension record. Vertical merge fill-down is not applied (`.xls`
+//! merge ranges are not surfaced on calamine's `Range`); a category label
+//! merged down a column therefore appears only on its first row.
+
+use super::{
+ emit_document_with_fields, sanitize_field_name, ExtractStats, FieldOrigin, RawRecord, Sink,
+ MAX_FIELDS_PER_RECORD,
+};
+use anyhow::Result;
+use calamine::{Data, DataType, Reader, Xls};
+use chrono::Timelike;
+use serde_json::{Map, Value};
+use std::path::Path;
+
+/// Non-empty rows searched for a header before the sheet is treated as a
+/// document (mirrors the XLSX extractor).
+const HEADER_SCAN_ROWS: usize = 20;
+
+/// Defensive cap on rows inspected per sheet, in case a crafted BIFF dimension
+/// record claims a range far larger than the format's own 65,536-row ceiling.
+const MAX_ROWS_PER_SHEET: usize = 1 << 20;
+
+/// Text kept from one sheet indexed as a document.
+const MAX_DOC_SHEET_BYTES: usize = 16 << 20;
+
+/// `name` is the file as the corpus names it (under durable preparation this is
+/// not `path`, a sealed snapshot blob); document sheets are titled from it.
+/// `per_sheet_limit` caps the rows emitted from each sheet (phase-A sampling);
+/// `None` reads every row.
+pub fn extract(
+ path: &Path,
+ name: &Path,
+ per_sheet_limit: Option<u64>,
+ sink: Sink,
+) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let f = std::fs::File::open(path)?;
+ // calamine reports a malformed/truncated workbook as an error; never a
+ // panic. Treat any such failure as junk rather than propagating — a single
+ // unreadable legacy file must not abort a folder scan.
+ let mut wb = match Xls::new(f) {
+ Ok(wb) => wb,
+ Err(_) => {
+ stats.junk += 1;
+ return Ok(stats);
+ }
+ };
+
+ let stem = name
+ .file_stem()
+ .map(|s| s.to_string_lossy().to_string())
+ .unwrap_or_else(|| "untitled".into());
+
+ let sheet_names = wb.sheet_names().to_vec();
+ for sheet_name in sheet_names {
+ let Ok(range) = wb.worksheet_range(&sheet_name) else {
+ continue;
+ };
+ let start_row = range.start().map(|(r, _)| r).unwrap_or(0);
+ let start_col = range.start().map(|(_, c)| c).unwrap_or(0);
+
+ // Build the non-empty rows, each carrying 1-based column numbers so the
+ // header/column machinery matches the XLSX extractor's.
+ let mut rows: Vec<Row> = Vec::new();
+ for (i, cells) in range.rows().take(MAX_ROWS_PER_SHEET).enumerate() {
+ let mut out = Vec::new();
+ for (j, data) in cells.iter().enumerate() {
+ if let Some(cell) = convert_cell(start_col + j as u32 + 1, data) {
+ out.push(cell);
+ }
+ }
+ if !out.is_empty() {
+ rows.push(Row {
+ num: start_row + i as u32 + 1,
+ cells: out,
+ });
+ }
+ }
+ if rows.is_empty() {
+ continue;
+ }
+
+ if let Some(h) = find_header(&rows) {
+ if !emit_table(&sheet_name, &rows, h, per_sheet_limit, sink, &mut stats) {
+ break;
+ }
+ } else if !emit_document_sheet(&sheet_name, &stem, &rows, sink, &mut stats) {
+ break;
+ }
+ }
+
+ if stats.records == 0 {
+ stats.junk += 1;
+ }
+ Ok(stats)
+}
+
+/// A cell's typed kind, mirroring the XLSX extractor so header detection and
+/// type inference agree across the two formats.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Kind {
+ Text,
+ Number,
+ Bool,
+ Date,
+}
+
+#[derive(Debug, Clone)]
+struct Cell {
+ /// 1-based column (A = 1).
+ col: u32,
+ value: Value,
+ kind: Kind,
+}
+
+impl Cell {
+ fn display(&self) -> String {
+ match &self.value {
+ Value::String(s) => s.clone(),
+ Value::Bool(true) => "TRUE".into(),
+ Value::Bool(false) => "FALSE".into(),
+ v => v.to_string(),
+ }
+ }
+}
+
+#[derive(Debug)]
+struct Row {
+ /// 1-based row number, as Excel shows it.
+ num: u32,
+ cells: Vec<Cell>,
+}
+
+/// Map one calamine cell onto the typed model; `None` for an empty cell or an
+/// error value (`#N/A`, `#DIV/0!`), exactly as the XLSX extractor drops them.
+fn convert_cell(col: u32, data: &Data) -> Option<Cell> {
+ match data {
+ Data::Empty | Data::Error(_) => None,
+ Data::String(s) => {
+ let s = s.trim();
+ (!s.is_empty()).then(|| Cell {
+ col,
+ value: Value::String(s.to_string()),
+ kind: Kind::Text,
+ })
+ }
+ Data::Int(i) => Some(Cell {
+ col,
+ value: Value::Number((*i).into()),
+ kind: Kind::Number,
+ }),
+ Data::Float(f) => Some(Cell {
+ col,
+ value: number_value(*f)?,
+ kind: Kind::Number,
+ }),
+ Data::Bool(b) => Some(Cell {
+ col,
+ value: Value::Bool(*b),
+ kind: Kind::Bool,
+ }),
+ Data::DateTimeIso(s) => {
+ let s = s.trim();
+ (!s.is_empty()).then(|| Cell {
+ col,
+ value: Value::String(s.to_string()),
+ kind: Kind::Date,
+ })
+ }
+ Data::DurationIso(s) => {
+ let s = s.trim();
+ (!s.is_empty()).then(|| Cell {
+ col,
+ value: Value::String(s.to_string()),
+ kind: Kind::Text,
+ })
+ }
+ Data::DateTime(_) => date_cell(col, data),
+ }
+}
+
+/// A calamine `Data::DateTime` rendered to the same ISO-8601 strings the XLSX
+/// extractor emits: a bare date at midnight, a `T`-joined datetime otherwise,
+/// and `HH:MM:SS` for a duration/time-of-day. Falls back to the raw serial
+/// number if calamine cannot resolve it to a chrono value.
+fn date_cell(col: u32, data: &Data) -> Option<Cell> {
+ // A time of day (fraction of a day with no date part) resolves to a
+ // `NaiveTime` but not a sensible `NaiveDateTime`; check it first.
+ if let Some(dt) = data.as_datetime() {
+ let iso = if dt.num_seconds_from_midnight() == 0 && dt.nanosecond() == 0 {
+ dt.format("%Y-%m-%d").to_string()
+ } else if dt.nanosecond() == 0 {
+ dt.format("%Y-%m-%dT%H:%M:%S").to_string()
+ } else {
+ dt.format("%Y-%m-%dT%H:%M:%S%.3f").to_string()
+ };
+ return Some(Cell {
+ col,
+ value: Value::String(iso),
+ kind: Kind::Date,
+ });
+ }
+ if let Some(t) = data.as_time() {
+ return Some(Cell {
+ col,
+ value: Value::String(t.format("%H:%M:%S").to_string()),
+ kind: Kind::Date,
+ });
+ }
+ // Could not resolve a calendar value: keep the serial number as a number so
+ // nothing is dropped.
+ let f = data.as_f64()?;
+ Some(Cell {
+ col,
+ value: number_value(f)?,
+ kind: Kind::Number,
+ })
+}
+
+/// Integral floats inside the i64-exact range become integers (so `1250.0`
+/// indexes as `1250`), matching the XLSX extractor's `number_value`.
+fn number_value(f: f64) -> Option<Value> {
+ if !f.is_finite() {
+ return None;
+ }
+ if f.fract() == 0.0 && f.abs() < 9_007_199_254_740_992.0 {
+ Some(Value::Number((f as i64).into()))
+ } else {
+ serde_json::Number::from_f64(f).map(Value::Number)
+ }
+}
+
+/// Column letters of a column number (28 → `AB`).
+fn column_letters(mut col: u32) -> String {
+ let mut out = Vec::new();
+ while col > 0 {
+ let rem = (col - 1) % 26;
+ out.push(b'A' + rem as u8);
+ col = (col - 1) / 26;
+ }
+ out.reverse();
+ String::from_utf8(out).unwrap_or_default()
+}
+
+/// Whether a row can be a header: two or more cells, some text, and no numbers
+/// or booleans (date cells are allowed, for month-column headers).
+fn is_header_candidate(row: &Row) -> bool {
+ row.cells.len() >= 2
+ && row
+ .cells
+ .iter()
+ .all(|c| matches!(c.kind, Kind::Text | Kind::Date))
+ && row.cells.iter().any(|c| c.kind == Kind::Text)
+}
+
+/// Index of the header row. A candidate followed by a WIDER candidate is a
+/// title line above the real header and is passed over. Only the first
+/// [`HEADER_SCAN_ROWS`] non-empty rows are considered.
+fn find_header(rows: &[Row]) -> Option<usize> {
+ let scan = rows.len().min(HEADER_SCAN_ROWS);
+ rows[..scan].windows(2).position(|w| {
+ is_header_candidate(&w[0])
+ && !(is_header_candidate(&w[1]) && w[1].cells.len() > w[0].cells.len())
+ })
+}
+
+/// Emit every data row under `header_idx` as a typed record keyed by the header
+/// cells. Returns false if the sink asked to stop the whole file.
+fn emit_table(
+ sheet: &str,
+ rows: &[Row],
+ header_idx: usize,
+ limit: Option<u64>,
+ sink: Sink,
+ stats: &mut ExtractStats,
+) -> bool {
+ let header = &rows[header_idx];
+ let mut seen = std::collections::HashSet::new();
+ let mut names: std::collections::HashMap<u32, String> = std::collections::HashMap::new();
+ for c in &header.cells {
+ let mut name = sanitize_field_name(&c.display());
+ while !seen.insert(name.clone()) {
+ name.push_str("_2");
+ }
+ names.insert(c.col, name);
+ }
+ let header_cells: Vec<(u32, String)> =
+ header.cells.iter().map(|c| (c.col, c.display())).collect();
+
+ let mut emitted = 0u64;
+ for row in &rows[header_idx + 1..] {
+ // A repeated header row (common when a sheet stacks sections) is not a
+ // data record.
+ let row_text: Vec<(u32, String)> = row.cells.iter().map(|c| (c.col, c.display())).collect();
+ if row_text == header_cells {
+ continue;
+ }
+ let mut fields = Map::new();
+ for c in &row.cells {
+ if fields.len() >= MAX_FIELDS_PER_RECORD {
+ break;
+ }
+ let name = names
+ .get(&c.col)
+ .cloned()
+ .unwrap_or_else(|| format!("col_{}", column_letters(c.col)));
+ fields.insert(name, c.value.clone());
+ }
+ stats.records += 1;
+ if !sink(RawRecord {
+ fields,
+ locator: format!("{sheet}!r{}", row.num),
+ group: Some(sheet.to_string()),
+ origin: FieldOrigin::Data,
+ }) {
+ return false;
+ }
+ emitted += 1;
+ if limit.is_some_and(|lim| emitted >= lim) {
+ break;
+ }
+ }
+ true
+}
+
+/// Emit a sheet with no detectable header as a document: one line per row,
+/// cells joined by ` | `. Returns false if the sink asked to stop.
+fn emit_document_sheet(
+ sheet: &str,
+ stem: &str,
+ rows: &[Row],
+ sink: Sink,
+ stats: &mut ExtractStats,
+) -> bool {
+ let mut body = String::new();
+ let mut last_row = 0u32;
+ for row in rows {
+ let line = row
+ .cells
+ .iter()
+ .map(Cell::display)
+ .collect::<Vec<_>>()
+ .join(" | ");
+ if body.len() + line.len() + 2 > MAX_DOC_SHEET_BYTES {
+ stats.truncated = true;
+ break;
+ }
+ if !body.is_empty() {
+ // A gap in row numbers is a paragraph break, where section
+ // splitting prefers to cut.
+ body.push_str(if row.num > last_row + 1 { "\n\n" } else { "\n" });
+ }
+ body.push_str(&line);
+ last_row = row.num;
+ }
+ if body.trim().is_empty() {
+ return true;
+ }
+ let mut base = Map::new();
+ base.insert("sheet".into(), Value::String(sheet.to_string()));
+ emit_document_with_fields(
+ &base,
+ &format!("{stem} — {sheet}"),
+ &body,
+ sheet,
+ sink,
+ stats,
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn fields(r: &RawRecord) -> Value {
+ Value::Object(r.fields.clone())
+ }
+
+ /// Build a one-sheet `.xls` in a tempdir with `rustexcel`-free, synthetic
+ /// content by driving calamine's own writer? calamine has no writer, so the
+ /// unit tests here exercise the typed-model mapping and the table/document
+ /// routing directly on synthetic rows. The end-to-end round-trip through a
+ /// real BIFF file is covered by `extract_real_xls_fixture` (gated on a
+ /// generated fixture), so these pin the behaviour deterministically.
+ fn row(num: u32, cells: &[Cell]) -> Row {
+ Row {
+ num,
+ cells: cells.to_vec(),
+ }
+ }
+ fn txt(col: u32, s: &str) -> Cell {
+ Cell {
+ col,
+ value: Value::String(s.into()),
+ kind: Kind::Text,
+ }
+ }
+ fn num(col: u32, v: i64) -> Cell {
+ Cell {
+ col,
+ value: Value::Number(v.into()),
+ kind: Kind::Number,
+ }
+ }
+ fn boolean(col: u32, v: bool) -> Cell {
+ Cell {
+ col,
+ value: Value::Bool(v),
+ kind: Kind::Bool,
+ }
+ }
+
+ fn collect_table(sheet: &str, rows: &[Row]) -> (ExtractStats, Vec<RawRecord>) {
+ let h = find_header(rows).expect("a header should be found");
+ let mut recs = Vec::new();
+ let mut stats = ExtractStats::default();
+ emit_table(
+ sheet,
+ rows,
+ h,
+ None,
+ &mut |r| {
+ recs.push(r);
+ true
+ },
+ &mut stats,
+ );
+ (stats, recs)
+ }
+
+ #[test]
+ fn convert_cell_types_mirror_the_xlsx_model() {
+ assert!(convert_cell(1, &Data::Empty).is_none());
+ assert!(convert_cell(1, &Data::Error(calamine::CellErrorType::Div0)).is_none());
+ // An integral float indexes as an integer, not 1250.0.
+ assert_eq!(
+ convert_cell(3, &Data::Float(1250.0)).unwrap().value,
+ json!(1250)
+ );
+ assert_eq!(
+ convert_cell(3, &Data::Float(980.5)).unwrap().value,
+ json!(980.5)
+ );
+ assert_eq!(convert_cell(4, &Data::Bool(true)).unwrap().kind, Kind::Bool);
+ // A whitespace-only string is dropped like an empty cell.
+ assert!(convert_cell(1, &Data::String(" ".into())).is_none());
+ }
+
+ #[test]
+ fn a_title_row_is_skipped_and_each_row_under_the_header_is_a_typed_record() {
+ // Row 1 is a single-cell title; row 3 the header; 4-5 the data.
+ let rows = vec![
+ row(1, &[txt(1, "Q3 Sales Report")]),
+ row(
+ 3,
+ &[
+ txt(1, "Region"),
+ txt(2, "Month"),
+ txt(3, "Revenue"),
+ txt(4, "Closed"),
+ ],
+ ),
+ row(
+ 4,
+ &[
+ txt(1, "EMEA"),
+ txt(2, "July"),
+ num(3, 1250),
+ boolean(4, true),
+ ],
+ ),
+ row(
+ 5,
+ &[
+ txt(1, "APAC"),
+ txt(2, "August"),
+ num(3, 980),
+ boolean(4, false),
+ ],
+ ),
+ ];
+ let (stats, recs) = collect_table("Sales", &rows);
+ assert_eq!((stats.records, stats.junk), (2, 0));
+ assert_eq!(recs[0].group.as_deref(), Some("Sales"));
+ assert_eq!(recs[0].locator, "Sales!r4");
+ assert_eq!(recs[0].origin, FieldOrigin::Data);
+ assert_eq!(
+ fields(&recs[0]),
+ json!({"Region": "EMEA", "Month": "July", "Revenue": 1250, "Closed": true})
+ );
+ assert_eq!(
+ fields(&recs[1]),
+ json!({"Region": "APAC", "Month": "August", "Revenue": 980, "Closed": false})
+ );
+ }
+
+ #[test]
+ fn a_repeated_header_row_is_not_a_data_record() {
+ let rows = vec![
+ row(1, &[txt(1, "Name"), txt(2, "Score")]),
+ row(2, &[txt(1, "Ann"), num(2, 10)]),
+ row(3, &[txt(1, "Name"), txt(2, "Score")]), // stacked header again
+ row(4, &[txt(1, "Bob"), num(2, 20)]),
+ ];
+ let (stats, recs) = collect_table("S", &rows);
+ assert_eq!(stats.records, 2);
+ assert_eq!(fields(&recs[0]), json!({"Name": "Ann", "Score": 10}));
+ assert_eq!(fields(&recs[1]), json!({"Name": "Bob", "Score": 20}));
+ }
+
+ #[test]
+ fn duplicate_header_names_are_disambiguated() {
+ let rows = vec![
+ row(1, &[txt(1, "Amount"), txt(2, "Amount")]),
+ row(2, &[num(1, 1), num(2, 2)]),
+ ];
+ let (_stats, recs) = collect_table("S", &rows);
+ let obj = recs[0].fields.clone();
+ assert!(obj.contains_key("Amount") && obj.contains_key("Amount_2"));
+ }
+
+ #[test]
+ fn a_sheet_with_no_header_is_emitted_as_a_document() {
+ // Every row leads with a number, so no row is a header candidate.
+ let rows = vec![
+ row(1, &[num(1, 1), txt(2, "alpha")]),
+ row(2, &[num(1, 2), txt(2, "beta")]),
+ ];
+ assert!(find_header(&rows).is_none());
+ let mut recs = Vec::new();
+ let mut stats = ExtractStats::default();
+ emit_document_sheet(
+ "Notes",
+ "book",
+ &rows,
+ &mut |r| {
+ recs.push(r);
+ true
+ },
+ &mut stats,
+ );
+ assert_eq!(stats.records, 1);
+ let body = recs[0].fields["body"].as_str().unwrap();
+ assert!(body.contains("1 | alpha") && body.contains("2 | beta"));
+ assert_eq!(recs[0].fields["title"], "book — Notes");
+ assert_eq!(recs[0].fields["sheet"], "Notes");
+ }
+
+ #[test]
+ fn date_cell_formats_match_the_xlsx_extractor() {
+ // calamine resolves an ExcelDateTime; check the bare-date vs datetime
+ // split against a serial we can predict (1904-01-01 is serial 1462 in
+ // the 1900 system; use a known midnight serial instead).
+ use calamine::ExcelDateTime;
+ // 2025-07-01 00:00:00 — serial 45839 in the 1900 date system.
+ let d = Data::DateTime(ExcelDateTime::new(
+ 45839.0,
+ calamine::ExcelDateTimeType::DateTime,
+ false,
+ ));
+ let cell = date_cell(1, &d).unwrap();
+ assert_eq!(cell.kind, Kind::Date);
+ assert_eq!(cell.value, json!("2025-07-01"));
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index e89b1fea2..9c20cbb47 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -1841,6 +1841,7 @@ fn sample_limit_bytes(family: Family, path: &Path) -> Option<u64> {
Family::Mbox => Some(SAMPLE_LIMIT_BYTES),
Family::Sqlite => Some(1), // signals per-table row cap inside the extractor
Family::Xlsx => Some(1), // likewise a per-sheet row cap
+ Family::Xls => Some(1), // legacy workbook: same per-sheet row cap
_ => None, // whole-file extractors cap themselves
}
}
@@ -2663,6 +2664,7 @@ fn scan_file(
| Family::TxtProse
| Family::Pdf
| Family::Docx
+ | Family::Doc
| Family::Pptx
| Family::Epub
| Family::Ipynb
@@ -2692,7 +2694,7 @@ fn scan_file(
// untyped.
let grouped_family = matches!(
sn.family,
- Family::SqlDump | Family::Sqlite | Family::Xlsx | Family::UnityYaml
+ Family::SqlDump | Family::Sqlite | Family::Xlsx | Family::Xls | Family::UnityYaml
);
let mut sink = |rec: extract::RawRecord| -> bool {
let entry = groups.entry(rec.group.clone()).or_default();
diff --git a/engine/crates/xerj-autoindex/src/order.rs b/engine/crates/xerj-autoindex/src/order.rs
index 35bb18413..dac58ab90 100644
--- a/engine/crates/xerj-autoindex/src/order.rs
+++ b/engine/crates/xerj-autoindex/src/order.rs
@@ -186,6 +186,7 @@ pub fn band(rel: &str, family: Family) -> Band {
| Family::Html
| Family::Pdf
| Family::Docx
+ | Family::Doc
| Family::Pptx
| Family::Epub
| Family::Ipynb
@@ -193,7 +194,12 @@ pub fn band(rel: &str, family: Family) -> Band {
| Family::Eml
| Family::Mbox => Band::SourceAndDocs,
Family::Yaml | Family::Json | Family::Xml => Band::Config,
- Family::Csv | Family::Jsonl | Family::Sqlite | Family::SqlDump | Family::Xlsx => Band::Data,
+ Family::Csv
+ | Family::Jsonl
+ | Family::Sqlite
+ | Family::SqlDump
+ | Family::Xlsx
+ | Family::Xls => Band::Data,
Family::Logs | Family::TxtLines => Band::Bulk,
// Unity assets are the project's own authored content — scenes,
// prefabs and materials are to a Unity repo what source files are to
@@ -227,10 +233,10 @@ pub fn band_from_family_str(rel: &str, family: &str) -> Band {
// docs, so a RESUMED run queued every email behind the bulk data the
// planning run had put it ahead of. The agreement test did not catch it
// because its family list was written by hand and omitted `Eml` too.
- "code" | "txt-prose" | "html" | "pdf" | "docx" | "pptx" | "epub" | "ipynb" | "man"
- | "unity" | "eml" | "mbox" => Band::SourceAndDocs,
+ "code" | "txt-prose" | "html" | "pdf" | "docx" | "doc" | "pptx" | "epub" | "ipynb"
+ | "man" | "unity" | "eml" | "mbox" => Band::SourceAndDocs,
"yaml" | "json" | "xml" | "unity-meta" => Band::Config,
- "csv" | "jsonl" | "sqlite" | "sqldump" | "xlsx" | "bvh" => Band::Data,
+ "csv" | "jsonl" | "sqlite" | "sqldump" | "xlsx" | "xls" | "bvh" => Band::Data,
// Pre-existing drift, caught by
// `estimate::tests::the_two_band_functions_agree_for_every_family`:
// `band` ranks Binary `Vendored`, but the catch-all below ranked the
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index bbb522b58..56c93a958 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -34,6 +34,18 @@ pub enum Family {
/// Excel workbook: one dataset per worksheet, one record per row under
/// the sheet's header row, typed from the stored cell values.
Xlsx,
+ /// Legacy Excel 97-2003 workbook (`.xls`, BIFF8 inside an OLE2 compound
+ /// file): same table model as [`Family::Xlsx`] — one dataset per
+ /// worksheet, one record per row under the header row — read through
+ /// calamine. Detected by the OLE2 magic plus a `Workbook`/`Book` stream,
+ /// never by extension.
+ Xls,
+ /// Legacy Word 97-2003 document (`.doc`, inside an OLE2 compound file):
+ /// the WordDocument stream's prose, recovered through the FIB piece table
+ /// (CLX in the 0Table/1Table stream), emitted as section records with the
+ /// title taken from SummaryInformation when present. Detected by the OLE2
+ /// magic plus a `WordDocument` stream, never by extension.
+ Doc,
/// EPUB e-book: chapters in reading order (spine documents, cut at their
/// table-of-contents anchors), with the book's metadata and each
/// chapter's title from the table of contents.
@@ -85,6 +97,8 @@ impl Family {
Family::Docx => "docx",
Family::Pptx => "pptx",
Family::Xlsx => "xlsx",
+ Family::Xls => "xls",
+ Family::Doc => "doc",
Family::Epub => "epub",
Family::Ipynb => "ipynb",
Family::Man => "man",
@@ -104,6 +118,7 @@ impl Family {
self,
Family::Pdf
| Family::Docx
+ | Family::Doc
| Family::Pptx
| Family::Epub
| Family::Ipynb
@@ -143,6 +158,10 @@ pub struct Sniffed {
/// Bytes read for ordinary classification.
const PREFIX: usize = 8192;
+/// OLE2 / Compound File Binary header signature (MS-CFB §2.2). Legacy Office
+/// (`.doc`/`.xls`/`.ppt`), 1C `.epf`, MSI and others all begin with it.
+const OLE2_MAGIC: &[u8] = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1";
+
/// Bytes read when the file already carries GIF magic. A Comment Extension is
/// unbounded by the format, so this is a budget rather than a guarantee: it
/// covers every commented GIF in the measured corpus with room to spare.
@@ -270,6 +289,31 @@ fn sniff_bytes(
s.binary_kind = Some(kind.into());
return Ok(s);
}
+ // OLE2 / Compound File Binary (`.doc`, `.xls`, `.ppt`, 1C `.epf`, MSI, …):
+ // one 8-byte magic for a whole family of unrelated formats, so — exactly as
+ // with the zip container above — the magic alone cannot say which. Open the
+ // compound file (bounded: `cfb` reads the header, FAT and directory, never
+ // the whole file) and route by the top-level stream names a `Workbook`/
+ // `Book` stream is a legacy Excel workbook, a `WordDocument` stream a legacy
+ // Word document, anything else an OLE2 blob we name but do not parse. A
+ // crafted or truncated container must not be able to panic or hang the
+ // sniffer, so `cfb::CompoundFile::open` (which reports malformation as an
+ // `io::Err`, not a panic) is the whole mechanism and every error path lands
+ // on `Family::Binary` / `"ole2"`.
+ if prefix.starts_with(OLE2_MAGIC) {
+ if !gzip {
+ if let Ok(f) = std::fs::File::open(content_path) {
+ match ole2_family(f) {
+ Some(Family::Xls) => return Ok(mk(Family::Xls)),
+ Some(Family::Doc) => return Ok(mk(Family::Doc)),
+ _ => {}
+ }
+ }
+ }
+ let mut s = mk(Family::Binary);
+ s.binary_kind = Some("ole2".into());
+ return Ok(s);
+ }
// Compressed image/audio/model payloads routinely pass the NUL and
// control-char heuristics below (windows-1252 decodes almost every byte
// to something printable), and a multi-MB PSD misread as prose turns into
@@ -1160,6 +1204,34 @@ fn looks_like_tar_header(prefix: &[u8]) -> bool {
/// folder of XML parts that is no more searchable than the file was. OOXML is
/// identified by its main part; OpenDocument and EPUB by the `mimetype` member
/// their container specs require, read through a bounded reader.
+/// Route an OLE2 / Compound File Binary container by its top-level stream
+/// names: [`Family::Xls`] for a `Workbook`/`Book` stream, [`Family::Doc`] for
+/// a `WordDocument` stream, and `None` for anything else (an unrecognised OLE2
+/// blob — PowerPoint, Visio, MSI, a 1C `.epf`, a crafted or truncated file).
+///
+/// `None` deliberately covers every failure: `cfb::CompoundFile::open` reports
+/// a malformed, truncated or cyclic container as an `io::Err` rather than a
+/// panic, and it reads only the header, FAT and directory (bounded by the file
+/// size, since a declared sector past EOF fails the read), so this is safe on
+/// hostile input and never has to open a stream to decide. The `.doc` ↔ `.xls`
+/// precedence (Word before Excel) matches real files, which carry only one of
+/// the two main streams.
+fn ole2_family<F: Read + std::io::Seek>(f: F) -> Option<Family> {
+ let comp = cfb::CompoundFile::open(f).ok()?;
+ let mut xls = false;
+ for entry in comp.read_root_storage() {
+ if !entry.is_stream() {
+ continue;
+ }
+ match entry.name() {
+ "WordDocument" => return Some(Family::Doc),
+ "Workbook" | "Book" => xls = true,
+ _ => {}
+ }
+ }
+ xls.then_some(Family::Xls)
+}
+
fn zip_container_kind<R: Read + std::io::Seek>(z: &mut zip::ZipArchive<R>) -> &'static str {
const PARTS: &[(&str, &str)] = &[
("word/document.xml", "docx"),
@@ -4466,3 +4538,99 @@ mod data_container_tests {
}
}
}
+
+#[cfg(test)]
+mod ole2_sniff_tests {
+ use super::*;
+ use std::io::Write;
+ use std::path::Path;
+
+ /// Write a synthetic OLE2 compound file carrying exactly the named streams
+ /// (each with a byte of content), and return its path inside `dir`. Built
+ /// with cfb's own writer, so no real Office file is ever needed.
+ fn write_ole2(dir: &tempfile::TempDir, file: &str, streams: &[&str]) -> std::path::PathBuf {
+ let path = dir.path().join(file);
+ {
+ let mut comp = cfb::create(&path).unwrap();
+ for name in streams {
+ let mut s = comp.create_stream(name).unwrap();
+ s.write_all(b"x").unwrap();
+ s.flush().unwrap();
+ }
+ comp.flush().unwrap();
+ }
+ path
+ }
+
+ /// A `Workbook` stream inside an OLE2 container routes to `Xls`, by content
+ /// and not by the file's extension.
+ #[test]
+ fn a_workbook_stream_is_detected_as_xls() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = write_ole2(&dir, "legacy.bin", &["Workbook"]);
+ assert_eq!(sniff(&path).unwrap().family, Family::Xls);
+ // The pre-BIFF8 name `Book` also counts.
+ let path = write_ole2(&dir, "old.bin", &["Book"]);
+ assert_eq!(sniff(&path).unwrap().family, Family::Xls);
+ }
+
+ /// A `WordDocument` stream routes to `Doc`.
+ #[test]
+ fn a_worddocument_stream_is_detected_as_doc() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = write_ole2(
+ &dir,
+ "report.bin",
+ &["WordDocument", "\u{5}SummaryInformation"],
+ );
+ assert_eq!(sniff(&path).unwrap().family, Family::Doc);
+ }
+
+ /// Any other OLE2 container (PowerPoint, Visio, a 1C `.epf`, …) is named
+ /// `ole2` and left for the skip report — never parsed, never a panic.
+ #[test]
+ fn an_unrecognised_ole2_container_is_named_but_not_parsed() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = write_ole2(&dir, "deck.bin", &["PowerPoint Document"]);
+ let sn = sniff(&path).unwrap();
+ assert_eq!(sn.family, Family::Binary);
+ assert_eq!(sn.binary_kind.as_deref(), Some("ole2"));
+ }
+
+ /// The OLE2 magic followed by garbage (a truncated/crafted compound file)
+ /// must classify as binary `ole2` without panicking or hanging.
+ #[test]
+ fn a_truncated_ole2_header_is_binary_not_a_panic() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("broken.bin");
+ let mut bytes = OLE2_MAGIC.to_vec();
+ bytes.extend_from_slice(&[0xFF; 600]); // nonsense header fields
+ std::fs::write(&path, &bytes).unwrap();
+ let sn = sniff(&path).unwrap();
+ assert_eq!(sn.family, Family::Binary);
+ assert_eq!(sn.binary_kind.as_deref(), Some("ole2"));
+ }
+
+ /// A `.ssc`-style spreadsheet export is a TEXT XML body, not an OLE2
+ /// container, and must stay `Xml` — pinned so the new OLE2 routing (and any
+ /// future "spreadsheet" heuristic) never steals it.
+ #[test]
+ fn an_ssc_xml_spreadsheet_body_stays_xml() {
+ let body = concat!(
+ r#"<?xml version="1.0" encoding="UTF-8"?>"#,
+ "\n",
+ r#"<spreadsheet-workbook-data version="2" generator="xerj">"#,
+ "\n <sheet name=\"Totals\"><row><cell>Region</cell><cell>Revenue</cell></row></sheet>\n",
+ r#"</spreadsheet-workbook-data>"#,
+ );
+ let fam = sniff_bytes(
+ body.as_bytes(),
+ Path::new("export.ssc"),
+ Path::new("export.ssc"),
+ false,
+ )
+ .unwrap()
+ .family;
+ assert_eq!(fam, Family::Xml);
+ }
+}