AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): OLE2 Office support — legacy .doc and .xls
xerj-org5235d9b25dbb
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 62b38b6c1..b1c209aed 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -963,6 +963,22 @@ dependencies = ["either",]+[[package]]+name = "calamine"+version = "0.26.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "138646b9af2c5d7f1804ea4bf93afc597737d2bd4f7341d67c48b03316976eb1"+dependencies = [+ "byteorder",+ "chrono",+ "codepage",+ "encoding_rs",+ "log",+ "quick-xml 0.31.0",+ "serde",+ "zip 2.4.2",+]+[[package]]name = "candle-core"version = "0.9.2"@@ -1057,6 +1073,17 @@ dependencies = ["shlex",]+[[package]]+name = "cfb"+version = "0.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0f13298d9178622b7916ec5d01d470425c04273c8ed07e87b1c38b34c4f4612b"+dependencies = [+ "fnv",+ "uuid",+ "web-time",+]+[[package]]name = "cfg-if"version = "1.0.4"@@ -1199,6 +1226,15 @@ version = "0.5.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"+[[package]]+name = "codepage"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26"+dependencies = [+ "encoding_rs",+]+[[package]]name = "colorchoice"version = "1.0.5"@@ -3675,7 +3711,7 @@ dependencies = ["fast-float2","libc","log",- "quick-xml",+ "quick-xml 0.41.0","serde","serde_json","thiserror 2.0.18",@@ -3881,7 +3917,7 @@ dependencies = ["nom 8.0.0","office_oxide","phf 0.14.0",- "quick-xml",+ "quick-xml 0.41.0","regex","serde","serde_json",@@ -4418,6 +4454,16 @@ version = "2.0.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"+[[package]]+name = "quick-xml"+version = "0.31.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"+dependencies = [+ "encoding_rs",+ "memchr",+]+[[package]]name = "quick-xml"version = "0.41.0"@@ -7320,6 +7366,8 @@ dependencies = ["aws-sdk-s3","aws-smithy-http-client","base64 0.23.1",+ "calamine",+ "cfb","chrono","csv","encoding_rs",@@ -7332,7 +7380,7 @@ dependencies = ["memchr","notify","pdf_oxide",- "quick-xml",+ "quick-xml 0.41.0","rayon","regex","reqwest",diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex 964a564da..8be565134 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -133,6 +133,22 @@ pdf_oxide = { version = "=0.3.75", default-features = false, features = ["legacycsv = "1" # streaming CSV with sniffed dialectquick-xml = "0.41" # XML pull parser + DOCX internalszip = { version = "2", default-features = false, features = ["deflate"] } # DOCX container+# --- Legacy Microsoft Office binary formats (OLE2 / Compound File Binary) ---+# calamine reads the legacy BIFF `.xls` worksheet stream (it ships its own CFB+# reader internally); pinned to 0.26.1, the last line that depends on `zip ^2`+# so it reuses this crate's existing `zip 2` rather than pulling a second major+# of zip into the tree. Licence: MIT. Its transitive adds are codepage (MIT,+# for the legacy 8-bit cell codepage) and log (MIT/Apache-2.0); byteorder and+# encoding_rs are already here.+calamine = { version = "=0.26.1", features = ["dates"] }+# `cfb` reads the OLE2/Compound File Binary container directly: used by the+# content sniffer to route a `.doc`/`.xls`/other OLE2 file by its stream names,+# and by the `.doc` extractor to pull the WordDocument + Table streams and the+# SummaryInformation title. `open()` returns `io::Result` and treats malformed+# or truncated input as an error (never a panic), which the fuzz-minded sniffer+# relies on. Licence: MIT. Runtime deps (fnv, uuid) are already in the lock+# file; web-time is a tiny wasm-time shim. Pinned exact.+cfb = "=0.15.0"serde_yaml = "0.9" # YAML multi-doc# The one C dependency (bundled SQLite), isolated to this crate.rusqlite = { version = "0.32", features = ["bundled"] }diff --git a/engine/crates/xerj-autoindex/src/estimate.rs b/engine/crates/xerj-autoindex/src/estimate.rsindex 6874996fa..6fe5d8573 100644--- a/engine/crates/xerj-autoindex/src/estimate.rs+++ b/engine/crates/xerj-autoindex/src/estimate.rs@@ -120,6 +120,7 @@ pub fn exact_scan_bytes(| Family::TxtProse| Family::Code| Family::Docx+ | Family::Doc| Family::Pptx| Family::Epub| Family::Ipynb@@ -144,7 +145,7 @@ pub fn exact_scan_bytes(Family::UnityMeta => (size <= UNITY_META_CAP).then_some(size),// Row-capped per table (per sheet); bytes read bear no fixed relation// to `size`.- Family::Sqlite | Family::Xlsx => None,+ Family::Sqlite | Family::Xlsx | Family::Xls => None,// Deliberately partial: `bvh::extract` stops at `Frame Time:` and// never pulls the motion block off disk, so `size` is not the number// of bytes this machine demonstrated it can chew through.diff --git a/engine/crates/xerj-autoindex/src/extract/doc.rs b/engine/crates/xerj-autoindex/src/extract/doc.rsnew file mode 100644index 000000000..2333587d4--- /dev/null+++ b/engine/crates/xerj-autoindex/src/extract/doc.rs@@ -0,0 +1,500 @@+//! DOC — legacy Word 97-2003 documents (`.doc`, an OLE2 compound file).+//!+//! Word 97 and later store the document's characters in the `WordDocument`+//! stream and describe where they live with a *piece table* (the CLX) kept in+//! the `0Table` or `1Table` stream. We read the File Information Block (FIB) at+//! the head of `WordDocument` to locate the CLX, walk the piece table, and+//! concatenate each piece — decoding it as little-endian UTF-16 or, when the+//! piece is flagged "compressed", as the legacy 8-bit Windows-1252 code page+//! (MS-DOC §2.3.1, §2.8.35, §2.9.177). The recovered text is emitted as section+//! records, with the title taken from the `\x05SummaryInformation` property set+//! when present.+//!+//! LIMITS — read honestly. This recovers the main document text of Word 97++//! files, which is the overwhelming majority of surviving `.doc` attachments.+//! It does NOT recover:+//! * formatting, styles, or heading structure — the output is plain prose, so+//! unlike the DOCX extractor there is no `headings` list;+//! * tables as tables — cell and row marks become line breaks, so a table+//! reads as text, not as rows;+//! * headers/footers, footnotes, endnotes, comments and textboxes, which live+//! in separate ranges past the main `fcMac` and are not followed here;+//! * Word 6.0/95 (and earlier) documents, which predate this piece-table+//! layout — those fall back to reading the raw main stream as Windows-1252,+//! which is lossy, and if even that yields nothing the file is counted as+//! junk rather than mis-indexed.+//!+//! Field instruction codes (hidden `{ }` field source between the field-begin+//! and field-separator marks) are dropped; the field RESULT text is kept.++use super::{emit_document, ExtractStats, Sink, MAX_RECORDS_PER_FILE};+use anyhow::Result;+use std::io::Read;+use std::path::Path;++/// Largest `WordDocument` / table stream we read. A `.doc`'s text stream is+/// far below this; the cap only bounds what a crafted compound file can cost.+const MAX_STREAM_BYTES: u64 = 256 << 20;++/// Largest property-set stream (`SummaryInformation`) we read for the title.+const MAX_SUMMARY_BYTES: u64 = 1 << 20;++/// Body text cap: past this the document is already far larger than any real+/// `.doc`, and section emission is bounded by [`MAX_RECORDS_PER_FILE`] anyway.+const MAX_BODY_CHARS: usize = 64 << 20;++/// Defensive cap on piece-table entries, against a crafted `lcbClx`.+const MAX_PIECES: usize = 1 << 20;++// --- little-endian field reads, all bounds-checked (never panic on content) --++fn u16_at(b: &[u8], off: usize) -> Option<u16> {+ b.get(off..off + 2)+ .map(|s| u16::from_le_bytes([s[0], s[1]]))+}++fn u32_at(b: &[u8], off: usize) -> Option<u32> {+ b.get(off..off + 4)+ .map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]))+}++pub fn extract(path: &Path, name: &Path, sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let f = std::fs::File::open(path)?;+ // A malformed or truncated compound file is reported by `cfb` as an error,+ // never a panic — so every failure below degrades to "junk", and a single+ // unreadable legacy file never aborts a folder scan.+ let mut comp = match cfb::CompoundFile::open(f) {+ Ok(c) => c,+ Err(_) => {+ stats.junk += 1;+ return Ok(stats);+ }+ };++ let Some(word) = read_stream(&mut comp, "WordDocument", MAX_STREAM_BYTES) else {+ stats.junk += 1;+ return Ok(stats);+ };++ let body = recover_text(&mut comp, &word).unwrap_or_default();+ let body = body.trim();+ if body.is_empty() {+ stats.junk += 1;+ return Ok(stats);+ }++ let title = summary_title(&mut comp)+ .filter(|t| !t.trim().is_empty())+ .or_else(|| {+ body.lines()+ .find(|l| !l.trim().is_empty())+ .map(|l| l.trim().chars().take(200).collect())+ })+ .unwrap_or_else(|| {+ name.file_stem()+ .map(|s| s.to_string_lossy().to_string())+ .unwrap_or_else(|| "untitled".into())+ });++ emit_document(&title, &[], body, MAX_RECORDS_PER_FILE, sink, &mut stats);+ Ok(stats)+}++/// Read a named stream (bounded) into memory, or `None` if it is absent or+/// unreadable.+fn read_stream<F: Read + std::io::Seek>(+ comp: &mut cfb::CompoundFile<F>,+ name: &str,+ cap: u64,+) -> Option<Vec<u8>> {+ if !comp.is_stream(name) {+ return None;+ }+ let stream = comp.open_stream(name).ok()?;+ let mut buf = Vec::new();+ stream.take(cap).read_to_end(&mut buf).ok()?;+ Some(buf)+}++/// Recover the main document text via the FIB piece table. Returns `None` only+/// when the FIB is too short to parse; a FIB with no usable CLX falls back to+/// the raw-range reader (`fallback_text`).+fn recover_text<F: Read + std::io::Seek>(+ comp: &mut cfb::CompoundFile<F>,+ word: &[u8],+) -> Option<String> {+ // FIB.base.flags1 (offset 0x0A): bit 0x0200 (fWhichTblStm) selects the+ // table stream. fcClx / lcbClx sit at the fixed Word97 offsets 0x01A2 /+ // 0x01A6 within the FIB (MS-DOC §2.5.6 FibRgFcLcb97 — the 67th/68th FC/LCB+ // slot).+ let flags1 = u16_at(word, 0x0A)?;+ let table_name = if flags1 & 0x0200 != 0 {+ "1Table"+ } else {+ "0Table"+ };+ let fc_clx = u32_at(word, 0x01A2)? as usize;+ let lcb_clx = u32_at(word, 0x01A6)? as usize;++ if lcb_clx > 0 {+ // The table stream named by the flag, falling back to the other (some+ // writers disagree with the flag).+ let table = read_stream(comp, table_name, MAX_STREAM_BYTES)+ .or_else(|| {+ let other = if table_name == "1Table" {+ "0Table"+ } else {+ "1Table"+ };+ read_stream(comp, other, MAX_STREAM_BYTES)+ })+ .unwrap_or_default();+ if let Some(text) = text_from_piece_table(word, &table, fc_clx, lcb_clx) {+ if !text.trim().is_empty() {+ return Some(text);+ }+ }+ }++ // No piece table (older Word, or an unreadable CLX): read the raw main+ // range as Windows-1252. fcMin/fcMac are at FIB offsets 0x0018 / 0x001C.+ Some(fallback_text(word))+}++/// Walk the CLX in `table[fc_clx .. fc_clx+lcb_clx]`, then materialise each+/// piece's characters out of `word`.+fn text_from_piece_table(+ word: &[u8],+ table: &[u8],+ fc_clx: usize,+ lcb_clx: usize,+) -> Option<String> {+ let clx = table.get(fc_clx..fc_clx.checked_add(lcb_clx)?)?;++ // The CLX is zero or more Prc blocks (clxt 0x01: a u16 byte count then that+ // many bytes) followed by exactly one Pcdt (clxt 0x02: a u32 byte count then+ // the PlcPcd). Walk to the Pcdt.+ let mut i = 0usize;+ let plc = loop {+ match clx.get(i)? {+ 0x01 => {+ let cb = u16_at(clx, i + 1)? as usize;+ i = i.checked_add(3)?.checked_add(cb)?;+ }+ 0x02 => {+ let lcb = u32_at(clx, i + 1)? as usize;+ let start = i.checked_add(5)?;+ break clx.get(start..start.checked_add(lcb)?)?;+ }+ _ => return None,+ }+ };++ // PlcPcd: (n+1) CPs (u32, 4 bytes) then n PCDs (8 bytes), so the total is+ // 4·(n+1) + 8·n = 4 + 12·n bytes, i.e. n = (len - 4) / 12.+ if plc.len() < 4 {+ return None;+ }+ let n = (plc.len() - 4) / 12;+ if n == 0 || n > MAX_PIECES {+ return None;+ }+ let pcd_base = (n + 1) * 4;++ let mut out = String::new();+ for p in 0..n {+ let cp_start = u32_at(plc, p * 4)? as usize;+ let cp_end = u32_at(plc, (p + 1) * 4)? as usize;+ let char_count = cp_end.checked_sub(cp_start)?;+ if char_count == 0 {+ continue;+ }+ // PCD: [0..2] flags, [2..6] FcCompressed, [6..8] prm.+ let fc_field = u32_at(plc, pcd_base + p * 8 + 2)?;+ let compressed = fc_field & 0x4000_0000 != 0;+ let fc = (fc_field & 0x3FFF_FFFF) as usize;++ if compressed {+ // 8-bit: the real byte offset is fc/2, one byte per character.+ let start = fc / 2;+ let bytes = word.get(start..start.checked_add(char_count)?)?;+ let (decoded, _, _) = encoding_rs::WINDOWS_1252.decode(bytes);+ out.push_str(&decoded);+ } else {+ // 16-bit little-endian UTF-16, two bytes per character.+ let start = fc;+ let byte_len = char_count.checked_mul(2)?;+ let bytes = word.get(start..start.checked_add(byte_len)?)?;+ let units: Vec<u16> = bytes+ .chunks_exact(2)+ .map(|c| u16::from_le_bytes([c[0], c[1]]))+ .collect();+ out.push_str(&String::from_utf16_lossy(&units));+ }+ if out.len() >= MAX_BODY_CHARS {+ break;+ }+ }+ Some(clean_text(&out))+}++/// Last-resort reader for pre-piece-table documents: the main text range+/// `[fcMin, fcMac)` of `WordDocument`, decoded as Windows-1252. Lossy by+/// nature; returns an empty string when the range is unusable.+fn fallback_text(word: &[u8]) -> String {+ let (Some(fc_min), Some(fc_mac)) = (u32_at(word, 0x0018), u32_at(word, 0x001C)) else {+ return String::new();+ };+ let (min, mac) = (fc_min as usize, fc_mac as usize);+ if mac <= min {+ return String::new();+ }+ let end = mac.min(min.saturating_add(MAX_BODY_CHARS));+ let Some(bytes) = word.get(min..end.min(word.len())) else {+ return String::new();+ };+ let (decoded, _, _) = encoding_rs::WINDOWS_1252.decode(bytes);+ clean_text(&decoded)+}++/// Turn Word's in-stream control characters into clean prose: paragraph, line,+/// page, cell and row marks become line breaks; field INSTRUCTION text (between+/// field-begin 0x13 and field-separator 0x14) is dropped while the field RESULT+/// is kept; and the remaining sub-0x20 control codes are discarded.+fn clean_text(s: &str) -> String {+ let mut out = String::with_capacity(s.len());+ // Depth of nested field instruction regions we are currently inside.+ let mut field_instr = 0u32;+ for c in s.chars() {+ match c {+ '\u{13}' => field_instr += 1, // field begin → enter instruction+ '\u{14}' => field_instr = field_instr.saturating_sub(1), // separator → result+ '\u{15}' => {} // field end+ _ if field_instr > 0 => {} // inside instruction: drop+ // \r (0x0D) is the paragraph mark; 0x0B line break, 0x0C page+ // break, 0x07 cell/row mark — all become line breaks.+ '\r' | '\u{0B}' | '\u{0C}' | '\u{07}' => out.push('\n'),+ '\u{1E}' => out.push('-'), // non-breaking hyphen+ '\u{A0}' => out.push(' '), // non-breaking space+ '\t' | '\n' => out.push(c),+ _ if (c as u32) < 0x20 => {} // other control codes: drop+ _ => out.push(c),+ }+ }+ // Collapse the runs of blank lines the mark-to-newline mapping produces+ // into paragraph breaks, and trim trailing whitespace on each line.+ let mut collapsed = String::with_capacity(out.len());+ let mut blank_run = 0u32;+ for line in out.lines() {+ let line = line.trim_end();+ if line.is_empty() {+ blank_run += 1;+ continue;+ }+ if !collapsed.is_empty() {+ collapsed.push_str(if blank_run > 0 { "\n\n" } else { "\n" });+ }+ blank_run = 0;+ collapsed.push_str(line);+ }+ collapsed+}++/// Best-effort document title from the `\x05SummaryInformation` property set+/// (MS-OLEPS): property id 2 (PIDSI_TITLE). Returns `None` on any shortfall.+fn summary_title<F: Read + std::io::Seek>(comp: &mut cfb::CompoundFile<F>) -> Option<String> {+ let s = read_stream(comp, "\u{5}SummaryInformation", MAX_SUMMARY_BYTES)?;+ property_string(&s, 2)+}++/// Read a single string property (`pid`) out of the first section of a+/// property-set stream. Handles VT_LPSTR (code-page bytes) and VT_LPWSTR+/// (UTF-16). Everything is bounds-checked; `None` on any malformation.+fn property_string(s: &[u8], pid: u32) -> Option<String> {+ // PropertySetStream header: byteOrder(2) version(2) sysId(4) clsid(16)+ // cSections(4) = 28 bytes, then the first FMTID(16) + section offset(4).+ let section_off = u32_at(s, 44)? as usize;+ let cb_section = u32_at(s, section_off)? as usize;+ let num_props = u32_at(s, section_off + 4)? as usize;+ // The section must fit, and the property index table must fit inside it.+ let section_end = section_off.checked_add(cb_section)?;+ if section_end > s.len() || num_props > 1 << 16 {+ return None;+ }++ // Find the code page (pid 1, VT_I2) for VT_LPSTR decoding; default 1252.+ let mut code_page: u16 = 1252;+ let mut target: Option<usize> = None;+ for i in 0..num_props {+ let entry = section_off + 8 + i * 8;+ let this_pid = u32_at(s, entry)?;+ let value_off = section_off + u32_at(s, entry + 4)? as usize;+ if this_pid == 1 {+ // VT_I2 code page: type(4) then i16.+ if u32_at(s, value_off) == Some(2) {+ if let Some(cp) = u16_at(s, value_off + 4) {+ code_page = cp;+ }+ }+ }+ if this_pid == pid {+ target = Some(value_off);+ }+ }++ let value_off = target?;+ let vt = u32_at(s, value_off)?;+ let len = u32_at(s, value_off + 4)? as usize;+ let data_off = value_off + 8;+ match vt {+ 0x1E => {+ // VT_LPSTR: `len` bytes incl terminating NUL, code-page encoded.+ let bytes = s.get(data_off..data_off.checked_add(len)?)?;+ let bytes = bytes.split(|&b| b == 0).next().unwrap_or(bytes);+ let enc = encoding_rs::Encoding::for_label(format!("windows-{code_page}").as_bytes())+ .unwrap_or(encoding_rs::WINDOWS_1252);+ let (decoded, _, _) = enc.decode(bytes);+ Some(decoded.trim().to_string())+ }+ 0x1F => {+ // VT_LPWSTR: `len` UTF-16 code units incl NUL.+ let byte_len = len.checked_mul(2)?;+ let bytes = s.get(data_off..data_off.checked_add(byte_len)?)?;+ let mut units: Vec<u16> = bytes+ .chunks_exact(2)+ .map(|c| u16::from_le_bytes([c[0], c[1]]))+ .collect();+ if let Some(pos) = units.iter().position(|&u| u == 0) {+ units.truncate(pos);+ }+ Some(String::from_utf16_lossy(&units).trim().to_string())+ }+ _ => None,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn clean_text_maps_marks_and_drops_field_instructions() {+ // A paragraph mark, then an inline field `{ HYPERLINK "x" }result`: the+ // field RESULT ("click here") is kept inline where it sits and the+ // instruction source is dropped.+ let raw = "Hello\u{0D}See \u{13}HYPERLINK \"http://x\"\u{14}click here\u{15}\u{0D}End";+ let cleaned = clean_text(raw);+ assert_eq!(cleaned, "Hello\nSee click here\nEnd");+ assert!(+ !cleaned.contains("HYPERLINK"),+ "field source must be dropped"+ );+ }++ #[test]+ fn clean_text_collapses_blank_runs_and_trims() {+ let raw = "A\u{0D}\u{0D}\u{0D}B \u{0D}C";+ assert_eq!(clean_text(raw), "A\n\nB\nC");+ }++ #[test]+ fn field_reads_are_bounds_checked() {+ assert_eq!(u16_at(&[1], 0), None);+ assert_eq!(u32_at(&[1, 2, 3], 0), None);+ assert_eq!(u32_at(&[1, 2, 3, 4], 0), Some(0x04030201));+ }++ /// A piece table with one compressed (Windows-1252) and one uncompressed+ /// (UTF-16) piece, assembled by hand, round-trips to the expected text.+ /// This is the core of the extractor and is exercised without needing a+ /// real BIFF file on disk.+ #[test]+ fn piece_table_decodes_compressed_and_uncompressed_pieces() {+ // WordDocument: FIB header padded to 0x0200, then the text region.+ // Piece 0 (compressed, 5 chars "Hello") at byte 0x200.+ // Piece 1 (uncompressed UTF-16, " wörld") right after.+ let mut word = vec![0u8; 0x200];+ let p0_bytes = b"Hello"; // CP1252, 1 byte/char+ let p0_off = 0x200usize;+ word.extend_from_slice(p0_bytes);+ let p1_text = " wörld";+ let p1_off = word.len();+ for u in p1_text.encode_utf16() {+ word.extend_from_slice(&u.to_le_bytes());+ }++ // Build the table stream: CLX = Pcdt (0x02) + lcb + PlcPcd.+ // CPs: [0, 5, 5+6] ; PCDs: piece0 fc=p0_off*2 | compressed, piece1 fc=p1_off.+ let p0_chars = p0_bytes.len() as u32; // 5+ let p1_chars = p1_text.chars().count() as u32; // 6+ let mut plc = Vec::new();+ for cp in [0u32, p0_chars, p0_chars + p1_chars] {+ plc.extend_from_slice(&cp.to_le_bytes());+ }+ // PCD 0: flags(2)=0, fc(4)= (p0_off*2) | 0x40000000, prm(2)=0+ let fc0 = (p0_off as u32 * 2) | 0x4000_0000;+ plc.extend_from_slice(&[0, 0]);+ plc.extend_from_slice(&fc0.to_le_bytes());+ plc.extend_from_slice(&[0, 0]);+ // PCD 1: fc = p1_off (uncompressed)+ plc.extend_from_slice(&[0, 0]);+ plc.extend_from_slice(&(p1_off as u32).to_le_bytes());+ plc.extend_from_slice(&[0, 0]);++ let mut clx = vec![0x02u8];+ clx.extend_from_slice(&(plc.len() as u32).to_le_bytes());+ clx.extend_from_slice(&plc);++ // Table stream: put the CLX at offset 0 so fc_clx = 0.+ let table = clx.clone();+ let text = text_from_piece_table(&word, &table, 0, clx.len()).unwrap();+ assert_eq!(text, "Hello wörld");+ }++ #[test]+ fn property_string_reads_a_vt_lpstr_title() {+ // Minimal SummaryInformation with one section holding pid 2 = "Budget".+ // Layout: 28-byte header, FMTID(16)+sectionOffset(4) at 28..48.+ let title = b"Budget\0";+ let mut s = vec![0u8; 48];+ // byteOrder 0xFFFE, version 0, cSections 1+ s[0] = 0xFE;+ s[1] = 0xFF;+ let section_off = 48u32;+ s[44..48].copy_from_slice(§ion_off.to_le_bytes());+ // Section: cbSection, cProperties=1, then (pid=2, off=16), then value.+ // Property table entry sits at section_off+8; value at section_off+16.+ let value_rel = 16u32;+ let mut section = Vec::new();+ // cbSection filled in after; cProperties+ section.extend_from_slice(&[0, 0, 0, 0]); // placeholder cbSection+ section.extend_from_slice(&1u32.to_le_bytes()); // cProperties+ section.extend_from_slice(&2u32.to_le_bytes()); // pid 2+ section.extend_from_slice(&value_rel.to_le_bytes()); // value offset+ // value: VT_LPSTR (0x1E), len, bytes+ section.extend_from_slice(&0x1Eu32.to_le_bytes());+ section.extend_from_slice(&(title.len() as u32).to_le_bytes());+ section.extend_from_slice(title);+ let cb = section.len() as u32;+ section[0..4].copy_from_slice(&cb.to_le_bytes());+ s.extend_from_slice(§ion);++ assert_eq!(property_string(&s, 2).as_deref(), Some("Budget"));+ assert_eq!(property_string(&s, 99), None, "absent pid yields None");+ }++ #[test]+ fn malformed_clx_never_panics() {+ // Truncated / nonsense CLX bytes must return None, not panic.+ let word = vec![0u8; 0x400];+ for clx in [+ vec![0x02, 0xFF, 0xFF, 0xFF, 0xFF], // lcb far past end+ vec![0x01, 0x10, 0x00], // Prc with no body+ vec![0x09], // unknown clxt+ vec![], // empty+ ] {+ let _ = text_from_piece_table(&word, &clx, 0, clx.len());+ }+ }+}diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 774878ec9..6c17ac7ff 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -5,6 +5,7 @@pub mod bvh;pub mod code;pub mod csv_x;+pub mod doc;pub mod docx;pub mod eml;pub mod epub;@@ -22,6 +23,7 @@ pub mod sqldump;pub mod sqlite_x;pub mod txt;pub mod unity;+pub mod xls;pub mod xlsx;pub mod xml_x;pub mod yaml_x;@@ -214,6 +216,7 @@ pub fn extract(Family::Eml => eml::extract(path, sn.gzip, sink),Family::Mbox => mbox::extract(path, sn.gzip, limit_bytes, sink),Family::Docx => docx::extract(path, sink),+ Family::Doc => doc::extract(path, sn.logical_name.as_deref().unwrap_or(path), sink),Family::Pptx => pptx::extract(path, sn.logical_name.as_deref().unwrap_or(path), sink),Family::Epub => epub::extract(path, sn.logical_name.as_deref().unwrap_or(path), sink),Family::Ipynb => ipynb::extract(@@ -234,6 +237,12 @@ pub fn extract(limit_bytes.map(|_| 500),sink,),+ Family::Xls => xls::extract(+ path,+ sn.logical_name.as_deref().unwrap_or(path),+ limit_bytes.map(|_| 500),+ sink,+ ),Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),Family::Code => code::extract(path, sn, sink),diff --git a/engine/crates/xerj-autoindex/src/extract/xls.rs b/engine/crates/xerj-autoindex/src/extract/xls.rsnew file mode 100644index 000000000..337c8ebf2--- /dev/null+++ b/engine/crates/xerj-autoindex/src/extract/xls.rs@@ -0,0 +1,590 @@+//! XLS — legacy Excel 97-2003 workbooks (BIFF8 inside an OLE2 compound file),+//! read as tables on the same record model as [`super::xlsx`].+//!+//! Each worksheet is its own dataset (`RawRecord.group` = sheet name) and each+//! row below the sheet's header row is one record keyed by the header cells,+//! with values carrying the type the cell stored: numbers stay numbers,+//! booleans stay booleans, and a date/time cell becomes an ISO-8601 string so+//! type inference elects `date` rather than seeing a serial number. The header+//! is the first row (among the first [`HEADER_SCAN_ROWS`] non-empty rows) with+//! at least two cells, all text or date and at least one text, that is not+//! immediately followed by a wider row of the same kind; rows above it are not+//! indexed. A sheet with no such header is indexed as a document, one line per+//! row with cells joined by ` | `.+//!+//! Reading is delegated to calamine, which ships its own OLE2/BIFF reader. The+//! heavy lifting of BIFF record parsing, the shared-string table and the+//! legacy 8-bit cell codepage all live there; this module only maps calamine's+//! `Data` cells onto the extractor's typed record model.+//!+//! Differences from the XLSX extractor, by design: calamine materialises a+//! whole worksheet as a `Range` (the BIFF format caps a sheet at 65,536 rows ×+//! 256 columns, and calamine reads it in one pass), so there is no streaming+//! row cap of our own — the per-sheet sample limit bounds how many rows we+//! EMIT, and [`MAX_ROWS_PER_SHEET`] bounds how many we even look at against a+//! crafted dimension record. Vertical merge fill-down is not applied (`.xls`+//! merge ranges are not surfaced on calamine's `Range`); a category label+//! merged down a column therefore appears only on its first row.++use super::{+ emit_document_with_fields, sanitize_field_name, ExtractStats, FieldOrigin, RawRecord, Sink,+ MAX_FIELDS_PER_RECORD,+};+use anyhow::Result;+use calamine::{Data, DataType, Reader, Xls};+use chrono::Timelike;+use serde_json::{Map, Value};+use std::path::Path;++/// Non-empty rows searched for a header before the sheet is treated as a+/// document (mirrors the XLSX extractor).+const HEADER_SCAN_ROWS: usize = 20;++/// Defensive cap on rows inspected per sheet, in case a crafted BIFF dimension+/// record claims a range far larger than the format's own 65,536-row ceiling.+const MAX_ROWS_PER_SHEET: usize = 1 << 20;++/// Text kept from one sheet indexed as a document.+const MAX_DOC_SHEET_BYTES: usize = 16 << 20;++/// `name` is the file as the corpus names it (under durable preparation this is+/// not `path`, a sealed snapshot blob); document sheets are titled from it.+/// `per_sheet_limit` caps the rows emitted from each sheet (phase-A sampling);+/// `None` reads every row.+pub fn extract(+ path: &Path,+ name: &Path,+ per_sheet_limit: Option<u64>,+ sink: Sink,+) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let f = std::fs::File::open(path)?;+ // calamine reports a malformed/truncated workbook as an error; never a+ // panic. Treat any such failure as junk rather than propagating — a single+ // unreadable legacy file must not abort a folder scan.+ let mut wb = match Xls::new(f) {+ Ok(wb) => wb,+ Err(_) => {+ stats.junk += 1;+ return Ok(stats);+ }+ };++ let stem = name+ .file_stem()+ .map(|s| s.to_string_lossy().to_string())+ .unwrap_or_else(|| "untitled".into());++ let sheet_names = wb.sheet_names().to_vec();+ for sheet_name in sheet_names {+ let Ok(range) = wb.worksheet_range(&sheet_name) else {+ continue;+ };+ let start_row = range.start().map(|(r, _)| r).unwrap_or(0);+ let start_col = range.start().map(|(_, c)| c).unwrap_or(0);++ // Build the non-empty rows, each carrying 1-based column numbers so the+ // header/column machinery matches the XLSX extractor's.+ let mut rows: Vec<Row> = Vec::new();+ for (i, cells) in range.rows().take(MAX_ROWS_PER_SHEET).enumerate() {+ let mut out = Vec::new();+ for (j, data) in cells.iter().enumerate() {+ if let Some(cell) = convert_cell(start_col + j as u32 + 1, data) {+ out.push(cell);+ }+ }+ if !out.is_empty() {+ rows.push(Row {+ num: start_row + i as u32 + 1,+ cells: out,+ });+ }+ }+ if rows.is_empty() {+ continue;+ }++ if let Some(h) = find_header(&rows) {+ if !emit_table(&sheet_name, &rows, h, per_sheet_limit, sink, &mut stats) {+ break;+ }+ } else if !emit_document_sheet(&sheet_name, &stem, &rows, sink, &mut stats) {+ break;+ }+ }++ if stats.records == 0 {+ stats.junk += 1;+ }+ Ok(stats)+}++/// A cell's typed kind, mirroring the XLSX extractor so header detection and+/// type inference agree across the two formats.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum Kind {+ Text,+ Number,+ Bool,+ Date,+}++#[derive(Debug, Clone)]+struct Cell {+ /// 1-based column (A = 1).+ col: u32,+ value: Value,+ kind: Kind,+}++impl Cell {+ fn display(&self) -> String {+ match &self.value {+ Value::String(s) => s.clone(),+ Value::Bool(true) => "TRUE".into(),+ Value::Bool(false) => "FALSE".into(),+ v => v.to_string(),+ }+ }+}++#[derive(Debug)]+struct Row {+ /// 1-based row number, as Excel shows it.+ num: u32,+ cells: Vec<Cell>,+}++/// Map one calamine cell onto the typed model; `None` for an empty cell or an+/// error value (`#N/A`, `#DIV/0!`), exactly as the XLSX extractor drops them.+fn convert_cell(col: u32, data: &Data) -> Option<Cell> {+ match data {+ Data::Empty | Data::Error(_) => None,+ Data::String(s) => {+ let s = s.trim();+ (!s.is_empty()).then(|| Cell {+ col,+ value: Value::String(s.to_string()),+ kind: Kind::Text,+ })+ }+ Data::Int(i) => Some(Cell {+ col,+ value: Value::Number((*i).into()),+ kind: Kind::Number,+ }),+ Data::Float(f) => Some(Cell {+ col,+ value: number_value(*f)?,+ kind: Kind::Number,+ }),+ Data::Bool(b) => Some(Cell {+ col,+ value: Value::Bool(*b),+ kind: Kind::Bool,+ }),+ Data::DateTimeIso(s) => {+ let s = s.trim();+ (!s.is_empty()).then(|| Cell {+ col,+ value: Value::String(s.to_string()),+ kind: Kind::Date,+ })+ }+ Data::DurationIso(s) => {+ let s = s.trim();+ (!s.is_empty()).then(|| Cell {+ col,+ value: Value::String(s.to_string()),+ kind: Kind::Text,+ })+ }+ Data::DateTime(_) => date_cell(col, data),+ }+}++/// A calamine `Data::DateTime` rendered to the same ISO-8601 strings the XLSX+/// extractor emits: a bare date at midnight, a `T`-joined datetime otherwise,+/// and `HH:MM:SS` for a duration/time-of-day. Falls back to the raw serial+/// number if calamine cannot resolve it to a chrono value.+fn date_cell(col: u32, data: &Data) -> Option<Cell> {+ // A time of day (fraction of a day with no date part) resolves to a+ // `NaiveTime` but not a sensible `NaiveDateTime`; check it first.+ if let Some(dt) = data.as_datetime() {+ let iso = if dt.num_seconds_from_midnight() == 0 && dt.nanosecond() == 0 {+ dt.format("%Y-%m-%d").to_string()+ } else if dt.nanosecond() == 0 {+ dt.format("%Y-%m-%dT%H:%M:%S").to_string()+ } else {+ dt.format("%Y-%m-%dT%H:%M:%S%.3f").to_string()+ };+ return Some(Cell {+ col,+ value: Value::String(iso),+ kind: Kind::Date,+ });+ }+ if let Some(t) = data.as_time() {+ return Some(Cell {+ col,+ value: Value::String(t.format("%H:%M:%S").to_string()),+ kind: Kind::Date,+ });+ }+ // Could not resolve a calendar value: keep the serial number as a number so+ // nothing is dropped.+ let f = data.as_f64()?;+ Some(Cell {+ col,+ value: number_value(f)?,+ kind: Kind::Number,+ })+}++/// Integral floats inside the i64-exact range become integers (so `1250.0`+/// indexes as `1250`), matching the XLSX extractor's `number_value`.+fn number_value(f: f64) -> Option<Value> {+ if !f.is_finite() {+ return None;+ }+ if f.fract() == 0.0 && f.abs() < 9_007_199_254_740_992.0 {+ Some(Value::Number((f as i64).into()))+ } else {+ serde_json::Number::from_f64(f).map(Value::Number)+ }+}++/// Column letters of a column number (28 → `AB`).+fn column_letters(mut col: u32) -> String {+ let mut out = Vec::new();+ while col > 0 {+ let rem = (col - 1) % 26;+ out.push(b'A' + rem as u8);+ col = (col - 1) / 26;+ }+ out.reverse();+ String::from_utf8(out).unwrap_or_default()+}++/// Whether a row can be a header: two or more cells, some text, and no numbers+/// or booleans (date cells are allowed, for month-column headers).+fn is_header_candidate(row: &Row) -> bool {+ row.cells.len() >= 2+ && row+ .cells+ .iter()+ .all(|c| matches!(c.kind, Kind::Text | Kind::Date))+ && row.cells.iter().any(|c| c.kind == Kind::Text)+}++/// Index of the header row. A candidate followed by a WIDER candidate is a+/// title line above the real header and is passed over. Only the first+/// [`HEADER_SCAN_ROWS`] non-empty rows are considered.+fn find_header(rows: &[Row]) -> Option<usize> {+ let scan = rows.len().min(HEADER_SCAN_ROWS);+ rows[..scan].windows(2).position(|w| {+ is_header_candidate(&w[0])+ && !(is_header_candidate(&w[1]) && w[1].cells.len() > w[0].cells.len())+ })+}++/// Emit every data row under `header_idx` as a typed record keyed by the header+/// cells. Returns false if the sink asked to stop the whole file.+fn emit_table(+ sheet: &str,+ rows: &[Row],+ header_idx: usize,+ limit: Option<u64>,+ sink: Sink,+ stats: &mut ExtractStats,+) -> bool {+ let header = &rows[header_idx];+ let mut seen = std::collections::HashSet::new();+ let mut names: std::collections::HashMap<u32, String> = std::collections::HashMap::new();+ for c in &header.cells {+ let mut name = sanitize_field_name(&c.display());+ while !seen.insert(name.clone()) {+ name.push_str("_2");+ }+ names.insert(c.col, name);+ }+ let header_cells: Vec<(u32, String)> =+ header.cells.iter().map(|c| (c.col, c.display())).collect();++ let mut emitted = 0u64;+ for row in &rows[header_idx + 1..] {+ // A repeated header row (common when a sheet stacks sections) is not a+ // data record.+ let row_text: Vec<(u32, String)> = row.cells.iter().map(|c| (c.col, c.display())).collect();+ if row_text == header_cells {+ continue;+ }+ let mut fields = Map::new();+ for c in &row.cells {+ if fields.len() >= MAX_FIELDS_PER_RECORD {+ break;+ }+ let name = names+ .get(&c.col)+ .cloned()+ .unwrap_or_else(|| format!("col_{}", column_letters(c.col)));+ fields.insert(name, c.value.clone());+ }+ stats.records += 1;+ if !sink(RawRecord {+ fields,+ locator: format!("{sheet}!r{}", row.num),+ group: Some(sheet.to_string()),+ origin: FieldOrigin::Data,+ }) {+ return false;+ }+ emitted += 1;+ if limit.is_some_and(|lim| emitted >= lim) {+ break;+ }+ }+ true+}++/// Emit a sheet with no detectable header as a document: one line per row,+/// cells joined by ` | `. Returns false if the sink asked to stop.+fn emit_document_sheet(+ sheet: &str,+ stem: &str,+ rows: &[Row],+ sink: Sink,+ stats: &mut ExtractStats,+) -> bool {+ let mut body = String::new();+ let mut last_row = 0u32;+ for row in rows {+ let line = row+ .cells+ .iter()+ .map(Cell::display)+ .collect::<Vec<_>>()+ .join(" | ");+ if body.len() + line.len() + 2 > MAX_DOC_SHEET_BYTES {+ stats.truncated = true;+ break;+ }+ if !body.is_empty() {+ // A gap in row numbers is a paragraph break, where section+ // splitting prefers to cut.+ body.push_str(if row.num > last_row + 1 { "\n\n" } else { "\n" });+ }+ body.push_str(&line);+ last_row = row.num;+ }+ if body.trim().is_empty() {+ return true;+ }+ let mut base = Map::new();+ base.insert("sheet".into(), Value::String(sheet.to_string()));+ emit_document_with_fields(+ &base,+ &format!("{stem} — {sheet}"),+ &body,+ sheet,+ sink,+ stats,+ )+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn fields(r: &RawRecord) -> Value {+ Value::Object(r.fields.clone())+ }++ /// Build a one-sheet `.xls` in a tempdir with `rustexcel`-free, synthetic+ /// content by driving calamine's own writer? calamine has no writer, so the+ /// unit tests here exercise the typed-model mapping and the table/document+ /// routing directly on synthetic rows. The end-to-end round-trip through a+ /// real BIFF file is covered by `extract_real_xls_fixture` (gated on a+ /// generated fixture), so these pin the behaviour deterministically.+ fn row(num: u32, cells: &[Cell]) -> Row {+ Row {+ num,+ cells: cells.to_vec(),+ }+ }+ fn txt(col: u32, s: &str) -> Cell {+ Cell {+ col,+ value: Value::String(s.into()),+ kind: Kind::Text,+ }+ }+ fn num(col: u32, v: i64) -> Cell {+ Cell {+ col,+ value: Value::Number(v.into()),+ kind: Kind::Number,+ }+ }+ fn boolean(col: u32, v: bool) -> Cell {+ Cell {+ col,+ value: Value::Bool(v),+ kind: Kind::Bool,+ }+ }++ fn collect_table(sheet: &str, rows: &[Row]) -> (ExtractStats, Vec<RawRecord>) {+ let h = find_header(rows).expect("a header should be found");+ let mut recs = Vec::new();+ let mut stats = ExtractStats::default();+ emit_table(+ sheet,+ rows,+ h,+ None,+ &mut |r| {+ recs.push(r);+ true+ },+ &mut stats,+ );+ (stats, recs)+ }++ #[test]+ fn convert_cell_types_mirror_the_xlsx_model() {+ assert!(convert_cell(1, &Data::Empty).is_none());+ assert!(convert_cell(1, &Data::Error(calamine::CellErrorType::Div0)).is_none());+ // An integral float indexes as an integer, not 1250.0.+ assert_eq!(+ convert_cell(3, &Data::Float(1250.0)).unwrap().value,+ json!(1250)+ );+ assert_eq!(+ convert_cell(3, &Data::Float(980.5)).unwrap().value,+ json!(980.5)+ );+ assert_eq!(convert_cell(4, &Data::Bool(true)).unwrap().kind, Kind::Bool);+ // A whitespace-only string is dropped like an empty cell.+ assert!(convert_cell(1, &Data::String(" ".into())).is_none());+ }++ #[test]+ fn a_title_row_is_skipped_and_each_row_under_the_header_is_a_typed_record() {+ // Row 1 is a single-cell title; row 3 the header; 4-5 the data.+ let rows = vec![+ row(1, &[txt(1, "Q3 Sales Report")]),+ row(+ 3,+ &[+ txt(1, "Region"),+ txt(2, "Month"),+ txt(3, "Revenue"),+ txt(4, "Closed"),+ ],+ ),+ row(+ 4,+ &[+ txt(1, "EMEA"),+ txt(2, "July"),+ num(3, 1250),+ boolean(4, true),+ ],+ ),+ row(+ 5,+ &[+ txt(1, "APAC"),+ txt(2, "August"),+ num(3, 980),+ boolean(4, false),+ ],+ ),+ ];+ let (stats, recs) = collect_table("Sales", &rows);+ assert_eq!((stats.records, stats.junk), (2, 0));+ assert_eq!(recs[0].group.as_deref(), Some("Sales"));+ assert_eq!(recs[0].locator, "Sales!r4");+ assert_eq!(recs[0].origin, FieldOrigin::Data);+ assert_eq!(+ fields(&recs[0]),+ json!({"Region": "EMEA", "Month": "July", "Revenue": 1250, "Closed": true})+ );+ assert_eq!(+ fields(&recs[1]),+ json!({"Region": "APAC", "Month": "August", "Revenue": 980, "Closed": false})+ );+ }++ #[test]+ fn a_repeated_header_row_is_not_a_data_record() {+ let rows = vec![+ row(1, &[txt(1, "Name"), txt(2, "Score")]),+ row(2, &[txt(1, "Ann"), num(2, 10)]),+ row(3, &[txt(1, "Name"), txt(2, "Score")]), // stacked header again+ row(4, &[txt(1, "Bob"), num(2, 20)]),+ ];+ let (stats, recs) = collect_table("S", &rows);+ assert_eq!(stats.records, 2);+ assert_eq!(fields(&recs[0]), json!({"Name": "Ann", "Score": 10}));+ assert_eq!(fields(&recs[1]), json!({"Name": "Bob", "Score": 20}));+ }++ #[test]+ fn duplicate_header_names_are_disambiguated() {+ let rows = vec![+ row(1, &[txt(1, "Amount"), txt(2, "Amount")]),+ row(2, &[num(1, 1), num(2, 2)]),+ ];+ let (_stats, recs) = collect_table("S", &rows);+ let obj = recs[0].fields.clone();+ assert!(obj.contains_key("Amount") && obj.contains_key("Amount_2"));+ }++ #[test]+ fn a_sheet_with_no_header_is_emitted_as_a_document() {+ // Every row leads with a number, so no row is a header candidate.+ let rows = vec![+ row(1, &[num(1, 1), txt(2, "alpha")]),+ row(2, &[num(1, 2), txt(2, "beta")]),+ ];+ assert!(find_header(&rows).is_none());+ let mut recs = Vec::new();+ let mut stats = ExtractStats::default();+ emit_document_sheet(+ "Notes",+ "book",+ &rows,+ &mut |r| {+ recs.push(r);+ true+ },+ &mut stats,+ );+ assert_eq!(stats.records, 1);+ let body = recs[0].fields["body"].as_str().unwrap();+ assert!(body.contains("1 | alpha") && body.contains("2 | beta"));+ assert_eq!(recs[0].fields["title"], "book — Notes");+ assert_eq!(recs[0].fields["sheet"], "Notes");+ }++ #[test]+ fn date_cell_formats_match_the_xlsx_extractor() {+ // calamine resolves an ExcelDateTime; check the bare-date vs datetime+ // split against a serial we can predict (1904-01-01 is serial 1462 in+ // the 1900 system; use a known midnight serial instead).+ use calamine::ExcelDateTime;+ // 2025-07-01 00:00:00 — serial 45839 in the 1900 date system.+ let d = Data::DateTime(ExcelDateTime::new(+ 45839.0,+ calamine::ExcelDateTimeType::DateTime,+ false,+ ));+ let cell = date_cell(1, &d).unwrap();+ assert_eq!(cell.kind, Kind::Date);+ assert_eq!(cell.value, json!("2025-07-01"));+ }+}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex e89b1fea2..9c20cbb47 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -1841,6 +1841,7 @@ fn sample_limit_bytes(family: Family, path: &Path) -> Option<u64> {Family::Mbox => Some(SAMPLE_LIMIT_BYTES),Family::Sqlite => Some(1), // signals per-table row cap inside the extractorFamily::Xlsx => Some(1), // likewise a per-sheet row cap+ Family::Xls => Some(1), // legacy workbook: same per-sheet row cap_ => None, // whole-file extractors cap themselves}}@@ -2663,6 +2664,7 @@ fn scan_file(| Family::TxtProse| Family::Pdf| Family::Docx+ | Family::Doc| Family::Pptx| Family::Epub| Family::Ipynb@@ -2692,7 +2694,7 @@ fn scan_file(// untyped.let grouped_family = matches!(sn.family,- Family::SqlDump | Family::Sqlite | Family::Xlsx | Family::UnityYaml+ Family::SqlDump | Family::Sqlite | Family::Xlsx | Family::Xls | Family::UnityYaml);let mut sink = |rec: extract::RawRecord| -> bool {let entry = groups.entry(rec.group.clone()).or_default();diff --git a/engine/crates/xerj-autoindex/src/order.rs b/engine/crates/xerj-autoindex/src/order.rsindex 35bb18413..dac58ab90 100644--- a/engine/crates/xerj-autoindex/src/order.rs+++ b/engine/crates/xerj-autoindex/src/order.rs@@ -186,6 +186,7 @@ pub fn band(rel: &str, family: Family) -> Band {| Family::Html| Family::Pdf| Family::Docx+ | Family::Doc| Family::Pptx| Family::Epub| Family::Ipynb@@ -193,7 +194,12 @@ pub fn band(rel: &str, family: Family) -> Band {| Family::Eml| Family::Mbox => Band::SourceAndDocs,Family::Yaml | Family::Json | Family::Xml => Band::Config,- Family::Csv | Family::Jsonl | Family::Sqlite | Family::SqlDump | Family::Xlsx => Band::Data,+ Family::Csv+ | Family::Jsonl+ | Family::Sqlite+ | Family::SqlDump+ | Family::Xlsx+ | Family::Xls => Band::Data,Family::Logs | Family::TxtLines => Band::Bulk,// Unity assets are the project's own authored content — scenes,// prefabs and materials are to a Unity repo what source files are to@@ -227,10 +233,10 @@ pub fn band_from_family_str(rel: &str, family: &str) -> Band {// docs, so a RESUMED run queued every email behind the bulk data the// planning run had put it ahead of. The agreement test did not catch it// because its family list was written by hand and omitted `Eml` too.- "code" | "txt-prose" | "html" | "pdf" | "docx" | "pptx" | "epub" | "ipynb" | "man"- | "unity" | "eml" | "mbox" => Band::SourceAndDocs,+ "code" | "txt-prose" | "html" | "pdf" | "docx" | "doc" | "pptx" | "epub" | "ipynb"+ | "man" | "unity" | "eml" | "mbox" => Band::SourceAndDocs,"yaml" | "json" | "xml" | "unity-meta" => Band::Config,- "csv" | "jsonl" | "sqlite" | "sqldump" | "xlsx" | "bvh" => Band::Data,+ "csv" | "jsonl" | "sqlite" | "sqldump" | "xlsx" | "xls" | "bvh" => Band::Data,// Pre-existing drift, caught by// `estimate::tests::the_two_band_functions_agree_for_every_family`:// `band` ranks Binary `Vendored`, but the catch-all below ranked thediff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex bbb522b58..56c93a958 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -34,6 +34,18 @@ pub enum Family {/// Excel workbook: one dataset per worksheet, one record per row under/// the sheet's header row, typed from the stored cell values.Xlsx,+ /// Legacy Excel 97-2003 workbook (`.xls`, BIFF8 inside an OLE2 compound+ /// file): same table model as [`Family::Xlsx`] — one dataset per+ /// worksheet, one record per row under the header row — read through+ /// calamine. Detected by the OLE2 magic plus a `Workbook`/`Book` stream,+ /// never by extension.+ Xls,+ /// Legacy Word 97-2003 document (`.doc`, inside an OLE2 compound file):+ /// the WordDocument stream's prose, recovered through the FIB piece table+ /// (CLX in the 0Table/1Table stream), emitted as section records with the+ /// title taken from SummaryInformation when present. Detected by the OLE2+ /// magic plus a `WordDocument` stream, never by extension.+ Doc,/// EPUB e-book: chapters in reading order (spine documents, cut at their/// table-of-contents anchors), with the book's metadata and each/// chapter's title from the table of contents.@@ -85,6 +97,8 @@ impl Family {Family::Docx => "docx",Family::Pptx => "pptx",Family::Xlsx => "xlsx",+ Family::Xls => "xls",+ Family::Doc => "doc",Family::Epub => "epub",Family::Ipynb => "ipynb",Family::Man => "man",@@ -104,6 +118,7 @@ impl Family {self,Family::Pdf| Family::Docx+ | Family::Doc| Family::Pptx| Family::Epub| Family::Ipynb@@ -143,6 +158,10 @@ pub struct Sniffed {/// Bytes read for ordinary classification.const PREFIX: usize = 8192;+/// OLE2 / Compound File Binary header signature (MS-CFB §2.2). Legacy Office+/// (`.doc`/`.xls`/`.ppt`), 1C `.epf`, MSI and others all begin with it.+const OLE2_MAGIC: &[u8] = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1";+/// Bytes read when the file already carries GIF magic. A Comment Extension is/// unbounded by the format, so this is a budget rather than a guarantee: it/// covers every commented GIF in the measured corpus with room to spare.@@ -270,6 +289,31 @@ fn sniff_bytes(s.binary_kind = Some(kind.into());return Ok(s);}+ // OLE2 / Compound File Binary (`.doc`, `.xls`, `.ppt`, 1C `.epf`, MSI, …):+ // one 8-byte magic for a whole family of unrelated formats, so — exactly as+ // with the zip container above — the magic alone cannot say which. Open the+ // compound file (bounded: `cfb` reads the header, FAT and directory, never+ // the whole file) and route by the top-level stream names a `Workbook`/+ // `Book` stream is a legacy Excel workbook, a `WordDocument` stream a legacy+ // Word document, anything else an OLE2 blob we name but do not parse. A+ // crafted or truncated container must not be able to panic or hang the+ // sniffer, so `cfb::CompoundFile::open` (which reports malformation as an+ // `io::Err`, not a panic) is the whole mechanism and every error path lands+ // on `Family::Binary` / `"ole2"`.+ if prefix.starts_with(OLE2_MAGIC) {+ if !gzip {+ if let Ok(f) = std::fs::File::open(content_path) {+ match ole2_family(f) {+ Some(Family::Xls) => return Ok(mk(Family::Xls)),+ Some(Family::Doc) => return Ok(mk(Family::Doc)),+ _ => {}+ }+ }+ }+ let mut s = mk(Family::Binary);+ s.binary_kind = Some("ole2".into());+ return Ok(s);+ }// Compressed image/audio/model payloads routinely pass the NUL and// control-char heuristics below (windows-1252 decodes almost every byte// to something printable), and a multi-MB PSD misread as prose turns into@@ -1160,6 +1204,34 @@ fn looks_like_tar_header(prefix: &[u8]) -> bool {/// folder of XML parts that is no more searchable than the file was. OOXML is/// identified by its main part; OpenDocument and EPUB by the `mimetype` member/// their container specs require, read through a bounded reader.+/// Route an OLE2 / Compound File Binary container by its top-level stream+/// names: [`Family::Xls`] for a `Workbook`/`Book` stream, [`Family::Doc`] for+/// a `WordDocument` stream, and `None` for anything else (an unrecognised OLE2+/// blob — PowerPoint, Visio, MSI, a 1C `.epf`, a crafted or truncated file).+///+/// `None` deliberately covers every failure: `cfb::CompoundFile::open` reports+/// a malformed, truncated or cyclic container as an `io::Err` rather than a+/// panic, and it reads only the header, FAT and directory (bounded by the file+/// size, since a declared sector past EOF fails the read), so this is safe on+/// hostile input and never has to open a stream to decide. The `.doc` ↔ `.xls`+/// precedence (Word before Excel) matches real files, which carry only one of+/// the two main streams.+fn ole2_family<F: Read + std::io::Seek>(f: F) -> Option<Family> {+ let comp = cfb::CompoundFile::open(f).ok()?;+ let mut xls = false;+ for entry in comp.read_root_storage() {+ if !entry.is_stream() {+ continue;+ }+ match entry.name() {+ "WordDocument" => return Some(Family::Doc),+ "Workbook" | "Book" => xls = true,+ _ => {}+ }+ }+ xls.then_some(Family::Xls)+}+fn zip_container_kind<R: Read + std::io::Seek>(z: &mut zip::ZipArchive<R>) -> &'static str {const PARTS: &[(&str, &str)] = &[("word/document.xml", "docx"),@@ -4466,3 +4538,99 @@ mod data_container_tests {}}}++#[cfg(test)]+mod ole2_sniff_tests {+ use super::*;+ use std::io::Write;+ use std::path::Path;++ /// Write a synthetic OLE2 compound file carrying exactly the named streams+ /// (each with a byte of content), and return its path inside `dir`. Built+ /// with cfb's own writer, so no real Office file is ever needed.+ fn write_ole2(dir: &tempfile::TempDir, file: &str, streams: &[&str]) -> std::path::PathBuf {+ let path = dir.path().join(file);+ {+ let mut comp = cfb::create(&path).unwrap();+ for name in streams {+ let mut s = comp.create_stream(name).unwrap();+ s.write_all(b"x").unwrap();+ s.flush().unwrap();+ }+ comp.flush().unwrap();+ }+ path+ }++ /// A `Workbook` stream inside an OLE2 container routes to `Xls`, by content+ /// and not by the file's extension.+ #[test]+ fn a_workbook_stream_is_detected_as_xls() {+ let dir = tempfile::tempdir().unwrap();+ let path = write_ole2(&dir, "legacy.bin", &["Workbook"]);+ assert_eq!(sniff(&path).unwrap().family, Family::Xls);+ // The pre-BIFF8 name `Book` also counts.+ let path = write_ole2(&dir, "old.bin", &["Book"]);+ assert_eq!(sniff(&path).unwrap().family, Family::Xls);+ }++ /// A `WordDocument` stream routes to `Doc`.+ #[test]+ fn a_worddocument_stream_is_detected_as_doc() {+ let dir = tempfile::tempdir().unwrap();+ let path = write_ole2(+ &dir,+ "report.bin",+ &["WordDocument", "\u{5}SummaryInformation"],+ );+ assert_eq!(sniff(&path).unwrap().family, Family::Doc);+ }++ /// Any other OLE2 container (PowerPoint, Visio, a 1C `.epf`, …) is named+ /// `ole2` and left for the skip report — never parsed, never a panic.+ #[test]+ fn an_unrecognised_ole2_container_is_named_but_not_parsed() {+ let dir = tempfile::tempdir().unwrap();+ let path = write_ole2(&dir, "deck.bin", &["PowerPoint Document"]);+ let sn = sniff(&path).unwrap();+ assert_eq!(sn.family, Family::Binary);+ assert_eq!(sn.binary_kind.as_deref(), Some("ole2"));+ }++ /// The OLE2 magic followed by garbage (a truncated/crafted compound file)+ /// must classify as binary `ole2` without panicking or hanging.+ #[test]+ fn a_truncated_ole2_header_is_binary_not_a_panic() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("broken.bin");+ let mut bytes = OLE2_MAGIC.to_vec();+ bytes.extend_from_slice(&[0xFF; 600]); // nonsense header fields+ std::fs::write(&path, &bytes).unwrap();+ let sn = sniff(&path).unwrap();+ assert_eq!(sn.family, Family::Binary);+ assert_eq!(sn.binary_kind.as_deref(), Some("ole2"));+ }++ /// A `.ssc`-style spreadsheet export is a TEXT XML body, not an OLE2+ /// container, and must stay `Xml` — pinned so the new OLE2 routing (and any+ /// future "spreadsheet" heuristic) never steals it.+ #[test]+ fn an_ssc_xml_spreadsheet_body_stays_xml() {+ let body = concat!(+ r#"<?xml version="1.0" encoding="UTF-8"?>"#,+ "\n",+ r#"<spreadsheet-workbook-data version="2" generator="xerj">"#,+ "\n <sheet name=\"Totals\"><row><cell>Region</cell><cell>Revenue</cell></row></sheet>\n",+ r#"</spreadsheet-workbook-data>"#,+ );+ let fam = sniff_bytes(+ body.as_bytes(),+ Path::new("export.ssc"),+ Path::new("export.ssc"),+ false,+ )+ .unwrap()+ .family;+ assert_eq!(fam, Family::Xml);+ }+}