AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(xls): isolate the calamine BIFF read in an rlimit-capped subprocess
xerj-orgb85b459605f9
diff --git a/engine/crates/xerj-autoindex/src/extract/xls.rs b/engine/crates/xerj-autoindex/src/extract/xls.rsindex 657cecb3f..a3217b4f6 100644--- a/engine/crates/xerj-autoindex/src/extract/xls.rs+++ b/engine/crates/xerj-autoindex/src/extract/xls.rs@@ -33,6 +33,7 @@ use super::{use anyhow::Result;use calamine::{Data, DataType, Reader, Xls};use chrono::Timelike;+use serde::{Deserialize, Serialize};use serde_json::{Map, Value};use std::io::Read;use std::path::Path;@@ -85,7 +86,11 @@ fn xls_allocations_bounded(path: &Path) -> bool {return true;};let mut buf = Vec::new();- if stream.take(MAX_WORKBOOK_STREAM).read_to_end(&mut buf).is_err() {+ if stream+ .take(MAX_WORKBOOK_STREAM)+ .read_to_end(&mut buf)+ .is_err()+ {return true;}biff_allocations_bounded(&buf)@@ -166,15 +171,452 @@ fn biff_allocations_bounded(stream: &[u8]) -> bool {true}+/// Hidden worker argv[1] the isolated BIFF reader is dispatched on. Reachable+/// through the ordinary `xerj` binary (see `xerj-server/src/main.rs`), spawned+/// via `current_exe()` — the same mechanism the PDF worker and the `xerj brain`+/// server boot already use.+pub const WORKER_ARG: &str = "__extract-xls";++/// Test/developer hook: substitute the worker executable. Production uses+/// `current_exe()`. A cargo test's `current_exe()` is the *test* binary, which+/// does not carry the `__extract-xls` dispatch, so the isolation integration+/// test points this at the built `xerj` binary — mirrors `XERJ_PDF_WORKER_BIN`.+const WORKER_BIN_ENV: &str = "XERJ_XLS_WORKER_BIN";++/// Address-space ceiling imposed on the worker (`RLIMIT_AS`). A crafted+/// `DIMENSIONS`/`SST` that slips the pre-filter, or any other attacker-driven+/// `Vec::with_capacity`, hits this and fails the allocation inside the worker —+/// where it only kills the worker — instead of aborting the indexer.+#[cfg(unix)]+const WORKER_ADDRESS_SPACE: u64 = 2 << 30; // 2 GiB++/// CPU-time ceiling imposed on the worker (`RLIMIT_CPU`, seconds). A record+/// stream crafted to spin inside calamine is bounded here; the kernel sends+/// `SIGXCPU` at the soft limit and `SIGKILL` at the hard limit, either of which+/// the parent reports as junk.+#[cfg(unix)]+const WORKER_CPU_SECONDS: u64 = 15;++/// Wall-clock backstop for the whole worker, independent of CPU time — covers a+/// worker wedged in a blocking syscall that burns no CPU. `RLIMIT_CPU` is the+/// primary guard; this only fires if the worker is stuck without the kernel+/// noticing.+#[cfg(unix)]+const WORKER_WALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);++/// Upper bound on the NDJSON the parent will buffer from one worker. A legacy+/// `.xls` sheet is capped by the format at 65,536 rows × 256 columns, so a+/// well-formed workbook is far below this; a worker emitting more than this is+/// treated as junk rather than letting the parent's own memory grow unbounded+/// on the worker's say-so.+#[cfg(unix)]+const MAX_WORKER_OUTPUT: usize = 128 << 20; // 128 MiB++/// One line of the parent↔worker protocol. The worker serializes every produced+/// record as a `Rec` line and closes with exactly one `End` line carrying the+/// truncation flag; the parent parses them back into the identical records the+/// in-process path would have emitted. Internally tagged on `k` so a record+/// line (which never carries a `k` field) and the terminator are distinguished+/// without a second pass.+#[derive(Debug, Serialize, Deserialize)]+#[serde(tag = "k")]+enum WorkerLine {+ /// One extracted record, byte-for-byte the `RawRecord` the sink receives.+ #[serde(rename = "rec")]+ Rec(RawRecord),+ /// Terminator: the worker finished the file cleanly. Its presence is what+ /// tells the parent the stream was not cut short by a crash mid-emit.+ #[serde(rename = "end")]+ End { truncated: bool },+}+/// `name` is the file as the corpus names it (under durable preparation this is/// not `path`, a sealed snapshot blob); document sheets are titled from it./// `per_sheet_limit` caps the rows emitted from each sheet (phase-A sampling);/// `None` reads every row.+///+/// The dump being indexed is untrusted (Jira/Confluence/GitLab attachments) and+/// calamine 0.26.1 is *not* panic-safe on hostile BIFF: beyond the allocation+/// vectors [`xls_allocations_bounded`] pre-filters, it indexes a short `Lbl`+/// (`0x0018`) or `ExternSheet` (`0x0017`) record straight out of bounds and+/// panics. The engine release profile is `panic = abort` with no `catch_unwind`+/// around the scan, so such a panic aborts the WHOLE indexer, and `catch_unwind`+/// cannot help under `panic = abort`. So on unix the actual BIFF read runs in a+/// short-lived child process under `RLIMIT_AS`/`RLIMIT_CPU`: a panic, abort,+/// OOM-kill or runaway there dies with the child and is counted as junk, and the+/// parent never calls calamine on the raw file itself. Non-unix keeps the+/// in-process bounded path as a documented best-effort (the dump is Linux).pub fn extract(path: &Path,name: &Path,per_sheet_limit: Option<u64>,sink: Sink,+) -> Result<ExtractStats> {+ // Cheap pre-filter, kept from #1324: refuse the whole crafted-allocation+ // class before paying to spawn a worker at all. It is a pre-filter now, not+ // the whole defence — the worker's rlimits catch what byte-walking a record+ // stream cannot (every other allocation site, and the out-of-bounds panics).+ if !xls_allocations_bounded(path) {+ let mut stats = ExtractStats::default();+ stats.junk += 1;+ return Ok(stats);+ }+ #[cfg(unix)]+ {+ extract_isolated(path, name, per_sheet_limit, sink)+ }+ #[cfg(not(unix))]+ {+ // No `setrlimit`/`pre_exec` here; best-effort in-process read bounded by+ // the pre-filter above. Documented: the production dump is Linux.+ extract_in_process(path, name, per_sheet_limit, sink)+ }+}++/// Parent side of the isolation: spawn `current_exe() __extract-xls <path>+/// <name> <limit>`, read its NDJSON, and deliver the records to `sink` exactly+/// as the in-process path would. Any abnormal exit — non-zero, signal+/// (SIGABRT/SIGKILL/SIGXCPU), wall-clock timeout, over-cap output, a line that+/// will not parse, or a missing `End` terminator — becomes `junk += 1` and+/// `Ok`, so one hostile file is skipped and the scan continues. All-or-nothing:+/// records are buffered and only delivered once the worker has closed cleanly,+/// so a worker that crashes mid-file indexes nothing from it (no partial rows).+#[cfg(unix)]+fn extract_isolated(+ path: &Path,+ name: &Path,+ per_sheet_limit: Option<u64>,+ sink: Sink,+) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ match run_worker(path, name, per_sheet_limit) {+ Ok((records, truncated)) => {+ for rec in records {+ stats.records += 1;+ if !sink(rec) {+ // Sampling stop: the caller has what it wanted. Success.+ return Ok(stats);+ }+ }+ if stats.records == 0 {+ // Clean run that produced nothing (unreadable-but-not-crafted+ // workbook, or an empty one) — junk, mirroring the in-process+ // `records == 0` rule.+ stats.junk += 1;+ } else {+ stats.truncated = truncated;+ }+ }+ // Crash / timeout / protocol failure: skip the file, keep scanning.+ Err(_) => stats.junk += 1,+ }+ Ok(stats)+}++/// Run one worker to completion and return its buffered records plus the+/// truncation flag, or an error describing why the file could not be trusted.+#[cfg(unix)]+fn run_worker(+ path: &Path,+ name: &Path,+ per_sheet_limit: Option<u64>,+) -> Result<(Vec<RawRecord>, bool)> {+ use anyhow::{anyhow, Context};+ use std::os::unix::process::CommandExt;+ use std::process::{Command, Stdio};++ let exe = std::env::var_os(WORKER_BIN_ENV)+ .map(std::path::PathBuf::from)+ .or_else(|| std::env::current_exe().ok())+ .context("locate the xerj binary for isolated .xls extraction")?;++ let mut command = Command::new(exe);+ command+ .arg(WORKER_ARG)+ .arg(path.as_os_str())+ .arg(name.as_os_str())+ .arg(match per_sheet_limit {+ Some(n) => n.to_string(),+ None => "none".to_string(),+ })+ .stdin(Stdio::null())+ .stdout(Stdio::piped())+ .stderr(Stdio::null());+ // SAFETY: the closure runs in the forked child before exec and calls only+ // async-signal-safe libc (`setpgid`, `setrlimit`). Its own process group so+ // the parent can reap the whole tree; the rlimits cap the child's blast+ // radius to itself.+ unsafe {+ command.pre_exec(worker_pre_exec);+ }++ let mut child = command+ .spawn()+ .with_context(|| format!("spawn the isolated .xls reader for {}", path.display()))?;+ let pid = child.id();++ let stdout = child.stdout.take().expect("piped stdout");+ // Read on a thread so the wall-clock watchdog below can act even if the+ // worker stalls with the pipe open.+ let reader = std::thread::spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT));++ let started = std::time::Instant::now();+ let status = loop {+ match child.try_wait() {+ Ok(Some(status)) => break status,+ Ok(None) => {}+ Err(e) => {+ kill_group(pid);+ let _ = child.wait();+ let _ = reader.join();+ return Err(e)+ .with_context(|| format!("monitor the .xls reader for {}", path.display()));+ }+ }+ if started.elapsed() >= WORKER_WALL_TIMEOUT {+ kill_group(pid);+ let _ = child.wait();+ let _ = reader.join();+ return Err(anyhow!(+ "isolated .xls reader for {} exceeded the {}s wall-clock limit",+ path.display(),+ WORKER_WALL_TIMEOUT.as_secs()+ ));+ }+ std::thread::sleep(std::time::Duration::from_millis(10));+ };+ // Reap any stragglers left in the group, then collect the output.+ kill_group(pid);+ let captured = reader+ .join()+ .map_err(|_| anyhow!("the .xls reader output thread panicked"))??;+ if captured.overflow {+ return Err(anyhow!(+ "isolated .xls reader for {} produced more than {} MiB of output",+ path.display(),+ MAX_WORKER_OUTPUT >> 20+ ));+ }+ if !status.success() {+ return Err(anyhow!(+ "isolated .xls reader for {} exited abnormally ({status})",+ path.display()+ ));+ }+ parse_worker_output(&captured.bytes).with_context(|| {+ format!(+ "parse the isolated .xls reader output for {}",+ path.display()+ )+ })+}++/// Parse the worker's NDJSON into the records and truncation flag. A clean run+/// ends with exactly one `End` line; its absence means the stream was cut short+/// (a crash the exit status somehow did not surface) and is an error.+#[cfg(unix)]+fn parse_worker_output(bytes: &[u8]) -> Result<(Vec<RawRecord>, bool)> {+ use anyhow::anyhow;+ let mut records = Vec::new();+ let mut ended: Option<bool> = None;+ for line in bytes.split(|&b| b == b'\n') {+ if line.is_empty() {+ continue;+ }+ if ended.is_some() {+ return Err(anyhow!("trailing output after the .xls reader terminator"));+ }+ match serde_json::from_slice::<WorkerLine>(line)? {+ WorkerLine::Rec(rec) => records.push(rec),+ WorkerLine::End { truncated } => ended = Some(truncated),+ }+ }+ match ended {+ Some(truncated) => Ok((records, truncated)),+ None => Err(anyhow!("the .xls reader did not emit a terminator")),+ }+}++/// Pre-exec child setup: own process group + address-space and CPU-time caps.+/// Runs in the forked child; only async-signal-safe libc calls.+#[cfg(unix)]+fn worker_pre_exec() -> std::io::Result<()> {+ // SAFETY: `setpgid`/`setrlimit` are async-signal-safe and documented here.+ unsafe {+ if libc::setpgid(0, 0) != 0 {+ return Err(std::io::Error::last_os_error());+ }+ }+ set_worker_rlimits()+}++/// Apply `RLIMIT_AS` and `RLIMIT_CPU` to the current process. Used by+/// [`worker_pre_exec`] (parent-forked child) and again by the worker itself at+/// entry (belt-and-suspenders, in case it was spawned some other way).+///+/// Only ever *lowers* a limit: an unprivileged process cannot raise its hard+/// limit, so the target is clamped to whatever was inherited. This matters when+/// the indexer already runs under a tighter cgroup or `ulimit -v` — trying to+/// set a higher ceiling there would `EPERM` and fail the whole spawn, turning+/// every `.xls` (benign included) into junk.+#[cfg(unix)]+fn set_worker_rlimits() -> std::io::Result<()> {+ // Set `$res`'s soft/hard limits to the requested values, each capped at the+ // inherited hard limit so the call never tries to RAISE a ceiling (which an+ // unprivileged process may not do, and which would `EPERM` the whole spawn).+ // A macro, not a function, so the resource constant is passed straight to+ // libc without naming its type — that type is `__rlimit_resource_t` on+ // glibc but `c_int` on macOS/BSD.+ macro_rules! clamp {+ ($res:expr, $soft:expr, $hard:expr) => {{+ let mut current = libc::rlimit {+ rlim_cur: 0,+ rlim_max: 0,+ };+ // SAFETY: plain libc FFI over an out-param `rlimit`.+ if unsafe { libc::getrlimit($res, &mut current) } != 0 {+ return Err(std::io::Error::last_os_error());+ }+ let ceiling = current.rlim_max; // inherited hard limit (maybe RLIM_INFINITY)+ let want_hard = ($hard as libc::rlim_t).min(ceiling);+ let want_soft = ($soft as libc::rlim_t).min(want_hard);+ let limit = libc::rlimit {+ rlim_cur: want_soft,+ rlim_max: want_hard,+ };+ // SAFETY: as above; cur <= max <= inherited max, so this only lowers.+ if unsafe { libc::setrlimit($res, &limit) } != 0 {+ return Err(std::io::Error::last_os_error());+ }+ }};+ }+ clamp!(libc::RLIMIT_AS, WORKER_ADDRESS_SPACE, WORKER_ADDRESS_SPACE);+ // Soft limit sends SIGXCPU; hard limit (a touch higher) escalates to SIGKILL.+ clamp!(libc::RLIMIT_CPU, WORKER_CPU_SECONDS, WORKER_CPU_SECONDS + 2);+ Ok(())+}++/// SIGKILL the worker's whole process group (negative pid). Best effort.+#[cfg(unix)]+fn kill_group(pid: u32) {+ // SAFETY: plain libc FFI; an already-dead group is a harmless ESRCH.+ unsafe {+ libc::kill(-(pid as i32), libc::SIGKILL);+ }+}++#[cfg(unix)]+struct Captured {+ bytes: Vec<u8>,+ overflow: bool,+}++/// Read `input` to EOF, keeping at most `cap` bytes and flagging overflow.+/// Keeps draining past the cap so the child never blocks on a full pipe.+#[cfg(unix)]+fn read_capped(mut input: impl Read, cap: usize) -> Result<Captured> {+ use anyhow::Context;+ let mut bytes = Vec::new();+ let mut overflow = false;+ let mut chunk = [0u8; 64 << 10];+ loop {+ let n = input+ .read(&mut chunk)+ .context("read isolated .xls reader output")?;+ if n == 0 {+ break;+ }+ if !overflow {+ let room = cap.saturating_sub(bytes.len());+ let take = room.min(n);+ bytes.extend_from_slice(&chunk[..take]);+ overflow |= take < n;+ }+ }+ Ok(Captured { bytes, overflow })+}++/// Hidden same-binary worker entry point (`xerj __extract-xls <path> <name>+/// <limit|none>`). One invocation reads one workbook and streams its records as+/// NDJSON on stdout. If calamine panics/aborts here, this process dies and the+/// parent handles it; nothing in the parent ever touches the raw file.+pub fn run_worker_cli() -> i32 {+ // Re-apply the resource caps the parent set via `pre_exec`, so the worker is+ // bounded even if it was launched directly (tests, a future caller).+ #[cfg(unix)]+ if let Err(e) = set_worker_rlimits() {+ eprintln!(".xls worker could not set its resource limits: {e}");+ // Not fatal: the parent's `pre_exec` is the authoritative cap.+ }++ let mut args = std::env::args_os().skip(2);+ let (Some(path), Some(name), Some(limit)) = (args.next(), args.next(), args.next()) else {+ eprintln!(".xls worker protocol error: expected <path> <name> <limit>");+ return 2;+ };+ if args.next().is_some() {+ eprintln!(".xls worker protocol error: unexpected trailing arguments");+ return 2;+ }+ let per_sheet_limit = match limit.to_string_lossy().as_ref() {+ "none" => None,+ other => match other.parse::<u64>() {+ Ok(n) => Some(n),+ Err(_) => {+ eprintln!(".xls worker protocol error: bad limit {other:?}");+ return 2;+ }+ },+ };++ let path = std::path::PathBuf::from(path);+ let name = std::path::PathBuf::from(name);++ use std::io::Write;+ let stdout = std::io::stdout();+ let mut out = std::io::BufWriter::new(stdout.lock());+ let mut write_err = false;+ let mut emit = |rec: RawRecord| -> bool {+ if serde_json::to_writer(&mut out, &WorkerLine::Rec(rec)).is_err()+ || out.write_all(b"\n").is_err()+ {+ write_err = true;+ return false;+ }+ true+ };++ match extract_in_process(&path, &name, per_sheet_limit, &mut emit) {+ Ok(stats) => {+ if write_err {+ return 1;+ }+ let end = WorkerLine::End {+ truncated: stats.truncated,+ };+ if serde_json::to_writer(&mut out, &end).is_err()+ || out.write_all(b"\n").is_err()+ || out.flush().is_err()+ {+ return 1;+ }+ 0+ }+ Err(e) => {+ eprintln!(".xls worker failed: {e:#}");+ 1+ }+ }+}++/// The in-process BIFF read. On unix this runs *inside the worker*; on non-unix+/// it is the whole (best-effort) path. It still runs the allocation pre-filter+/// itself, so it is self-contained wherever it is called from.+fn extract_in_process(+ path: &Path,+ name: &Path,+ per_sheet_limit: Option<u64>,+ sink: Sink,) -> Result<ExtractStats> {let mut stats = ExtractStats::default();// DoS gate: calamine allocates a dense `Range` from the sheet's declared@@ -549,26 +991,39 @@ mod tests {// A 100-row × 10-column sheet is well within the BIFF8 ceiling.assert!(biff_allocations_bounded(&dimensions_record(0, 100, 0, 10)));// And the format's own maximum is still accepted.- assert!(biff_allocations_bounded(&dimensions_record(0, 65_536, 0, 256)));+ assert!(biff_allocations_bounded(&dimensions_record(+ 0, 65_536, 0, 256+ )));}#[test]fn a_crafted_dimension_past_the_biff_ceiling_is_refused() {// The shape of the 36 KB proof-of-concept: a DIMENSIONS claiming a vast// row range, which drove calamine to attempt a ~2.1 TB Range alloc.- assert!(!biff_allocations_bounded(&dimensions_record(0, 0x7FFF_FFFF, 0, 256)));+ assert!(!biff_allocations_bounded(&dimensions_record(+ 0,+ 0x7FFF_FFFF,+ 0,+ 256+ )));// An inflated column count is refused the same way.- assert!(!biff_allocations_bounded(&dimensions_record(0, 100, 0, 60_000)));+ assert!(!biff_allocations_bounded(&dimensions_record(+ 0, 100, 0, 60_000+ )));// Inverted ranges (mac < mic) are malformed and refused.- assert!(!biff_allocations_bounded(&dimensions_record(500, 100, 0, 10)));+ assert!(!biff_allocations_bounded(&dimensions_record(+ 500, 100, 0, 10+ )));}#[test]fn a_truncated_or_dimensionless_stream_is_not_mistaken_for_crafted() {// No DIMENSIONS record at all: nothing to object to, let calamine try.- assert!(biff_allocations_bounded(b"\x09\x08\x04\x00\x00\x00\x00\x00")); // a BOF-ish record, no 0x0200- // A record header whose declared length runs past the buffer end stops- // the walk without a panic and without a false refusal.+ assert!(biff_allocations_bounded(+ b"\x09\x08\x04\x00\x00\x00\x00\x00"+ )); // a BOF-ish record, no 0x0200+ // A record header whose declared length runs past the buffer end stops+ // the walk without a panic and without a false refusal.assert!(biff_allocations_bounded(&[0x00, 0x02, 0xFF, 0xFF, 0x01]));// Empty stream.assert!(biff_allocations_bounded(&[]));@@ -597,7 +1052,9 @@ mod tests {#[test]fn the_10_byte_dimensions_form_is_also_bounded() {// A real small BIFF5 sheet is accepted.- assert!(biff_allocations_bounded(&dimensions_record_biff5(0, 100, 0, 10)));+ assert!(biff_allocations_bounded(&dimensions_record_biff5(+ 0, 100, 0, 10+ )));// The reviewer's bypass: rl = cl = 0xFFFF drove calamine to a ~171 GB// reserve because the earlier gate only checked the 14-byte form. The// 65,535-column claim is past the 256 ceiling and is now refused.@@ -805,4 +1262,82 @@ mod tests {assert_eq!(cell.kind, Kind::Date);assert_eq!(cell.value, json!("2025-07-01"));}++ fn rec(locator: &str) -> RawRecord {+ let mut fields = Map::new();+ fields.insert("Region".into(), Value::String("EMEA".into()));+ fields.insert("Revenue".into(), Value::Number(1250.into()));+ RawRecord {+ fields,+ locator: locator.to_string(),+ group: Some("Sales".into()),+ origin: FieldOrigin::Data,+ }+ }++ /// The parent reconstructs the worker's records byte-for-byte: a `Rec` line+ /// round-trips every field, the locator, the group and the origin, and the+ /// `End` line is not mistaken for a record.+ #[test]+ fn worker_line_rec_and_end_round_trip() {+ let line = serde_json::to_string(&WorkerLine::Rec(rec("Sales!r4"))).unwrap();+ // A record line carries no discriminator a RawRecord could own.+ assert!(line.contains("\"k\":\"rec\""));+ match serde_json::from_str::<WorkerLine>(&line).unwrap() {+ WorkerLine::Rec(r) => {+ assert_eq!(r.locator, "Sales!r4");+ assert_eq!(r.group.as_deref(), Some("Sales"));+ assert_eq!(r.origin, FieldOrigin::Data);+ assert_eq!(r.fields["Region"], json!("EMEA"));+ assert_eq!(r.fields["Revenue"], json!(1250));+ }+ WorkerLine::End { .. } => panic!("record line decoded as terminator"),+ }+ let end = serde_json::to_string(&WorkerLine::End { truncated: true }).unwrap();+ assert!(matches!(+ serde_json::from_str::<WorkerLine>(&end).unwrap(),+ WorkerLine::End { truncated: true }+ ));+ }++ /// The NDJSON parser rebuilds the record list and the truncation flag, and+ /// tolerates the blank splits a trailing newline produces.+ #[cfg(unix)]+ #[test]+ fn parse_worker_output_reconstructs_records_and_truncation() {+ let mut buf = Vec::new();+ buf.extend_from_slice(serde_json::to_string(&WorkerLine::Rec(rec("Sales!r4"))).unwrap().as_bytes());+ buf.push(b'\n');+ buf.extend_from_slice(serde_json::to_string(&WorkerLine::Rec(rec("Sales!r5"))).unwrap().as_bytes());+ buf.push(b'\n');+ buf.extend_from_slice(+ serde_json::to_string(&WorkerLine::End { truncated: true })+ .unwrap()+ .as_bytes(),+ );+ buf.push(b'\n');+ let (records, truncated) = parse_worker_output(&buf).unwrap();+ assert_eq!(records.len(), 2);+ assert_eq!(records[0].locator, "Sales!r4");+ assert_eq!(records[1].locator, "Sales!r5");+ assert!(truncated);+ }++ /// A stream cut short before the terminator — exactly the shape of a worker+ /// that crashed mid-emit after flushing a record — is an error, so the+ /// parent discards the partial file rather than indexing half of it.+ #[cfg(unix)]+ #[test]+ fn parse_worker_output_rejects_a_missing_terminator() {+ let mut buf = Vec::new();+ buf.extend_from_slice(serde_json::to_string(&WorkerLine::Rec(rec("Sales!r4"))).unwrap().as_bytes());+ buf.push(b'\n');+ assert!(parse_worker_output(&buf).is_err());+ // Garbage that is not our protocol is also rejected.+ assert!(parse_worker_output(b"{not json}\n").is_err());+ // A clean empty run (no rows, just a terminator) is valid and empty.+ let end = serde_json::to_string(&WorkerLine::End { truncated: false }).unwrap();+ let (records, truncated) = parse_worker_output(end.as_bytes()).unwrap();+ assert!(records.is_empty() && !truncated);+ }}diff --git a/engine/crates/xerj-autoindex/tests/xls_isolation.rs b/engine/crates/xerj-autoindex/tests/xls_isolation.rsnew file mode 100644index 000000000..a3033c38f--- /dev/null+++ b/engine/crates/xerj-autoindex/tests/xls_isolation.rs@@ -0,0 +1,133 @@+//! End-to-end proof that the isolated `.xls` reader keeps the indexer alive on+//! hostile BIFF input. These tests drive the REAL parent path+//! (`xerj_autoindex::extract::xls::extract`), which spawns the real `xerj`+//! binary as the rlimit-capped worker — so they need that binary and are+//! `#[ignore]`d by default. Run them against a release build, under a tight+//! address-space ulimit, exactly as the live threat model would see it:+//!+//! ```sh+//! cargo build --release -p xerj-server+//! XERJ_XLS_WORKER_BIN=$PWD/target/release/xerj \+//! bash -c 'ulimit -v 2000000; \+//! cargo test --release -p xerj-autoindex --test xls_isolation -- --ignored --nocapture'+//! ```+//!+//! A cargo test's own `current_exe()` is the test binary, which carries no+//! `__extract-xls` dispatch, so `XERJ_XLS_WORKER_BIN` must point at the built+//! `xerj` executable. The crafted files are synthesised here with the `cfb`+//! writer and never committed; the optional benign fixture (a real multi-sheet+//! `.xls`, e.g. produced by `xlwt`) is passed via `XERJ_XLS_BENIGN_FIXTURE`.++use std::io::Write;+use std::path::{Path, PathBuf};++use xerj_autoindex::extract::xls;+use xerj_autoindex::extract::{ExtractStats, RawRecord};++/// Write a synthetic OLE2 workbook whose `Workbook` stream is exactly `biff`.+fn write_workbook(dir: &Path, file: &str, biff: &[u8]) -> PathBuf {+ let path = dir.join(file);+ {+ let mut comp = cfb::create(&path).unwrap();+ let mut s = comp.create_stream("Workbook").unwrap();+ s.write_all(biff).unwrap();+ s.flush().unwrap();+ comp.flush().unwrap();+ }+ path+}++/// One BIFF record: `[type u16][len u16][body…]`.+fn record(typ: u16, body: &[u8]) -> Vec<u8> {+ let mut r = Vec::with_capacity(4 + body.len());+ r.extend_from_slice(&typ.to_le_bytes());+ r.extend_from_slice(&(body.len() as u16).to_le_bytes());+ r.extend_from_slice(body);+ r+}++/// A plausible BIFF8 workbook-globals BOF so calamine starts iterating records+/// normally and reaches the hostile one that follows.+fn bof() -> Vec<u8> {+ // vers=0x0600, dt=0x0005 (globals), rupBuild, rupYear, + reserved.+ record(0x0809, &[0x00, 0x06, 0x05, 0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])+}++fn drain(path: &Path) -> ExtractStats {+ let mut recs: Vec<RawRecord> = Vec::new();+ let stats = xls::extract(path, path, None, &mut |r| {+ recs.push(r);+ true+ })+ .expect("parent extract must return Ok — never propagate a worker crash");+ // The record count the sink saw must agree with the reported stats.+ assert_eq!(recs.len() as u64, stats.records);+ stats+}++/// The six crafted vectors from the adversarial review. Four are caught by the+/// allocation pre-filter (no worker spawned); two (`Lbl`/`ExternSheet`) slip the+/// pre-filter and crash calamine inside the worker. Every one must come back as+/// junk, with the parent returning `Ok` and the process still standing.+#[test]+#[ignore = "spawns the real xerj worker binary; run under ulimit with XERJ_XLS_WORKER_BIN"]+fn hostile_xls_yields_junk_and_the_parent_survives() {+ let tmp = tempfile::tempdir().unwrap();+ let dir = tmp.path();++ // 1) 14-byte DIMENSIONS, huge rwMac — pre-filter DoS vector.+ let mut dim14 = vec![0u8; 14];+ dim14[4..8].copy_from_slice(&0x7FFF_FFFFu32.to_le_bytes()); // rwMac+ dim14[10..12].copy_from_slice(&0x0100u16.to_le_bytes()); // colMac = 256+ let f1 = write_workbook(dir, "dim14.xls", &[bof(), record(0x0200, &dim14)].concat());++ // 2) 10-byte DIMENSIONS, rl = cl = 0xFFFF — pre-filter DoS vector.+ let mut dim10 = vec![0u8; 10];+ dim10[2..4].copy_from_slice(&0xFFFFu16.to_le_bytes()); // rl+ dim10[6..8].copy_from_slice(&0xFFFFu16.to_le_bytes()); // cl+ let f2 = write_workbook(dir, "dim10.xls", &[bof(), record(0x0200, &dim10)].concat());++ // 3) SST, huge cstUnique — pre-filter DoS vector.+ let mut sst_huge = vec![0u8; 8];+ sst_huge[4..8].copy_from_slice(&0x7FFF_FFFFi32.to_le_bytes());+ let f3 = write_workbook(dir, "sst_huge.xls", &[bof(), record(0x00FC, &sst_huge)].concat());++ // 4) SST, negative cstUnique — pre-filter DoS vector (calamine's unwrap).+ let mut sst_neg = vec![0u8; 8];+ sst_neg[4..8].copy_from_slice(&(-1i32).to_le_bytes());+ let f4 = write_workbook(dir, "sst_neg.xls", &[bof(), record(0x00FC, &sst_neg)].concat());++ // 5) Lbl (0x0018) with an 8-byte body — calamine does `&r.data[14..]`.+ let f5 = write_workbook(dir, "lbl.xls", &[bof(), record(0x0018, &[0u8; 8])].concat());++ // 6) ExternSheet (0x0017) with a 0-byte body — calamine read_u16 on < 2 bytes.+ let f6 = write_workbook(dir, "externsheet.xls", &[bof(), record(0x0017, &[])].concat());++ for path in [&f1, &f2, &f3, &f4, &f5, &f6] {+ let stats = drain(path);+ assert_eq!(+ (stats.records, stats.junk),+ (0, 1),+ "hostile file {} must be skipped as junk, got {stats:?}",+ path.display()+ );+ }+}++/// A real, benign multi-sheet workbook still indexes through the isolated path.+/// Skipped unless a fixture is supplied (calamine has no writer, so a genuine+/// BIFF file cannot be synthesised in-process).+#[test]+#[ignore = "needs a real .xls at XERJ_XLS_BENIGN_FIXTURE; run under ulimit with XERJ_XLS_WORKER_BIN"]+fn benign_xls_still_yields_rows_through_the_worker() {+ let Some(fixture) = std::env::var_os("XERJ_XLS_BENIGN_FIXTURE") else {+ eprintln!("skipping: set XERJ_XLS_BENIGN_FIXTURE to a real .xls to run this");+ return;+ };+ let path = PathBuf::from(fixture);+ let stats = drain(&path);+ assert!(+ stats.records > 0 && stats.junk == 0,+ "a benign workbook must yield real rows, got {stats:?}"+ );+}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 6396589a5..bb4c24ba4 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -1816,6 +1816,19 @@ fn main() -> Result<()> {std::process::exit(xerj_autoindex::extract::pdf::run_worker_cli());}+ // The isolated legacy-`.xls` BIFF reader. calamine 0.26.1 panics on crafted+ // records (short `Lbl`/`ExternSheet`) and the release profile is+ // `panic = abort`, so the read runs here, in a short-lived rlimit-capped+ // child, where a crash dies with the child instead of aborting the indexer.+ // Dispatched before the Tokio pool for the same smallness reason as the PDF+ // worker above.+ if matches!(+ std::env::args().nth(1).as_deref(),+ Some(xerj_autoindex::extract::xls::WORKER_ARG)+ ) {+ std::process::exit(xerj_autoindex::extract::xls::run_worker_cli());+ }+// `xerj mcp` — MCP stdio server (newline-delimited JSON-RPC on// stdin/stdout), proxying to an already-running XERJ node.//