AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): bound docx decompression + never index dotfiles/secrets
xerj-org4b9e27f2f0a6
diff --git a/engine/crates/xerj-autoindex/src/extract/docx.rs b/engine/crates/xerj-autoindex/src/extract/docx.rsindex 2a6f68616..cef0b370d 100644--- a/engine/crates/xerj-autoindex/src/extract/docx.rs+++ b/engine/crates/xerj-autoindex/src/extract/docx.rs@@ -20,9 +20,20 @@ pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {return Ok(stats);}};- let mut reader = Reader::from_reader(BufReader::new(entry));+ // SECURITY: bound the DECOMPRESSED read. `word/document.xml` inflates from+ // the zip at whatever ratio the author chose — a ~400 KB docx can expand to+ // hundreds of MB, and a single 400 MB `<w:t>` text run makes quick-xml+ // allocate that whole run for one event (measured 1.68 GB RSS from an 815 KB+ // file). Reading through a `Take` caps peak memory no matter the ratio; a+ // real document's document.xml is far below this. Past the cap the reader+ // hits EOF and the loop ends (a truncated tag is handled as junk below).+ const MAX_DECOMPRESSED_BYTES: u64 = 72 << 20;+ let capped = std::io::Read::take(entry, MAX_DECOMPRESSED_BYTES);+ let mut reader = Reader::from_reader(BufReader::new(capped));reader.config_mut().trim_text(false);+ // Extraction body cap (also bounds any single paragraph — see the Text arm).+ const MAX_BODY_BYTES: usize = 64 << 20;let mut body = String::new();let mut headings: Vec<String> = Vec::new();let mut para = String::new();@@ -49,7 +60,16 @@ pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {}}Ok(Event::Text(t)) => {- para.push_str(&t.unescape().unwrap_or_default());+ // SECURITY: `para` only flushes into `body` (and gets length-+ // checked) at `</w:p>`. A crafted docx with a single never-closed+ // `<w:p>` — an 815 KB file whose document.xml inflates to+ // hundreds of MB inside one paragraph — otherwise grows `para`+ // without bound: measured 1.68 GB RSS from that 815 KB input.+ // Stop appending once one paragraph reaches the body cap; the+ // per-paragraph text a real document holds is far below it.+ if para.len() < MAX_BODY_BYTES {+ para.push_str(&t.unescape().unwrap_or_default());+ }}Ok(Event::End(e)) => {if e.name().as_ref() == b"w:p" {@@ -73,7 +93,7 @@ pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {}}buf.clear();- if body.len() > 64 << 20 {+ if body.len() > MAX_BODY_BYTES {break;}}diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rsindex 02c2a7bad..27278f2c5 100644--- a/engine/crates/xerj-autoindex/src/walk.rs+++ b/engine/crates/xerj-autoindex/src/walk.rs@@ -25,7 +25,20 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {anyhow::bail!("{} is not a directory", root_canon.display());}let mut out = Vec::new();- for entry in walkdir::WalkDir::new(&root_canon).follow_links(follow_symlinks) {+ // SECURITY / hygiene: never index hidden files or descend into hidden+ // directories. Without this the walker happily indexed `.env` (secrets,+ // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a queryable+ // brain with no per-brain authorization — a real exposure for the+ // "point it at my project folder" use case. `filter_entry` prunes a hidden+ // directory before descending, so `.git/` is skipped whole. The root itself+ // (depth 0) is exempt so a brain over a dot-named folder still works.+ let walker = walkdir::WalkDir::new(&root_canon)+ .follow_links(follow_symlinks)+ .into_iter()+ .filter_entry(|e| {+ e.depth() == 0 || !e.file_name().to_str().is_some_and(|n| n.starts_with('.'))+ });+ for entry in walker {let entry = match entry {Ok(e) => e,Err(e) => {@@ -89,3 +102,58 @@ fn stable_path_id(path: &Path) -> String {fn stable_path_id(path: &Path) -> String {format!("other:{}", path.to_string_lossy())}++#[cfg(test)]+mod hidden_skip_tests {+ use super::walk;+ use std::fs;++ #[test]+ fn hidden_files_and_dirs_are_never_indexed() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ fs::write(root.join("README.md"), "hello").unwrap();+ fs::write(root.join(".env"), "SECRET=pat-abc123").unwrap();+ fs::create_dir(root.join(".git")).unwrap();+ fs::write(root.join(".git").join("config"), "[core]").unwrap();+ fs::create_dir(root.join("src")).unwrap();+ fs::write(root.join("src").join("main.rs"), "fn main() {}").unwrap();+ fs::write(root.join("src").join(".secret"), "nope").unwrap();++ let rels: Vec<String> = walk(root, false)+ .unwrap()+ .into_iter()+ .map(|e| e.rel)+ .collect();+ assert!(rels.contains(&"README.md".to_string()));+ assert!(rels.contains(&"src/main.rs".to_string()));+ // secrets and VCS internals must not be present+ assert!(!rels.iter().any(|r| r == ".env"), "indexed .env: {rels:?}");+ assert!(+ !rels.iter().any(|r| r.starts_with(".git")),+ "descended .git: {rels:?}"+ );+ assert!(+ !rels.iter().any(|r| r.ends_with(".secret")),+ "indexed a nested dotfile: {rels:?}"+ );+ }++ #[test]+ fn a_brain_over_a_dot_named_root_still_works() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path().join(".notes");+ fs::create_dir(&root).unwrap();+ fs::write(root.join("a.md"), "x").unwrap();+ let rels: Vec<String> = walk(&root, false)+ .unwrap()+ .into_iter()+ .map(|e| e.rel)+ .collect();+ assert_eq!(+ rels,+ vec!["a.md".to_string()],+ "root exemption failed: {rels:?}"+ );+ }+}