Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): bound docx decompression + never index dotfiles/secrets

xerj-org4b9e27f2f0a6
diff --git a/engine/crates/xerj-autoindex/src/extract/docx.rs b/engine/crates/xerj-autoindex/src/extract/docx.rs
index 2a6f68616..cef0b370d 100644
--- a/engine/crates/xerj-autoindex/src/extract/docx.rs
+++ b/engine/crates/xerj-autoindex/src/extract/docx.rs
@@ -20,9 +20,20 @@ pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {
return Ok(stats);
}
};
- let mut reader = Reader::from_reader(BufReader::new(entry));
+ // SECURITY: bound the DECOMPRESSED read. `word/document.xml` inflates from
+ // the zip at whatever ratio the author chose — a ~400 KB docx can expand to
+ // hundreds of MB, and a single 400 MB `<w:t>` text run makes quick-xml
+ // allocate that whole run for one event (measured 1.68 GB RSS from an 815 KB
+ // file). Reading through a `Take` caps peak memory no matter the ratio; a
+ // real document's document.xml is far below this. Past the cap the reader
+ // hits EOF and the loop ends (a truncated tag is handled as junk below).
+ const MAX_DECOMPRESSED_BYTES: u64 = 72 << 20;
+ let capped = std::io::Read::take(entry, MAX_DECOMPRESSED_BYTES);
+ let mut reader = Reader::from_reader(BufReader::new(capped));
reader.config_mut().trim_text(false);
+ // Extraction body cap (also bounds any single paragraph — see the Text arm).
+ const MAX_BODY_BYTES: usize = 64 << 20;
let mut body = String::new();
let mut headings: Vec<String> = Vec::new();
let mut para = String::new();
@@ -49,7 +60,16 @@ pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {
}
}
Ok(Event::Text(t)) => {
- para.push_str(&t.unescape().unwrap_or_default());
+ // SECURITY: `para` only flushes into `body` (and gets length-
+ // checked) at `</w:p>`. A crafted docx with a single never-closed
+ // `<w:p>` — an 815 KB file whose document.xml inflates to
+ // hundreds of MB inside one paragraph — otherwise grows `para`
+ // without bound: measured 1.68 GB RSS from that 815 KB input.
+ // Stop appending once one paragraph reaches the body cap; the
+ // per-paragraph text a real document holds is far below it.
+ if para.len() < MAX_BODY_BYTES {
+ para.push_str(&t.unescape().unwrap_or_default());
+ }
}
Ok(Event::End(e)) => {
if e.name().as_ref() == b"w:p" {
@@ -73,7 +93,7 @@ pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {
}
}
buf.clear();
- if body.len() > 64 << 20 {
+ if body.len() > MAX_BODY_BYTES {
break;
}
}
diff --git a/engine/crates/xerj-autoindex/src/walk.rs b/engine/crates/xerj-autoindex/src/walk.rs
index 02c2a7bad..27278f2c5 100644
--- a/engine/crates/xerj-autoindex/src/walk.rs
+++ b/engine/crates/xerj-autoindex/src/walk.rs
@@ -25,7 +25,20 @@ pub fn walk(root: &Path, follow_symlinks: bool) -> Result<Vec<FileEntry>> {
anyhow::bail!("{} is not a directory", root_canon.display());
}
let mut out = Vec::new();
- for entry in walkdir::WalkDir::new(&root_canon).follow_links(follow_symlinks) {
+ // SECURITY / hygiene: never index hidden files or descend into hidden
+ // directories. Without this the walker happily indexed `.env` (secrets,
+ // API tokens), `.git`, `.ssh`, `.aws`, and other dotfiles into a queryable
+ // brain with no per-brain authorization — a real exposure for the
+ // "point it at my project folder" use case. `filter_entry` prunes a hidden
+ // directory before descending, so `.git/` is skipped whole. The root itself
+ // (depth 0) is exempt so a brain over a dot-named folder still works.
+ let walker = walkdir::WalkDir::new(&root_canon)
+ .follow_links(follow_symlinks)
+ .into_iter()
+ .filter_entry(|e| {
+ e.depth() == 0 || !e.file_name().to_str().is_some_and(|n| n.starts_with('.'))
+ });
+ for entry in walker {
let entry = match entry {
Ok(e) => e,
Err(e) => {
@@ -89,3 +102,58 @@ fn stable_path_id(path: &Path) -> String {
fn stable_path_id(path: &Path) -> String {
format!("other:{}", path.to_string_lossy())
}
+
+#[cfg(test)]
+mod hidden_skip_tests {
+ use super::walk;
+ use std::fs;
+
+ #[test]
+ fn hidden_files_and_dirs_are_never_indexed() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ fs::write(root.join("README.md"), "hello").unwrap();
+ fs::write(root.join(".env"), "SECRET=pat-abc123").unwrap();
+ fs::create_dir(root.join(".git")).unwrap();
+ fs::write(root.join(".git").join("config"), "[core]").unwrap();
+ fs::create_dir(root.join("src")).unwrap();
+ fs::write(root.join("src").join("main.rs"), "fn main() {}").unwrap();
+ fs::write(root.join("src").join(".secret"), "nope").unwrap();
+
+ let rels: Vec<String> = walk(root, false)
+ .unwrap()
+ .into_iter()
+ .map(|e| e.rel)
+ .collect();
+ assert!(rels.contains(&"README.md".to_string()));
+ assert!(rels.contains(&"src/main.rs".to_string()));
+ // secrets and VCS internals must not be present
+ assert!(!rels.iter().any(|r| r == ".env"), "indexed .env: {rels:?}");
+ assert!(
+ !rels.iter().any(|r| r.starts_with(".git")),
+ "descended .git: {rels:?}"
+ );
+ assert!(
+ !rels.iter().any(|r| r.ends_with(".secret")),
+ "indexed a nested dotfile: {rels:?}"
+ );
+ }
+
+ #[test]
+ fn a_brain_over_a_dot_named_root_still_works() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path().join(".notes");
+ fs::create_dir(&root).unwrap();
+ fs::write(root.join("a.md"), "x").unwrap();
+ let rels: Vec<String> = walk(&root, false)
+ .unwrap()
+ .into_iter()
+ .map(|e| e.rel)
+ .collect();
+ assert_eq!(
+ rels,
+ vec!["a.md".to_string()],
+ "root exemption failed: {rels:?}"
+ );
+ }
+}