AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): skip binary assets during the walk (#1692)
xerj-orgb1adb4f50bd3
diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rsindex 2397cf272..0da29a7f2 100644--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs@@ -94,6 +94,27 @@ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &["build/",".venv/","__pycache__/",+ // Media and native binaries. Sniffing classifies these as binary and+ // records each one as junk, so a repo of images and shared libraries+ // spent the walk opening files that can never become a document. A+ // gzipped log (`.gz`) and a PDF stay: those families are extracted.+ // `!*.png` in an ignore file puts one extension back.+ "*.png",+ "*.jpg",+ "*.jpeg",+ "*.gif",+ "*.webp",+ "*.ico",+ "*.bmp",+ "*.woff",+ "*.woff2",+ "*.ttf",+ "*.otf",+ "*.wasm",+ "*.so",+ "*.dll",+ "*.dylib",+ "*.exe",];/// Rule label used for the pre-existing dotfile skip. It is not one of the@@ -1016,6 +1037,67 @@ mod tests {assert_eq!(kept, vec!["vendor/lib/lib.go"], "{kept:?}");}+ /// Images, fonts and native binaries are junk after the sniff. On a repo+ /// that contains thousands of them the walk opened every one. They are+ /// now dropped with the built-in rules. A PDF and a gzipped log are not+ /// on that list: both are extracted.+ #[test]+ fn defaults_skip_binary_assets_and_keep_extractable_files() {+ let dir = tempfile::TempDir::new().unwrap();+ let root = dir.path();+ write(root, "src/lib.rs", "fn main() {}");+ write(root, "README.md", "hi");+ write(root, "logs/app.log.gz", "not really gzip");+ write(root, "docs/manual.pdf", "%PDF-");+ write(root, "assets/logo.png", "png");+ write(root, "assets/photo.jpeg", "jpg");+ write(root, "web/app.wasm", "wasm");+ write(root, "native/libfoo.so", "so");+ write(root, "native/helper.exe", "exe");++ let (kept, report) = survivors(root, IgnoreOptions::default());+ assert_eq!(+ kept,+ vec![+ "README.md",+ "docs/manual.pdf",+ "logs/app.log.gz",+ "src/lib.rs",+ ],+ "{kept:?}"+ );+ assert!(+ report.by_rule.contains_key("<built-in>:*.png"),+ "{report:?}"+ );+ assert!(report.by_rule.contains_key("<built-in>:*.so"), "{report:?}");+ assert_eq!(report.files_skipped, 5, "{report:?}");++ let (all, _) = survivors(+ root,+ IgnoreOptions {+ defaults: false,+ ..IgnoreOptions::default()+ },+ );+ assert_eq!(+ all.len(),+ 9,+ "--no-default-ignores keeps the assets: {all:?}"+ );++ write(root, ".xerjignore", "!*.png\n");+ let (reincluded, _) = survivors(root, IgnoreOptions::default());+ assert!(+ reincluded.iter().any(|p| p == "assets/logo.png"),+ "a negation must put the png back: {reincluded:?}"+ );+ assert!(+ !reincluded.iter().any(|p| p.ends_with(".so")),+ "other binary extensions stay skipped: {reincluded:?}"+ );+ }+#[test]fn no_ignore_disables_every_rule_but_not_the_dotfile_skip() {let dir = tempfile::TempDir::new().unwrap();