Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): prune third_party and bower_components by default (#1697)

xerj-org1cfdefd413ed
diff --git a/engine/crates/xerj-autoindex/src/ignore_rules.rs b/engine/crates/xerj-autoindex/src/ignore_rules.rs
index 0da29a7f2..e2fd89fd0 100644
--- a/engine/crates/xerj-autoindex/src/ignore_rules.rs
+++ b/engine/crates/xerj-autoindex/src/ignore_rules.rs
@@ -79,16 +79,21 @@ pub const GITIGNORE: &str = ".gitignore";
pub const XERJIGNORE: &str = ".xerjignore";
/// Build output that is junk in practice but is *not* reliably gitignored —
-/// `vendor/` is committed by Go projects, `dist/` and `build/` are gitignored
-/// in one repo and checked in the next, and a folder full of downloaded
-/// packages is very often not a git repository at all.
+/// `vendor/` is committed by Go projects, `third_party/` and `bower_components/`
+/// are committed dependency trees, `dist/` and `build/` are gitignored in one
+/// repo and checked in the next, and a folder full of downloaded packages is
+/// very often not a git repository at all.
///
/// These are defaults, not law: `--no-default-ignores` turns them off, any
-/// ignore file in the tree outranks them, and `--dry-run` names them as the
-/// reason a directory disappeared.
+/// ignore file in the tree outranks them (`!third_party/` keeps a tree the
+/// operator wants searched), and `--dry-run` names them as the reason a
+/// directory disappeared.
pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
"node_modules/",
+ "bower_components/",
"vendor/",
+ "third_party/",
+ "thirdparty/",
"target/",
"dist/",
"build/",
@@ -1037,6 +1042,49 @@ mod tests {
assert_eq!(kept, vec!["vendor/lib/lib.go"], "{kept:?}");
}
+ /// `third_party/` and `bower_components/` are committed dependency trees.
+ /// They were only delayed to the end of the queue, so a monorepo still
+ /// walked and indexed them. A `!` line puts one back, same as `vendor/`.
+ #[test]
+ fn defaults_prune_committed_dependency_trees() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let root = dir.path();
+ write(root, "src/lib.rs", "fn main() {}");
+ write(root, "third_party/llvm/Support.cpp", "x");
+ write(root, "thirdparty/dep/dep.c", "x");
+ write(root, "bower_components/jquery/jquery.js", "x");
+ write(root, "kept/third_party_notes.md", "not a directory name");
+
+ let (kept, report) = survivors(root, IgnoreOptions::default());
+ assert_eq!(
+ kept,
+ vec!["kept/third_party_notes.md", "src/lib.rs"],
+ "{kept:?}"
+ );
+ assert!(
+ report.by_rule.contains_key("<built-in>:third_party/"),
+ "{report:?}"
+ );
+ assert!(
+ report.by_rule.contains_key("<built-in>:bower_components/"),
+ "{report:?}"
+ );
+ assert_eq!(report.dirs_pruned, 3, "{report:?}");
+
+ write(root, ".xerjignore", "!third_party/\n");
+ let (reincluded, _) = survivors(root, IgnoreOptions::default());
+ assert!(
+ reincluded
+ .iter()
+ .any(|p| p == "third_party/llvm/Support.cpp"),
+ "a negation must put the tree back: {reincluded:?}"
+ );
+ assert!(
+ !reincluded.iter().any(|p| p.contains("bower_components")),
+ "the other trees stay pruned: {reincluded:?}"
+ );
+ }
+
/// Images, fonts and native binaries are junk after the sniff. On a repo
/// that contains thousands of them the walk opened every one. They are
/// now dropped with the built-in rules. A PDF and a gzipped log are not