AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(corpus): do not index the root corpus.json manifest (#1622)
xerj-orgfc851b13540a
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex f050d3bca..dc9d5f8c4 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -120,6 +120,10 @@ pub struct IndexCfg {/// `--watch`'s quiet period. One editor save is several filesystem events,/// so a pass waits for the tree to go quiet for this long before running.pub debounce: Duration,+ /// Root-relative paths dropped after the walk. Empty for a normal+ /// `xerj autoindex <folder>`. `xerj corpus index` sets `corpus.json` so+ /// the hub manifest is not indexed as a document dataset.+ pub exclude_rels: Vec<String>,}#[derive(Debug, Clone)]@@ -1659,6 +1663,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {debounce: Duration::from_millis(debounce_ms.unwrap_or(crate::watch::DEFAULT_DEBOUNCE_MS),),+ exclude_rels: Vec::new(),})))}_ => Ok(Cmd::Help),diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rsindex f4366dde2..cdbd4d2df 100644--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs@@ -382,6 +382,7 @@ pub(super) fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {watch: false,label: None,debounce: std::time::Duration::from_millis(0),+ exclude_rels: Vec::new(),}}diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex f543a96f5..1d3554463 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -797,6 +797,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {watch: false,label: None,debounce: std::time::Duration::from_millis(0),+ exclude_rels: Vec::new(),}}diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 9ba5f464b..3fcd69ab9 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -971,6 +971,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {watch: false,label: None,debounce: std::time::Duration::from_millis(0),+ exclude_rels: Vec::new(),}}diff --git a/engine/crates/xerj-autoindex/src/label_http_tests.rs b/engine/crates/xerj-autoindex/src/label_http_tests.rsindex 6b2fe40fb..6a9fe920c 100644--- a/engine/crates/xerj-autoindex/src/label_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/label_http_tests.rs@@ -390,6 +390,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, label: &Path) -> IndexCfg {watch: false,debounce: std::time::Duration::from_millis(0),label: Some(label.to_owned()),+ exclude_rels: Vec::new(),}}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex b18a2348c..dd4d3c971 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -1688,6 +1688,64 @@ pub fn run_cli() -> i32 {}}+/// The hub manifest at the root of a corpus directory. Excluded only by+/// [`run_with_options`]; a folder named the same way is still indexed.+const CORPUS_MANIFEST_REL: &str = "corpus.json";++/// Drop walk entries whose root-relative path is in `exclude`. An empty list+/// returns the walk unchanged, so this is not a global skip of that filename.+fn without_excluded_rels(+ files: Vec<walk::FileEntry>,+ exclude: &[String],+) -> (Vec<walk::FileEntry>, usize) {+ if exclude.is_empty() {+ return (files, 0);+ }+ let before = files.len();+ let kept: Vec<walk::FileEntry> = files+ .into_iter()+ .filter(|file| !exclude.contains(&file.rel))+ .collect();+ let dropped = before - kept.len();+ (kept, dropped)+}++#[cfg(test)]+mod exclude_rels_tests {+ use super::*;++ fn entry(rel: &str) -> walk::FileEntry {+ walk::FileEntry {+ path: std::path::PathBuf::from(rel),+ rel: rel.to_string(),+ rel_id: rel.to_string(),+ is_symlink: false,+ size: 1,+ }+ }++ #[test]+ fn the_corpus_manifest_is_dropped_and_a_nested_copy_is_kept() {+ let files = vec![+ entry("corpus.json"),+ entry("src/lib.rs"),+ entry("vendor/corpus.json"),+ ];+ let (kept, dropped) = without_excluded_rels(files, &[CORPUS_MANIFEST_REL.to_string()]);+ assert_eq!(dropped, 1);+ let rels: Vec<&str> = kept.iter().map(|file| file.rel.as_str()).collect();+ assert_eq!(rels, ["src/lib.rs", "vendor/corpus.json"]);+ }++ #[test]+ fn an_empty_exclusion_list_keeps_a_root_corpus_json() {+ let files = vec![entry("corpus.json"), entry("README.md")];+ let (kept, dropped) = without_excluded_rels(files, &[]);+ assert_eq!(dropped, 0);+ assert_eq!(kept.len(), 2);+ }+}+/// In-process autoindex for the `xerj corpus` lifecycle (issue #977): builds/// the same argv the wrapper script passed and routes it through the same/// parse → run → error-route path as `run_cli`, so there is exactly ONE@@ -1726,7 +1784,14 @@ pub fn run_with_options(}};let res = match cmd {- Cmd::Index(cfg) => run_index(*cfg),+ Cmd::Index(cfg) => {+ // The corpus directory's own manifest is bookkeeping, not a+ // source file. A normal folder index leaves `exclude_rels` empty,+ // so a project that happens to contain `corpus.json` still indexes it.+ let mut cfg = *cfg;+ cfg.exclude_rels = vec![CORPUS_MANIFEST_REL.to_string()];+ run_index(cfg)+ }_ => return 2,};match res {@@ -3024,6 +3089,7 @@ mod phase_a_grouping_tests {watch: false,label: None,debounce: std::time::Duration::from_millis(0),+ exclude_rels: Vec::new(),}}@@ -6201,6 +6267,13 @@ fn run_index_report_inner(cfg.follow_symlinks_outside_root,cfg.ignore,)?;+ let (discovered_files, excluded) = without_excluded_rels(discovered_files, &cfg.exclude_rels);+ if excluded > 0 {+ pr.note(&format!(+ "autoindex: left out {excluded} path(s) ({}) — not part of this corpus index",+ cfg.exclude_rels.join(", ")+ ));+ }let discovered_bytes: u64 = discovered_files.iter().map(|f| f.size).sum();pr.note(&format!("autoindex: {} files ({} MB) under {}",diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rsindex 686bc9b0a..2cbbd12f3 100644--- a/engine/crates/xerj-server/src/brain.rs+++ b/engine/crates/xerj-server/src/brain.rs@@ -649,6 +649,7 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {watch: false,label: None,debounce: std::time::Duration::from_millis(0),+ exclude_rels: Vec::new(),}}