Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(corpus): do not index the root corpus.json manifest (#1622)

xerj-orgfc851b13540a
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index f050d3bca..dc9d5f8c4 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -120,6 +120,10 @@ pub struct IndexCfg {
/// `--watch`'s quiet period. One editor save is several filesystem events,
/// so a pass waits for the tree to go quiet for this long before running.
pub debounce: Duration,
+ /// Root-relative paths dropped after the walk. Empty for a normal
+ /// `xerj autoindex <folder>`. `xerj corpus index` sets `corpus.json` so
+ /// the hub manifest is not indexed as a document dataset.
+ pub exclude_rels: Vec<String>,
}
#[derive(Debug, Clone)]
@@ -1659,6 +1663,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
debounce: Duration::from_millis(
debounce_ms.unwrap_or(crate::watch::DEFAULT_DEBOUNCE_MS),
),
+ exclude_rels: Vec::new(),
})))
}
_ => Ok(Cmd::Help),
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index f4366dde2..cdbd4d2df 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -382,6 +382,7 @@ pub(super) fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
watch: false,
label: None,
debounce: std::time::Duration::from_millis(0),
+ exclude_rels: Vec::new(),
}
}
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index f543a96f5..1d3554463 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -797,6 +797,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
watch: false,
label: None,
debounce: std::time::Duration::from_millis(0),
+ exclude_rels: Vec::new(),
}
}
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 9ba5f464b..3fcd69ab9 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -971,6 +971,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {
watch: false,
label: None,
debounce: std::time::Duration::from_millis(0),
+ exclude_rels: Vec::new(),
}
}
diff --git a/engine/crates/xerj-autoindex/src/label_http_tests.rs b/engine/crates/xerj-autoindex/src/label_http_tests.rs
index 6b2fe40fb..6a9fe920c 100644
--- a/engine/crates/xerj-autoindex/src/label_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/label_http_tests.rs
@@ -390,6 +390,7 @@ fn cfg(root: &Path, state_dir: &Path, url: &str, label: &Path) -> IndexCfg {
watch: false,
debounce: std::time::Duration::from_millis(0),
label: Some(label.to_owned()),
+ exclude_rels: Vec::new(),
}
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index b18a2348c..dd4d3c971 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -1688,6 +1688,64 @@ pub fn run_cli() -> i32 {
}
}
+/// The hub manifest at the root of a corpus directory. Excluded only by
+/// [`run_with_options`]; a folder named the same way is still indexed.
+const CORPUS_MANIFEST_REL: &str = "corpus.json";
+
+/// Drop walk entries whose root-relative path is in `exclude`. An empty list
+/// returns the walk unchanged, so this is not a global skip of that filename.
+fn without_excluded_rels(
+ files: Vec<walk::FileEntry>,
+ exclude: &[String],
+) -> (Vec<walk::FileEntry>, usize) {
+ if exclude.is_empty() {
+ return (files, 0);
+ }
+ let before = files.len();
+ let kept: Vec<walk::FileEntry> = files
+ .into_iter()
+ .filter(|file| !exclude.contains(&file.rel))
+ .collect();
+ let dropped = before - kept.len();
+ (kept, dropped)
+}
+
+#[cfg(test)]
+mod exclude_rels_tests {
+ use super::*;
+
+ fn entry(rel: &str) -> walk::FileEntry {
+ walk::FileEntry {
+ path: std::path::PathBuf::from(rel),
+ rel: rel.to_string(),
+ rel_id: rel.to_string(),
+ is_symlink: false,
+ size: 1,
+ }
+ }
+
+ #[test]
+ fn the_corpus_manifest_is_dropped_and_a_nested_copy_is_kept() {
+ let files = vec![
+ entry("corpus.json"),
+ entry("src/lib.rs"),
+ entry("vendor/corpus.json"),
+ ];
+ let (kept, dropped) = without_excluded_rels(files, &[CORPUS_MANIFEST_REL.to_string()]);
+ assert_eq!(dropped, 1);
+ let rels: Vec<&str> = kept.iter().map(|file| file.rel.as_str()).collect();
+ assert_eq!(rels, ["src/lib.rs", "vendor/corpus.json"]);
+ }
+
+ #[test]
+ fn an_empty_exclusion_list_keeps_a_root_corpus_json() {
+ let files = vec![entry("corpus.json"), entry("README.md")];
+ let (kept, dropped) = without_excluded_rels(files, &[]);
+ assert_eq!(dropped, 0);
+ assert_eq!(kept.len(), 2);
+ }
+}
+
/// In-process autoindex for the `xerj corpus` lifecycle (issue #977): builds
/// the same argv the wrapper script passed and routes it through the same
/// parse → run → error-route path as `run_cli`, so there is exactly ONE
@@ -1726,7 +1784,14 @@ pub fn run_with_options(
}
};
let res = match cmd {
- Cmd::Index(cfg) => run_index(*cfg),
+ Cmd::Index(cfg) => {
+ // The corpus directory's own manifest is bookkeeping, not a
+ // source file. A normal folder index leaves `exclude_rels` empty,
+ // so a project that happens to contain `corpus.json` still indexes it.
+ let mut cfg = *cfg;
+ cfg.exclude_rels = vec![CORPUS_MANIFEST_REL.to_string()];
+ run_index(cfg)
+ }
_ => return 2,
};
match res {
@@ -3024,6 +3089,7 @@ mod phase_a_grouping_tests {
watch: false,
label: None,
debounce: std::time::Duration::from_millis(0),
+ exclude_rels: Vec::new(),
}
}
@@ -6201,6 +6267,13 @@ fn run_index_report_inner(
cfg.follow_symlinks_outside_root,
cfg.ignore,
)?;
+ let (discovered_files, excluded) = without_excluded_rels(discovered_files, &cfg.exclude_rels);
+ if excluded > 0 {
+ pr.note(&format!(
+ "autoindex: left out {excluded} path(s) ({}) — not part of this corpus index",
+ cfg.exclude_rels.join(", ")
+ ));
+ }
let discovered_bytes: u64 = discovered_files.iter().map(|f| f.size).sum();
pr.note(&format!(
"autoindex: {} files ({} MB) under {}",
diff --git a/engine/crates/xerj-server/src/brain.rs b/engine/crates/xerj-server/src/brain.rs
index 686bc9b0a..2cbbd12f3 100644
--- a/engine/crates/xerj-server/src/brain.rs
+++ b/engine/crates/xerj-server/src/brain.rs
@@ -649,6 +649,7 @@ fn index_cfg(cfg: &BrainCfg, brain: &str, api_key: Option<String>) -> IndexCfg {
watch: false,
label: None,
debounce: std::time::Duration::from_millis(0),
+ exclude_rels: Vec::new(),
}
}