AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): scope map to explicit --prefix and omit blank wall_seconds
Christian M. Todie94337f96364f
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rsindex b94719282..8b0b58f4f 100644--- a/engine/crates/xerj-autoindex/src/catalog.rs+++ b/engine/crates/xerj-autoindex/src/catalog.rs@@ -492,6 +492,59 @@ pub fn build_sample_queries(pd: &PlanDataset, correlations: &[KeyCorr]) -> Vec<V// ─── map rendering ───────────────────────────────────────────────────────+/// True when `index_name` belongs to corpus `prefix`.+///+/// Autoindex names dataset indices `{prefix}-{slug}`. When `slug` is stored on+/// the catalog document, reconstructing `{prefix}-{slug}` disambiguates+/// hyphenated prefixes (`prefix = "xc"` + `slug = "tantivy-code"` vs+/// `prefix = "xc-tantivy"` + `slug = "code"`). When `slug` is absent on a+/// legacy document, fall back to a `{prefix}-` boundary check.+pub fn index_matches_prefix(index_name: &str, slug: Option<&str>, prefix: &str) -> bool {+ match slug.filter(|s| !s.is_empty()) {+ Some(slug) => index_name+ .strip_prefix(prefix)+ .and_then(|rest| rest.strip_prefix('-'))+ .is_some_and(|rest| rest == slug),+ None => index_name+ .strip_prefix(prefix)+ .is_some_and(|rest| rest.starts_with('-')),+ }+}++/// True when a `doc_kind: "dataset"` catalog document belongs to `prefix`.+pub fn dataset_matches_prefix(doc: &Value, prefix: &str) -> bool {+ let slug = doc.get("slug").and_then(Value::as_str);+ doc.get("index_name")+ .and_then(Value::as_str)+ .is_some_and(|idx| index_matches_prefix(idx, slug, prefix))+}++/// True when both endpoints of a `doc_kind: "correlation"` catalog document+/// belong to `prefix`.+pub fn correlation_matches_prefix(doc: &Value, prefix: &str) -> bool {+ let a_ok = doc+ .get("a_index")+ .and_then(Value::as_str)+ .is_some_and(|idx| {+ index_matches_prefix(idx, doc.get("a_dataset").and_then(Value::as_str), prefix)+ });+ let b_ok = doc+ .get("b_index")+ .and_then(Value::as_str)+ .is_some_and(|idx| {+ index_matches_prefix(idx, doc.get("b_dataset").and_then(Value::as_str), prefix)+ });+ a_ok && b_ok+}++/// True when a `doc_kind: "file"` catalog document belongs to `prefix`.+pub fn file_matches_prefix(doc: &Value, prefix: &str) -> bool {+ doc.get("prefix")+ .or_else(|| doc.get(CORPUS_SCOPE_FIELD))+ .and_then(Value::as_str)+ == Some(prefix)+}+pub fn render_map(run: Option<&Value>,datasets: &[Value],@@ -504,8 +557,14 @@ pub fn render_map(s.push_str("# Data map (xerj autoindex)\n\n");if let Some(r) = run {let g = |k: &str| r.get(k).map(pretty_val).unwrap_or_default();+ let wall = g("wall_seconds");+ let wall_clause = if wall.is_empty() {+ String::new()+ } else {+ format!(", wall {wall}s")+ };s.push_str(&format!(- "run `{}` — root `{}` — {} paths ({} unique content, {} duplicate aliases), {} records indexed, {} junk records, wall {}s\n\n",+ "run `{}` — root `{}` — {} paths ({} unique content, {} duplicate aliases), {} records indexed, {} junk records{}\n\n",g("run_id"),g("root"),g("files_total"),@@ -513,7 +572,7 @@ pub fn render_map(g("duplicate_files"),g("records_total"),g("junk_records_total"),- g("wall_seconds"),+ wall_clause,));}// #929: a corpus that lacks a dataset says so at the top of its own map,@@ -799,6 +858,103 @@ mod duplicate_tests {assert!(rendered.contains("`copy.pdf` → `report.pdf`"));assert!(!rendered.contains("## Junk / skipped"));}++ #[test]+ fn render_map_omits_blank_wall_seconds_on_generation_catalog_run_docs() {+ // `generation_catalog.rs` omits `wall_seconds` so the run document is a+ // deterministic projection of the generation; `render_map` must not+ // emit a dangling `, wall s` when the field is absent.+ let gen_run = json!({+ "doc_kind": "run",+ "run_id": "gen-1",+ "root": "/corpus",+ "prefix": "xc-tantivy",+ "files_total": 10,+ "unique_content_files": 9,+ "duplicate_files": 1,+ "records_total": 42,+ "junk_records_total": 0+ });+ let rendered = render_map(Some(&gen_run), &[], &[], &[], &[], 0);+ assert!(+ !rendered.contains("wall s"),+ "must not render blank `wall s` when wall_seconds is absent: {rendered}"+ );+ assert!(+ rendered.contains("42 records indexed, 0 junk records\n\n"),+ "run summary line should end cleanly at junk records: {rendered}"+ );++ // When `wall_seconds` IS present (legacy graph path), keep `, wall Xs`.+ let legacy_run = json!({+ "doc_kind": "run",+ "run_id": "run-legacy",+ "root": "/corpus",+ "files_total": 10,+ "unique_content_files": 9,+ "duplicate_files": 1,+ "records_total": 42,+ "junk_records_total": 0,+ "wall_seconds": 1.4+ });+ let rendered_legacy = render_map(Some(&legacy_run), &[], &[], &[], &[], 0);+ assert!(+ rendered_legacy.contains("0 junk records, wall 1.4s\n\n"),+ "legacy wall_seconds must still render: {rendered_legacy}"+ );+ }++ #[test]+ fn prefix_scoping_disambiguates_hyphenated_prefixes_across_catalog_doc_kinds() {+ let tantivy_ds = json!({+ "doc_kind": "dataset",+ "slug": "code",+ "index_name": "xc-tantivy-code"+ });+ assert!(dataset_matches_prefix(&tantivy_ds, "xc-tantivy"));+ assert!(!dataset_matches_prefix(&tantivy_ds, "xc"));+ assert!(!dataset_matches_prefix(&tantivy_ds, "xc-tantivy-extra"));++ let xc_hyphen_slug_ds = json!({+ "doc_kind": "dataset",+ "slug": "tantivy-code",+ "index_name": "xc-tantivy-code"+ });+ assert!(dataset_matches_prefix(&xc_hyphen_slug_ds, "xc"));+ assert!(!dataset_matches_prefix(&xc_hyphen_slug_ds, "xc-tantivy"));++ // Legacy dataset doc without `slug` falls back to `{prefix}-` boundary.+ let legacy_ds = json!({+ "doc_kind": "dataset",+ "index_name": "ax-logs"+ });+ assert!(dataset_matches_prefix(&legacy_ds, "ax"));+ assert!(!dataset_matches_prefix(&legacy_ds, "ax2"));++ let corr = json!({+ "doc_kind": "correlation",+ "a_index": "xc-tantivy-code",+ "a_dataset": "code",+ "b_index": "xc-tantivy-symbols",+ "b_dataset": "symbols"+ });+ assert!(correlation_matches_prefix(&corr, "xc-tantivy"));+ assert!(!correlation_matches_prefix(&corr, "xc"));++ let file_with_prefix = json!({+ "doc_kind": "file",+ "prefix": "xc-tantivy",+ "path": "a.rs"+ });+ let file_with_scope_only = json!({+ "doc_kind": "file",+ CORPUS_SCOPE_FIELD: "xc-tantivy",+ "path": "b.rs"+ });+ assert!(file_matches_prefix(&file_with_prefix, "xc-tantivy"));+ assert!(!file_matches_prefix(&file_with_prefix, "xc"));+ assert!(file_matches_prefix(&file_with_scope_only, "xc-tantivy"));+ }}#[cfg(test)]diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex d95c401e9..e74bd6905 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -126,7 +126,9 @@ pub struct IndexCfg {pub struct MapCfg {pub url: String,pub api_key: Option<String>,- pub prefix: String,+ /// Explicit `--prefix` filter (`None` when the user did not pass `--prefix`,+ /// preserving the #971 multi-corpus view on a bare `autoindex map`).+ pub prefix: Option<String>,pub json: bool,pub dataset: Option<String>,/// A startup note when `XERJ_URL` was set but ignored (endpoint comes from@@ -704,6 +706,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {let mut snapshot_max_bytes = 64u64 << 30;let mut bulk_timeout_explicit = false;let mut prefix = "ax".to_string();+ let mut prefix_explicit = false;let mut state_dir: Option<PathBuf> = None;let mut endpoint_url: Option<String> = None;let mut endpoint_explicit = false;@@ -832,7 +835,10 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {"--in-flight" => {let _ = it.next(); // reserved (bulks are worker-synchronous in v1)}- "--prefix" => prefix = it.next().ok_or("--prefix needs a value")?,+ "--prefix" => {+ prefix = it.next().ok_or("--prefix needs a value")?;+ prefix_explicit = true;+ }"--state-dir" => state_dir = it.next().map(PathBuf::from),"--endpoint-url" => {let value = it.next().ok_or(@@ -1502,7 +1508,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {(Some("map"), _) => Ok(Cmd::Map(MapCfg {url,api_key,- prefix,+ prefix: prefix_explicit.then_some(prefix),json,dataset,xerj_url_note,@@ -2948,4 +2954,24 @@ mod tests {"a valueless --dataset must be refused: {text}");}++ #[test]+ fn map_records_explicit_prefix_and_leaves_unscoped_map_none() {+ match parse(["map"].iter().map(|s| s.to_string()).collect()).expect("bare map must parse") {+ Cmd::Map(cfg) => assert_eq!(cfg.prefix, None),+ other => panic!("expected a map config, got {other:?}"),+ }++ match parse(+ ["map", "--prefix", "xc-tantivy"]+ .iter()+ .map(|s| s.to_string())+ .collect(),+ )+ .expect("map --prefix must parse")+ {+ Cmd::Map(cfg) => assert_eq!(cfg.prefix.as_deref(), Some("xc-tantivy")),+ other => panic!("expected a map config, got {other:?}"),+ }+ }}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 5b62320ba..7ada26d4c 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -9358,16 +9358,35 @@ fn run_map(cfg: MapCfg) -> Result<i32> {{"term": {"slug": slug}}]}});}- let datasets = fetch(ds_query, 500, Some(json!([{"record_count": "desc"}])))?;+ let mut datasets = fetch(ds_query, 500, Some(json!([{"record_count": "desc"}])))?;+ if let Some(prefix) = cfg.prefix.as_deref() {+ datasets.retain(|d| catalog::dataset_matches_prefix(d, prefix));+ }if datasets.is_empty() {- eprintln!(- "no autoindex catalog found at {} (index {}) — run `xerj autoindex <folder>` first",- cfg.url,- catalog::CATALOG_INDEX- );+ if let Some(prefix) = cfg.prefix.as_deref() {+ eprintln!(+ "no autoindex catalog found for --prefix {prefix} at {} (index {}) — run `xerj autoindex <folder> --prefix {prefix}` first",+ cfg.url,+ catalog::CATALOG_INDEX+ );+ } else {+ eprintln!(+ "no autoindex catalog found at {} (index {}) — run `xerj autoindex <folder>` first",+ cfg.url,+ catalog::CATALOG_INDEX+ );+ }return Ok(1);}- let mut runs = fetch(json!({"term": {"doc_kind": "run"}}), 50, None)?;+ let mut runs = fetch(json!({"term": {"doc_kind": "run"}}), 500, None)?;+ if let Some(prefix) = cfg.prefix.as_deref() {+ runs.retain(|r| {+ r.get("prefix")+ .or_else(|| r.get(catalog::CORPUS_SCOPE_FIELD))+ .and_then(Value::as_str)+ == Some(prefix)+ });+ }runs.sort_by_key(|r| {std::cmp::Reverse(r.get("started")@@ -9378,6 +9397,9 @@ fn run_map(cfg: MapCfg) -> Result<i32> {});let correlations = {let mut all = fetch(json!({"term": {"doc_kind": "correlation"}}), 200, None)?;+ if let Some(prefix) = cfg.prefix.as_deref() {+ all.retain(|c| catalog::correlation_matches_prefix(c, prefix));+ }// stale-correlation hygiene: catalog docs upsert by deterministic id,// so older runs' correlations linger — show only the latest run that// produced each corr_kind.@@ -9411,20 +9433,25 @@ fn run_map(cfg: MapCfg) -> Result<i32> {// since #737; on a pre-rc.68 catalog it was inferred, where a term// matches only while the value has no separator — the same legacy caveat// the scoped sweeps in `catalog.rs` document.)- let prefixes: Vec<String> = runs- .iter()- .filter_map(|run| run.get("prefix"))- .filter_map(Value::as_str)- .collect::<BTreeSet<_>>()- .into_iter()- .map(String::from)- .collect::<Vec<_>>();+ let prefixes: Vec<String> = match cfg.prefix.as_deref() {+ Some(prefix) => vec![prefix.to_string()],+ None => runs+ .iter()+ .filter_map(|run| run.get("prefix"))+ .filter_map(Value::as_str)+ .collect::<BTreeSet<_>>()+ .into_iter()+ .map(String::from)+ .collect::<Vec<_>>(),+ };let prefix_filter = (!prefixes.is_empty()).then(|| {+ let mut should = Vec::with_capacity(prefixes.len() * 2);+ for prefix in &prefixes {+ should.push(json!({"term": {"prefix": prefix}}));+ should.push(json!({"term": {catalog::CORPUS_SCOPE_FIELD: prefix}}));+ }json!({"bool": {- "should": prefixes- .iter()- .map(|prefix| json!({"term": {"prefix": prefix}}))- .collect::<Vec<_>>(),+ "should": should,"minimum_should_match": 1}})});@@ -9432,7 +9459,7 @@ fn run_map(cfg: MapCfg) -> Result<i32> {if let Some(filter) = prefix_filter.clone() {junk_must.push(filter);}- let junk_files = fetch(+ let mut junk_files = fetch(json!({"bool": {"must": junk_must,"must_not": [{"term": {"status": "indexed"}},@@ -9448,7 +9475,11 @@ fn run_map(cfg: MapCfg) -> Result<i32> {if let Some(filter) = prefix_filter {duplicate_must.push(filter);}- let duplicate_files = fetch(json!({"bool": {"must": duplicate_must}}), 500, None)?;+ let mut duplicate_files = fetch(json!({"bool": {"must": duplicate_must}}), 500, None)?;+ if let Some(prefix) = cfg.prefix.as_deref() {+ junk_files.retain(|f| catalog::file_matches_prefix(f, prefix));+ duplicate_files.retain(|f| catalog::file_matches_prefix(f, prefix));+ }if cfg.json {println!("{}",