Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): scope map to explicit --prefix and omit blank wall_seconds

Christian M. Todie94337f96364f
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index b94719282..8b0b58f4f 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -492,6 +492,59 @@ pub fn build_sample_queries(pd: &PlanDataset, correlations: &[KeyCorr]) -> Vec<V
// ─── map rendering ───────────────────────────────────────────────────────
+/// True when `index_name` belongs to corpus `prefix`.
+///
+/// Autoindex names dataset indices `{prefix}-{slug}`. When `slug` is stored on
+/// the catalog document, reconstructing `{prefix}-{slug}` disambiguates
+/// hyphenated prefixes (`prefix = "xc"` + `slug = "tantivy-code"` vs
+/// `prefix = "xc-tantivy"` + `slug = "code"`). When `slug` is absent on a
+/// legacy document, fall back to a `{prefix}-` boundary check.
+pub fn index_matches_prefix(index_name: &str, slug: Option<&str>, prefix: &str) -> bool {
+ match slug.filter(|s| !s.is_empty()) {
+ Some(slug) => index_name
+ .strip_prefix(prefix)
+ .and_then(|rest| rest.strip_prefix('-'))
+ .is_some_and(|rest| rest == slug),
+ None => index_name
+ .strip_prefix(prefix)
+ .is_some_and(|rest| rest.starts_with('-')),
+ }
+}
+
+/// True when a `doc_kind: "dataset"` catalog document belongs to `prefix`.
+pub fn dataset_matches_prefix(doc: &Value, prefix: &str) -> bool {
+ let slug = doc.get("slug").and_then(Value::as_str);
+ doc.get("index_name")
+ .and_then(Value::as_str)
+ .is_some_and(|idx| index_matches_prefix(idx, slug, prefix))
+}
+
+/// True when both endpoints of a `doc_kind: "correlation"` catalog document
+/// belong to `prefix`.
+pub fn correlation_matches_prefix(doc: &Value, prefix: &str) -> bool {
+ let a_ok = doc
+ .get("a_index")
+ .and_then(Value::as_str)
+ .is_some_and(|idx| {
+ index_matches_prefix(idx, doc.get("a_dataset").and_then(Value::as_str), prefix)
+ });
+ let b_ok = doc
+ .get("b_index")
+ .and_then(Value::as_str)
+ .is_some_and(|idx| {
+ index_matches_prefix(idx, doc.get("b_dataset").and_then(Value::as_str), prefix)
+ });
+ a_ok && b_ok
+}
+
+/// True when a `doc_kind: "file"` catalog document belongs to `prefix`.
+pub fn file_matches_prefix(doc: &Value, prefix: &str) -> bool {
+ doc.get("prefix")
+ .or_else(|| doc.get(CORPUS_SCOPE_FIELD))
+ .and_then(Value::as_str)
+ == Some(prefix)
+}
+
pub fn render_map(
run: Option<&Value>,
datasets: &[Value],
@@ -504,8 +557,14 @@ pub fn render_map(
s.push_str("# Data map (xerj autoindex)\n\n");
if let Some(r) = run {
let g = |k: &str| r.get(k).map(pretty_val).unwrap_or_default();
+ let wall = g("wall_seconds");
+ let wall_clause = if wall.is_empty() {
+ String::new()
+ } else {
+ format!(", wall {wall}s")
+ };
s.push_str(&format!(
- "run `{}` — root `{}` — {} paths ({} unique content, {} duplicate aliases), {} records indexed, {} junk records, wall {}s\n\n",
+ "run `{}` — root `{}` — {} paths ({} unique content, {} duplicate aliases), {} records indexed, {} junk records{}\n\n",
g("run_id"),
g("root"),
g("files_total"),
@@ -513,7 +572,7 @@ pub fn render_map(
g("duplicate_files"),
g("records_total"),
g("junk_records_total"),
- g("wall_seconds"),
+ wall_clause,
));
}
// #929: a corpus that lacks a dataset says so at the top of its own map,
@@ -799,6 +858,103 @@ mod duplicate_tests {
assert!(rendered.contains("`copy.pdf` → `report.pdf`"));
assert!(!rendered.contains("## Junk / skipped"));
}
+
+ #[test]
+ fn render_map_omits_blank_wall_seconds_on_generation_catalog_run_docs() {
+ // `generation_catalog.rs` omits `wall_seconds` so the run document is a
+ // deterministic projection of the generation; `render_map` must not
+ // emit a dangling `, wall s` when the field is absent.
+ let gen_run = json!({
+ "doc_kind": "run",
+ "run_id": "gen-1",
+ "root": "/corpus",
+ "prefix": "xc-tantivy",
+ "files_total": 10,
+ "unique_content_files": 9,
+ "duplicate_files": 1,
+ "records_total": 42,
+ "junk_records_total": 0
+ });
+ let rendered = render_map(Some(&gen_run), &[], &[], &[], &[], 0);
+ assert!(
+ !rendered.contains("wall s"),
+ "must not render blank `wall s` when wall_seconds is absent: {rendered}"
+ );
+ assert!(
+ rendered.contains("42 records indexed, 0 junk records\n\n"),
+ "run summary line should end cleanly at junk records: {rendered}"
+ );
+
+ // When `wall_seconds` IS present (legacy graph path), keep `, wall Xs`.
+ let legacy_run = json!({
+ "doc_kind": "run",
+ "run_id": "run-legacy",
+ "root": "/corpus",
+ "files_total": 10,
+ "unique_content_files": 9,
+ "duplicate_files": 1,
+ "records_total": 42,
+ "junk_records_total": 0,
+ "wall_seconds": 1.4
+ });
+ let rendered_legacy = render_map(Some(&legacy_run), &[], &[], &[], &[], 0);
+ assert!(
+ rendered_legacy.contains("0 junk records, wall 1.4s\n\n"),
+ "legacy wall_seconds must still render: {rendered_legacy}"
+ );
+ }
+
+ #[test]
+ fn prefix_scoping_disambiguates_hyphenated_prefixes_across_catalog_doc_kinds() {
+ let tantivy_ds = json!({
+ "doc_kind": "dataset",
+ "slug": "code",
+ "index_name": "xc-tantivy-code"
+ });
+ assert!(dataset_matches_prefix(&tantivy_ds, "xc-tantivy"));
+ assert!(!dataset_matches_prefix(&tantivy_ds, "xc"));
+ assert!(!dataset_matches_prefix(&tantivy_ds, "xc-tantivy-extra"));
+
+ let xc_hyphen_slug_ds = json!({
+ "doc_kind": "dataset",
+ "slug": "tantivy-code",
+ "index_name": "xc-tantivy-code"
+ });
+ assert!(dataset_matches_prefix(&xc_hyphen_slug_ds, "xc"));
+ assert!(!dataset_matches_prefix(&xc_hyphen_slug_ds, "xc-tantivy"));
+
+ // Legacy dataset doc without `slug` falls back to `{prefix}-` boundary.
+ let legacy_ds = json!({
+ "doc_kind": "dataset",
+ "index_name": "ax-logs"
+ });
+ assert!(dataset_matches_prefix(&legacy_ds, "ax"));
+ assert!(!dataset_matches_prefix(&legacy_ds, "ax2"));
+
+ let corr = json!({
+ "doc_kind": "correlation",
+ "a_index": "xc-tantivy-code",
+ "a_dataset": "code",
+ "b_index": "xc-tantivy-symbols",
+ "b_dataset": "symbols"
+ });
+ assert!(correlation_matches_prefix(&corr, "xc-tantivy"));
+ assert!(!correlation_matches_prefix(&corr, "xc"));
+
+ let file_with_prefix = json!({
+ "doc_kind": "file",
+ "prefix": "xc-tantivy",
+ "path": "a.rs"
+ });
+ let file_with_scope_only = json!({
+ "doc_kind": "file",
+ CORPUS_SCOPE_FIELD: "xc-tantivy",
+ "path": "b.rs"
+ });
+ assert!(file_matches_prefix(&file_with_prefix, "xc-tantivy"));
+ assert!(!file_matches_prefix(&file_with_prefix, "xc"));
+ assert!(file_matches_prefix(&file_with_scope_only, "xc-tantivy"));
+ }
}
#[cfg(test)]
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index d95c401e9..e74bd6905 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -126,7 +126,9 @@ pub struct IndexCfg {
pub struct MapCfg {
pub url: String,
pub api_key: Option<String>,
- pub prefix: String,
+ /// Explicit `--prefix` filter (`None` when the user did not pass `--prefix`,
+ /// preserving the #971 multi-corpus view on a bare `autoindex map`).
+ pub prefix: Option<String>,
pub json: bool,
pub dataset: Option<String>,
/// A startup note when `XERJ_URL` was set but ignored (endpoint comes from
@@ -704,6 +706,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut snapshot_max_bytes = 64u64 << 30;
let mut bulk_timeout_explicit = false;
let mut prefix = "ax".to_string();
+ let mut prefix_explicit = false;
let mut state_dir: Option<PathBuf> = None;
let mut endpoint_url: Option<String> = None;
let mut endpoint_explicit = false;
@@ -832,7 +835,10 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
"--in-flight" => {
let _ = it.next(); // reserved (bulks are worker-synchronous in v1)
}
- "--prefix" => prefix = it.next().ok_or("--prefix needs a value")?,
+ "--prefix" => {
+ prefix = it.next().ok_or("--prefix needs a value")?;
+ prefix_explicit = true;
+ }
"--state-dir" => state_dir = it.next().map(PathBuf::from),
"--endpoint-url" => {
let value = it.next().ok_or(
@@ -1502,7 +1508,7 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
(Some("map"), _) => Ok(Cmd::Map(MapCfg {
url,
api_key,
- prefix,
+ prefix: prefix_explicit.then_some(prefix),
json,
dataset,
xerj_url_note,
@@ -2948,4 +2954,24 @@ mod tests {
"a valueless --dataset must be refused: {text}"
);
}
+
+ #[test]
+ fn map_records_explicit_prefix_and_leaves_unscoped_map_none() {
+ match parse(["map"].iter().map(|s| s.to_string()).collect()).expect("bare map must parse") {
+ Cmd::Map(cfg) => assert_eq!(cfg.prefix, None),
+ other => panic!("expected a map config, got {other:?}"),
+ }
+
+ match parse(
+ ["map", "--prefix", "xc-tantivy"]
+ .iter()
+ .map(|s| s.to_string())
+ .collect(),
+ )
+ .expect("map --prefix must parse")
+ {
+ Cmd::Map(cfg) => assert_eq!(cfg.prefix.as_deref(), Some("xc-tantivy")),
+ other => panic!("expected a map config, got {other:?}"),
+ }
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 5b62320ba..7ada26d4c 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -9358,16 +9358,35 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
{"term": {"slug": slug}}
]}});
}
- let datasets = fetch(ds_query, 500, Some(json!([{"record_count": "desc"}])))?;
+ let mut datasets = fetch(ds_query, 500, Some(json!([{"record_count": "desc"}])))?;
+ if let Some(prefix) = cfg.prefix.as_deref() {
+ datasets.retain(|d| catalog::dataset_matches_prefix(d, prefix));
+ }
if datasets.is_empty() {
- eprintln!(
- "no autoindex catalog found at {} (index {}) — run `xerj autoindex <folder>` first",
- cfg.url,
- catalog::CATALOG_INDEX
- );
+ if let Some(prefix) = cfg.prefix.as_deref() {
+ eprintln!(
+ "no autoindex catalog found for --prefix {prefix} at {} (index {}) — run `xerj autoindex <folder> --prefix {prefix}` first",
+ cfg.url,
+ catalog::CATALOG_INDEX
+ );
+ } else {
+ eprintln!(
+ "no autoindex catalog found at {} (index {}) — run `xerj autoindex <folder>` first",
+ cfg.url,
+ catalog::CATALOG_INDEX
+ );
+ }
return Ok(1);
}
- let mut runs = fetch(json!({"term": {"doc_kind": "run"}}), 50, None)?;
+ let mut runs = fetch(json!({"term": {"doc_kind": "run"}}), 500, None)?;
+ if let Some(prefix) = cfg.prefix.as_deref() {
+ runs.retain(|r| {
+ r.get("prefix")
+ .or_else(|| r.get(catalog::CORPUS_SCOPE_FIELD))
+ .and_then(Value::as_str)
+ == Some(prefix)
+ });
+ }
runs.sort_by_key(|r| {
std::cmp::Reverse(
r.get("started")
@@ -9378,6 +9397,9 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
});
let correlations = {
let mut all = fetch(json!({"term": {"doc_kind": "correlation"}}), 200, None)?;
+ if let Some(prefix) = cfg.prefix.as_deref() {
+ all.retain(|c| catalog::correlation_matches_prefix(c, prefix));
+ }
// stale-correlation hygiene: catalog docs upsert by deterministic id,
// so older runs' correlations linger — show only the latest run that
// produced each corr_kind.
@@ -9411,20 +9433,25 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
// since #737; on a pre-rc.68 catalog it was inferred, where a term
// matches only while the value has no separator — the same legacy caveat
// the scoped sweeps in `catalog.rs` document.)
- let prefixes: Vec<String> = runs
- .iter()
- .filter_map(|run| run.get("prefix"))
- .filter_map(Value::as_str)
- .collect::<BTreeSet<_>>()
- .into_iter()
- .map(String::from)
- .collect::<Vec<_>>();
+ let prefixes: Vec<String> = match cfg.prefix.as_deref() {
+ Some(prefix) => vec![prefix.to_string()],
+ None => runs
+ .iter()
+ .filter_map(|run| run.get("prefix"))
+ .filter_map(Value::as_str)
+ .collect::<BTreeSet<_>>()
+ .into_iter()
+ .map(String::from)
+ .collect::<Vec<_>>(),
+ };
let prefix_filter = (!prefixes.is_empty()).then(|| {
+ let mut should = Vec::with_capacity(prefixes.len() * 2);
+ for prefix in &prefixes {
+ should.push(json!({"term": {"prefix": prefix}}));
+ should.push(json!({"term": {catalog::CORPUS_SCOPE_FIELD: prefix}}));
+ }
json!({"bool": {
- "should": prefixes
- .iter()
- .map(|prefix| json!({"term": {"prefix": prefix}}))
- .collect::<Vec<_>>(),
+ "should": should,
"minimum_should_match": 1
}})
});
@@ -9432,7 +9459,7 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
if let Some(filter) = prefix_filter.clone() {
junk_must.push(filter);
}
- let junk_files = fetch(
+ let mut junk_files = fetch(
json!({"bool": {"must": junk_must,
"must_not": [
{"term": {"status": "indexed"}},
@@ -9448,7 +9475,11 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
if let Some(filter) = prefix_filter {
duplicate_must.push(filter);
}
- let duplicate_files = fetch(json!({"bool": {"must": duplicate_must}}), 500, None)?;
+ let mut duplicate_files = fetch(json!({"bool": {"must": duplicate_must}}), 500, None)?;
+ if let Some(prefix) = cfg.prefix.as_deref() {
+ junk_files.retain(|f| catalog::file_matches_prefix(f, prefix));
+ duplicate_files.retain(|f| catalog::file_matches_prefix(f, prefix));
+ }
if cfg.json {
println!(
"{}",