Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): say a prefix's first run has not finished on a search miss (#2670) (#2696)

Jake Elbirtbefe92035e2d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6d7f06204..823107650 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -166,6 +166,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
0 for a run that would then be refused. The `--max-minutes` note on this
route recommends `--dry-run` as the way to check a plan first (#2684).
+- `xerj search` and `xerj def` with no hits on a prefix whose autoindex
+ indices exist but whose first run has not finished now say so: search again
+ once the run finishes, or rerun the same command if it stopped. They used to
+ print the node's other indices and "the catalog lists no prefixes; --prefix
+ is the index name before the dataset slug", which read as a wrong `--prefix`
+ (#2670).
+
## [1.0.0-rc.100] - 2026-10-26
**rc.100: faster autoindex and search, shaped by your field reports.** This
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rs
index e22997d1b..a58c0565a 100644
--- a/engine/crates/xerj-autoindex/src/search.rs
+++ b/engine/crates/xerj-autoindex/src/search.rs
@@ -405,6 +405,13 @@ pub(crate) struct CorpusSnapshot {
pub all_indices: Option<Vec<String>>,
/// `prefix` values on catalog run documents.
pub catalog_prefixes: Option<Vec<String>>,
+ /// Whether a catalog run document names this prefix: written at the end
+ /// of a run, so `Some(false)` means no run under it has finished yet.
+ pub finished_run: Option<bool>,
+ /// Whether every index under the prefix maps autoindex's provenance
+ /// fields, which it creates them with. Read only when `finished_run` is
+ /// `Some(false)`; `None` otherwise or when the read failed.
+ pub autoindex_mapped: Option<bool>,
}
/// Lines to print, and the process exit code, when `xerj search` or
@@ -452,7 +459,27 @@ pub(crate) fn empty_lookup_report(
None => no_indices,
};
let catalog_empty = no_indices && no_prefixes;
- if catalog_empty {
+ // #2670: a first run creates its indices before they hold anything
+ // searchable. The prefix is right, so none of the hints below apply. A
+ // longer catalog prefix (`rp-2` under `rp-*`) or indices autoindex did
+ // not create keep those hints: there the prefix may well be wrong.
+ let sibling = format!("{prefix}-");
+ let longer_prefix = snap
+ .catalog_prefixes
+ .iter()
+ .flatten()
+ .any(|p| p.starts_with(&sibling));
+ let unfinished = !prefix_absent
+ && snap.finished_run == Some(false)
+ && snap.autoindex_mapped == Some(true)
+ && !longer_prefix;
+ if unfinished {
+ lines.push(format!(
+ "(no xerj autoindex run under --prefix {prefix} has finished on this node yet: \
+ if one is running, search again once it finishes; if it stopped, rerun the \
+ same command to resume)"
+ ));
+ } else if catalog_empty {
lines.push("(index a relevant repo first: xerj autoindex <folder>)".to_string());
} else if let Some(prefixes) = snap.catalog_prefixes.as_ref().filter(|p| !p.is_empty()) {
// The missing-prefix headline already lists these. A real miss (the
@@ -521,13 +548,78 @@ pub(crate) fn corpus_snapshot(es: &Es, prefix: &str) -> CorpusSnapshot {
names.sort();
names
});
+ let has_indices = under_prefix.as_ref().is_some_and(|n| !n.is_empty());
+ let finished_run = if has_indices {
+ catalog_has_run_for(es, prefix)
+ } else {
+ None
+ };
+ let autoindex_mapped = if finished_run == Some(false) {
+ all_autoindex_mapped(es, prefix)
+ } else {
+ None
+ };
CorpusSnapshot {
under_prefix,
all_indices,
catalog_prefixes: catalog_run_prefixes(es),
+ finished_run,
+ autoindex_mapped,
}
}
+/// Whether every index matching `{prefix}-*` maps `ax_run`, a provenance
+/// field autoindex puts in each mapping it creates (`build_mapping`).
+fn all_autoindex_mapped(es: &Es, prefix: &str) -> Option<bool> {
+ let v = es.get_json(&format!("/{prefix}-*/_mapping")).ok()?;
+ let indices = v.as_object().filter(|o| !o.is_empty())?;
+ Some(
+ indices
+ .values()
+ .all(|m| m.pointer("/mappings/properties/ax_run").is_some()),
+ )
+}
+
+/// Whether a catalog run document names `prefix` exactly.
+///
+/// A missing catalog is `Some(false)`. `None` when the answer is not known:
+/// the read failed or timed out, more runs exist than were read, or a run
+/// predates the `prefix` field (rc.68) and names no corpus.
+fn catalog_has_run_for(es: &Es, prefix: &str) -> Option<bool> {
+ let body = json!({
+ "size": 200,
+ "query": { "term": { "doc_kind": "run" } },
+ "_source": ["prefix", crate::catalog::CORPUS_SCOPE_FIELD]
+ });
+ let Some(v) = es
+ .search_present(xerj_common::AUTOINDEX_CATALOG_INDEX, &body)
+ .ok()?
+ else {
+ return Some(false);
+ };
+ if v.get("timed_out").and_then(Value::as_bool) == Some(true) {
+ return None;
+ }
+ let hits = v.pointer("/hits/hits").and_then(Value::as_array)?;
+ let mut found = false;
+ for hit in hits {
+ let named = hit
+ .pointer("/_source/prefix")
+ .or_else(|| hit.pointer(&format!("/_source/{}", crate::catalog::CORPUS_SCOPE_FIELD)))
+ .and_then(Value::as_str);
+ match named {
+ Some(name) if name == prefix => found = true,
+ Some(_) => {}
+ None => return None,
+ }
+ }
+ if found {
+ return Some(true);
+ }
+ let total = v.pointer("/hits/total/value").and_then(Value::as_u64)?;
+ (total <= hits.len() as u64).then_some(false)
+}
+
/// Distinct `prefix` values on catalog run and dataset documents, sorted.
///
/// Run documents are the historical source. Dataset documents carry the same
@@ -1959,6 +2051,8 @@ mod tests {
under_prefix: owned(under),
all_indices: owned(all),
catalog_prefixes: owned(prefixes),
+ finished_run: None,
+ autoindex_mapped: None,
}
}
@@ -2316,4 +2410,176 @@ mod tests {
vec!["src/model.py", "tests/test_model.py", "src/model.pyi"]
);
}
+
+ /// Serve `xerj search`'s no-hit reads from fixed bodies until `stop` is
+ /// set: `rp-docs` exists with the `ax_run` provenance field when
+ /// `autoindex` is set, and the catalog search answers `catalog` (`None`
+ /// is a missing catalog).
+ fn no_hit_node(
+ catalog: Option<&'static str>,
+ autoindex: bool,
+ ) -> (
+ String,
+ std::sync::Arc<std::sync::atomic::AtomicBool>,
+ std::thread::JoinHandle<()>,
+ ) {
+ use std::io::{Read, Write};
+ use std::sync::atomic::Ordering;
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ listener.set_nonblocking(true).unwrap();
+ let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let stopped = stop.clone();
+ let server = std::thread::spawn(move || {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ while !stopped.load(Ordering::SeqCst) && std::time::Instant::now() < deadline {
+ let Ok((mut sock, _)) = listener.accept() else {
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ continue;
+ };
+ sock.set_nonblocking(false).unwrap();
+ let mut req = Vec::new();
+ let mut buf = [0u8; 4096];
+ let header_end = loop {
+ let n = sock.read(&mut buf).unwrap_or(0);
+ if n == 0 {
+ break None;
+ }
+ req.extend_from_slice(&buf[..n]);
+ if let Some(i) = req.windows(4).position(|w| w == b"\r\n\r\n") {
+ break Some(i + 4);
+ }
+ };
+ let Some(header_end) = header_end else {
+ continue;
+ };
+ let head = String::from_utf8_lossy(&req[..header_end]).to_ascii_lowercase();
+ let body_len = head
+ .lines()
+ .find_map(|l| l.strip_prefix("content-length:"))
+ .and_then(|v| v.trim().parse::<usize>().ok())
+ .unwrap_or(0);
+ while req.len() < header_end + body_len {
+ let n = sock.read(&mut buf).unwrap_or(0);
+ if n == 0 {
+ break;
+ }
+ req.extend_from_slice(&buf[..n]);
+ }
+ let line = head.lines().next().unwrap_or("").to_string();
+ let (status, body) = if line.contains("/_mapping") {
+ let props = if autoindex {
+ r#"{"ax_run":{"type":"keyword"},"body":{"type":"text"}}"#
+ } else {
+ r#"{"body":{"type":"text"}}"#
+ };
+ (
+ "200 OK",
+ format!(r#"{{"rp-docs":{{"mappings":{{"properties":{props}}}}}}}"#),
+ )
+ } else if line.contains("/_cat/indices/") {
+ ("200 OK", r#"[{"index":"rp-docs"}]"#.to_string())
+ } else if line.contains("/_cat/indices") {
+ (
+ "200 OK",
+ "green open rp-docs u1 1 0 0 0 1kb 1kb\n".to_string(),
+ )
+ } else if line.contains("/autoindex-catalog/_search") {
+ match catalog {
+ Some(body) => ("200 OK", body.to_string()),
+ None => (
+ "404 Not Found",
+ r#"{"error":{"type":"index_not_found_exception","reason":"no such index [autoindex-catalog]"},"status":404}"#
+ .to_string(),
+ ),
+ }
+ } else {
+ ("404 Not Found", r#"{"error":"unexpected"}"#.to_string())
+ };
+ let _ = write!(
+ sock,
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ }
+ });
+ (url, stop, server)
+ }
+
+ fn no_hit_report(
+ command: &str,
+ catalog: Option<&'static str>,
+ autoindex: bool,
+ ) -> (String, i32) {
+ let (url, stop, server) = no_hit_node(catalog, autoindex);
+ let es = Es::new(&url, None).unwrap();
+ let snap = corpus_snapshot(&es, "rp");
+ let (lines, exit) = empty_lookup_report(command, "rp", "refresh_once", false, &snap);
+ stop.store(true, std::sync::atomic::Ordering::SeqCst);
+ server.join().unwrap();
+ (lines.join("\n"), exit)
+ }
+
+ /// #2670: during a first `xerj autoindex` run the prefix's indices exist
+ /// before they hold anything searchable. A miss then must say no run has
+ /// finished under the prefix, not hint that `--prefix` is wrong. A prefix
+ /// whose run did finish, a longer finished prefix (`rp-2` also matches
+ /// `rp-*`) and indices autoindex did not create keep the old hints.
+ #[test]
+ fn a_miss_before_any_run_finished_says_so_instead_of_blaming_the_prefix() {
+ let runs = |prefix: &str| -> &'static str {
+ match prefix {
+ "" => r#"{"hits":{"total":{"value":0,"relation":"eq"},"hits":[]}}"#,
+ "other" => {
+ r#"{"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_source":{"doc_kind":"run","prefix":"other"}}]}}"#
+ }
+ "rp-2" => {
+ r#"{"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_source":{"doc_kind":"run","prefix":"rp-2"}}]}}"#
+ }
+ _ => {
+ r#"{"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_source":{"doc_kind":"run","prefix":"rp"}}]}}"#
+ }
+ }
+ };
+ let pending = "no xerj autoindex run under --prefix rp has finished on this node yet";
+ for (label, catalog) in [
+ ("empty catalog", Some(runs(""))),
+ ("missing catalog", None),
+ ("another prefix finished", Some(runs("other"))),
+ ] {
+ let (text, exit) = no_hit_report("search", catalog, true);
+ assert_eq!(exit, 0, "{label}: {text}");
+ assert!(
+ text.contains("no match in rp-* for: refresh_once"),
+ "{label}: {text}"
+ );
+ assert!(text.contains(pending), "{label}: {text}");
+ assert!(
+ !text.contains("--prefix is the index name") && !text.contains("try: xerj"),
+ "{label}: the prefix is right, its run has not finished: {text}"
+ );
+ }
+
+ let (text, _) = no_hit_report("def", Some(runs("")), true);
+ assert!(
+ text.contains("no definition in rp-* for: refresh_once"),
+ "{text}"
+ );
+ assert!(text.contains(pending), "{text}");
+
+ let (text, _) = no_hit_report("search", Some(runs("rp")), true);
+ assert!(
+ text.contains("no match in rp-* for: refresh_once"),
+ "{text}"
+ );
+ assert!(!text.contains("has finished on this node"), "{text}");
+
+ let (text, _) = no_hit_report("search", Some(runs("rp-2")), true);
+ assert!(!text.contains("has finished on this node"), "{text}");
+ assert!(text.contains("try: xerj search --prefix rp-2"), "{text}");
+
+ let (text, _) = no_hit_report("search", Some(runs("")), false);
+ assert!(!text.contains("has finished on this node"), "{text}");
+ assert!(text.contains("--prefix is the index name"), "{text}");
+ }
}