Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(hub): bare `xerj code <slug>` finds the node `hub pull` restored into (#1729)

xerj-orgd481370f2d2a
diff --git a/engine/crates/xerj-hub/src/cli.rs b/engine/crates/xerj-hub/src/cli.rs
index 2a95226ab..8f8b717ee 100644
--- a/engine/crates/xerj-hub/src/cli.rs
+++ b/engine/crates/xerj-hub/src/cli.rs
@@ -1276,10 +1276,79 @@ pub fn parse_ask_args(args: &[String]) -> Result<Option<AskArgs>, String> {
/// `xerj code …`: `--def <SYMBOL>` answers from the `-defs` index; everything
/// else is the existing retrieval path, delegated unchanged.
pub fn run_code(args: &[String]) -> i32 {
+ let args = match with_pulled_node_url(args) {
+ Ok(args) => args,
+ Err(e) => {
+ eprintln!("xerj code: {e:#}");
+ return 2;
+ }
+ };
if !args.iter().any(|a| a == "--def") {
- return xerj_autoindex::xc::run_code_cli(args);
+ return xerj_autoindex::xc::run_code_cli(&args);
+ }
+ run_code_def(&args)
+}
+
+/// The `xerj code` flags (either mode) that take a value.
+const CODE_VALUE_FLAGS: &[&str] = &[
+ "-k",
+ "--lang",
+ "--mode",
+ "--full",
+ "--url",
+ "--api-key",
+ "--def",
+];
+
+/// The `<corpus>` argument of `xerj code …`: the first positional, skipping the
+/// values of [`CODE_VALUE_FLAGS`] (both parsers take it in that position).
+fn code_corpus_arg(args: &[String]) -> Option<&str> {
+ let mut i = 0;
+ while i < args.len() {
+ let a = args[i].as_str();
+ if CODE_VALUE_FLAGS.contains(&a) {
+ i += 2;
+ continue;
+ }
+ if !a.starts_with('-') {
+ return Some(a);
+ }
+ i += 1;
}
- run_code_def(args)
+ None
+}
+
+/// For a PULLED corpus queried with no `--url` and no `XERJ_URL`, add
+/// `--url <the node the pack lives on>` ([`node::resolve_pulled_node`]).
+///
+/// Without this, `xerj code <slug>` — the command `xerj hub pull` tells the user
+/// to run next — resolved `localhost:9200` while the pack had been restored into
+/// the automatic node on 9300+: `--def` printed "no match … answered from: local
+/// pack" and a question failed with a transport error. `xerj ask` already used
+/// the automatic node. A locally indexed corpus, an explicit `--url` or
+/// `XERJ_URL`, and anything that is not a pulled corpus pass through unchanged.
+fn with_pulled_node_url(args: &[String]) -> Result<Vec<String>> {
+ let explicit_env = std::env::var("XERJ_URL").is_ok_and(|u| !u.is_empty());
+ let help = args.iter().any(|a| a == "-h" || a == "--help");
+ if explicit_env || help || args.iter().any(|a| a == "--url") {
+ return Ok(args.to_vec());
+ }
+ let Some(corpus) = code_corpus_arg(args) else {
+ return Ok(args.to_vec());
+ };
+ let Ok(st) = state::load_state(&xerj_autoindex::xc::code_root(), corpus) else {
+ return Ok(args.to_vec()); // the code path reports "not indexed" itself
+ };
+ if st.source.as_deref() != Some(state::SOURCE_HUBPACK) {
+ return Ok(args.to_vec());
+ }
+ let launcher = NodeLauncher::current()?;
+ let resolved = node::resolve_pulled_node(&hub_home(), st.url.as_deref(), &launcher)
+ .with_context(|| format!("cannot reach the node holding the pulled corpus '{corpus}'"))?;
+ let mut out = args.to_vec();
+ out.push("--url".into());
+ out.push(resolved.url);
+ Ok(out)
}
fn run_code_def(args: &[String]) -> i32 {
@@ -1358,11 +1427,18 @@ fn run_code_def(args: &[String]) -> i32 {
return 2;
}
let prefix = state::query_prefix(&st);
- if let Ok(live) = node.cat_indices_json(&format!("{prefix}*")) {
- if live.is_empty() {
+ match node.cat_indices_json(&format!("{prefix}*")) {
+ Ok(live) if live.is_empty() => {
eprintln!("{}", state::not_loaded_message(&st, &prefix, &url));
return 3;
}
+ Ok(_) => {}
+ // An unreachable node is a failure to answer (exit 2), not "no match"
+ // (exit 1) — and never "answered from: local pack".
+ Err(e) => {
+ eprintln!("xerj code: cannot reach {url}: {e}");
+ return 2;
+ }
}
let answer = crate::def::lookup(&node, &slug, &symbol, k);
@@ -1461,6 +1537,22 @@ mod tests {
}
}
+ #[test]
+ fn code_corpus_arg_skips_flag_values() {
+ let v = |xs: &[&str]| xs.iter().map(|x| x.to_string()).collect::<Vec<_>>();
+ assert_eq!(code_corpus_arg(&v(&["lz4", "q"])), Some("lz4"));
+ assert_eq!(code_corpus_arg(&v(&["--def", "X", "lz4"])), Some("lz4"));
+ assert_eq!(
+ code_corpus_arg(&v(&["-k", "3", "--json", "lz4", "q"])),
+ Some("lz4")
+ );
+ assert_eq!(
+ code_corpus_arg(&v(&["--mode", "bm25", "--lang", "c"])),
+ None
+ );
+ assert_eq!(code_corpus_arg(&v(&["--def"])), None);
+ }
+
#[test]
fn run_code_without_def_does_not_enter_the_def_path() {
// No node is contacted and no panic: a bare `code` with a bad flag
diff --git a/engine/crates/xerj-hub/src/node.rs b/engine/crates/xerj-hub/src/node.rs
index 302888769..e59254b40 100644
--- a/engine/crates/xerj-hub/src/node.rs
+++ b/engine/crates/xerj-hub/src/node.rs
@@ -559,6 +559,43 @@ fn ensure_node_with(
}
}
+/// The node a PULLED corpus (a `source: "hubpack"` ledger) is queried on when
+/// the caller named none (no `--url`, no `XERJ_URL`).
+///
+/// `xerj hub pull` restores into whatever [`ensure_node`] resolved — on a
+/// machine with nothing on `localhost:9200`, the automatic node on 9300+ — and
+/// records that URL in the ledger. A bare `xerj code <slug>` used to ignore the
+/// ledger and query `localhost:9200`, so the corpus `pull` had just restored was
+/// unreachable from the command `pull` tells the user to run next.
+///
+/// The recorded URL takes the place `--url` has in [`ensure_node`]: used when it
+/// answers; refused when it is remote and silent; and when it is local and
+/// silent (the automatic node was stopped, or its port changed), the automatic
+/// node is reused or restarted on its own data directory, where the restored
+/// indices persist. `localhost:9200` is NOT probed: a node there that never held
+/// the pack would answer "not loaded" for a corpus that is one restart away.
+/// A ledger with no URL falls back to plain [`ensure_node`].
+pub fn resolve_pulled_node(
+ hub_home: &Path,
+ recorded: Option<&str>,
+ launcher: &NodeLauncher,
+) -> Result<ResolvedNode> {
+ resolve_pulled_node_with(hub_home, recorded, launcher, NODE_BOOT_TIMEOUT)
+}
+
+/// [`resolve_pulled_node`] with the boot timeout spelled out (tests shorten it).
+fn resolve_pulled_node_with(
+ hub_home: &Path,
+ recorded: Option<&str>,
+ launcher: &NodeLauncher,
+ boot_timeout: Duration,
+) -> Result<ResolvedNode> {
+ let recorded = recorded
+ .map(|u| u.trim().trim_end_matches('/'))
+ .filter(|u| !u.is_empty());
+ ensure_node_with(hub_home, recorded, launcher, boot_timeout)
+}
+
fn unconfirmed_message(hub_home: &Path, rec: &NodeRecord) -> String {
if rec.port.is_none() {
format!(
@@ -1832,6 +1869,35 @@ mod tests {
assert!(!hub.path().join("node.pid").exists());
}
+ // ── resolve_pulled_node: a bare `xerj code <pulled slug>` ────────────────
+
+ #[test]
+ fn a_pulled_corpus_is_queried_on_the_node_its_ledger_recorded() {
+ // The bug: `pull` restored into the automatic node (9300+) and recorded
+ // its URL; a bare `xerj code` then queried localhost:9200 instead.
+ let hub = tempfile::tempdir().unwrap();
+ let port = fake_node("200 OK", r#"{"status":"green"}"#);
+ let recorded = format!("http://127.0.0.1:{port}/");
+ let r = resolve_pulled_node(hub.path(), Some(&recorded), &forbidden_launcher()).unwrap();
+ assert_eq!(r.url, format!("http://127.0.0.1:{port}"));
+ assert!(!r.started_now, "an answering node is never started over");
+ assert!(!hub.path().join("node.pid").exists());
+ }
+
+ #[test]
+ fn a_pulled_corpus_on_a_silent_remote_node_is_an_error_not_a_local_node() {
+ let hub = tempfile::tempdir().unwrap();
+ let err = resolve_pulled_node(
+ hub.path(),
+ Some("http://nonexistent.invalid:9200"),
+ &forbidden_launcher(),
+ )
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("not a local address"), "{err}");
+ assert!(!hub.path().join("node.pid").exists());
+ }
+
#[test]
fn free_port_search_skips_a_busy_triple() {
let held = TcpListener::bind("127.0.0.1:0").unwrap();
@@ -2401,6 +2467,24 @@ mod tests {
live.wait().unwrap();
}
+ #[test]
+ fn a_pulled_corpus_whose_recorded_node_moved_finds_the_automatic_node() {
+ // `xerj hub stop` + a later restart put the automatic node on another
+ // port than the one `pull` recorded. The ledger URL is silent and
+ // local, so the automatic node is the answer — and localhost:9200 is
+ // never consulted (a node there never held the pack).
+ let hub = tempfile::tempdir().unwrap();
+ let mut live = spawn_node_like(&node_data_dir(hub.path()));
+ let port = fake_node_with_pid(live.id());
+ record(hub.path(), live.id(), Some(port));
+ let r = resolve_pulled_node(hub.path(), Some(&silent_url()), &forbidden_launcher())
+ .unwrap();
+ assert_eq!(r.url, format!("http://127.0.0.1:{port}"));
+ assert!(r.managed && !r.started_now);
+ live.kill().unwrap();
+ live.wait().unwrap();
+ }
+
#[test]
fn something_else_on_the_recorded_port_is_not_taken_for_the_recorded_node() {
// The node is alive but has not bound its port yet, and another process
diff --git a/engine/crates/xerj-hub/tests/pack_roundtrip.rs b/engine/crates/xerj-hub/tests/pack_roundtrip.rs
index 207d3f8be..98631d5b9 100644
--- a/engine/crates/xerj-hub/tests/pack_roundtrip.rs
+++ b/engine/crates/xerj-hub/tests/pack_roundtrip.rs
@@ -491,6 +491,35 @@ async fn pack_build_pull_def_roundtrip() {
assert_eq!(answer.hits[0].file, "limits.h");
assert_eq!(answer.hits[0].line, 5);
+ // A BARE `xerj code` (no --url, no XERJ_URL) finds the node the pack was
+ // restored into, from the ledger — not localhost:9200. `pull` names exactly
+ // this command as the next step; it used to query localhost:9200, where
+ // `--def` printed "no match … answered from: local pack" (exit 1) and a
+ // question failed to connect (exit 2).
+ let (rc_def, rc_how) = {
+ let (ch, hh) = (code_home.clone(), hub_home.clone());
+ tokio::task::spawn_blocking(move || {
+ std::env::set_var("XERJ_CODE_HOME", &ch);
+ std::env::set_var("XERJ_HUB_HOME", &hh);
+ std::env::remove_var("XERJ_URL");
+ let def = xerj_hub::cli::run_code(&[
+ "mini-c".into(),
+ "--def".into(),
+ "MINI_LIMIT".into(),
+ "--json".into(),
+ ]);
+ let how = xerj_hub::cli::run_code(&["mini-c".into(), "mini c reference".into()]);
+ (def, how)
+ })
+ .await
+ .unwrap()
+ };
+ assert_eq!(
+ rc_def, 0,
+ "bare `xerj code mini-c --def MINI_LIMIT` answers"
+ );
+ assert_eq!(rc_how, 0, "bare `xerj code mini-c \"<question>\"` answers");
+
// Second pull of the same pin → already present, restores nothing.
let rc2 = {
let (url_b, ch, hh, pk) = (
diff --git a/tools/hub-packs/consumer_e2e.sh b/tools/hub-packs/consumer_e2e.sh
index c4c54cd72..550c19855 100755
--- a/tools/hub-packs/consumer_e2e.sh
+++ b/tools/hub-packs/consumer_e2e.sh
@@ -17,6 +17,8 @@
# snapshot restore — all against the key compiled into <xerj>;
# 4. `xerj code <slug> --def <symbol> --json`, the symbol taken from the pack's
# own defs.jsonl, must return that symbol from the local pack;
+# 4b. the same lookup with NO XERJ_URL must find the node from the ledger
+# `hub pull` wrote (not localhost:9200);
# 5. `xerj code <slug> "<word from ask_for>" --json` must return a hit with a
# file and a line.
#
@@ -108,6 +110,18 @@ jq -e --argjson d "$DEF" '
|| { cat "$WORK/def.json"; fail "--def $SYM did not return $(jq -r '"\(.file):\(.line)"' <<<"$DEF") from the local pack"; }
echo "def ok: $SYM -> $(jq -r '"\(.file):\(.line)"' <<<"$DEF") (answered_from local_pack)"
+# 4b. The same lookup BARE: no XERJ_URL, no --url — the command `hub pull` tells
+# a user to run next. It must find the node the pack was restored into from
+# the ledger, not fall back to localhost:9200 (where nothing listens here).
+env -u XERJ_URL "$BIN" code "$SLUG" --def "$SYM" --json > "$WORK/def-bare.json" \
+ || { cat "$WORK/def-bare.json"; fail "bare xerj code $SLUG --def $SYM (no XERJ_URL) failed"; }
+jq -e --argjson d "$DEF" '
+ .answered_from == "local_pack"
+ and ([.hits[]? | select(.file == $d.file and .line == $d.line)] | length) > 0' \
+ "$WORK/def-bare.json" >/dev/null \
+ || { cat "$WORK/def-bare.json"; fail "bare --def $SYM (no XERJ_URL) did not find the pulled pack's node"; }
+echo "bare def ok: $SYM with no XERJ_URL found the pulled pack's node"
+
# 5. One question: a word from the catalogue entry's ask_for. Words the template
# sentence ("an agent working on ... queries this for ...") always carries say
# nothing about the corpus; the longest remaining words are tried, at most three.