AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(hub): name the upstream repo in hub find results (#2249)
xerj-orgd4e61cc5e57d
diff --git a/engine/crates/xerj-api/src/hub_api.rs b/engine/crates/xerj-api/src/hub_api.rsindex 2b45bac95..1de1acd67 100644--- a/engine/crates/xerj-api/src/hub_api.rs+++ b/engine/crates/xerj-api/src/hub_api.rs@@ -549,10 +549,11 @@ impl CatalogueData {let parsed = serde_json::from_slice::<Catalogue>(&bytes).map_err(|e| e.to_string()).and_then(|c| {- if c.format_version != CATALOGUE_FORMAT_VERSION {+ if !xerj_common::hub::catalogue_format_supported(c.format_version) {return Err(format!(- "catalogue format_version {} (this build reads {CATALOGUE_FORMAT_VERSION})",- c.format_version+ "catalogue format_version {} (this build reads {}..={CATALOGUE_FORMAT_VERSION})",+ c.format_version,+ xerj_common::hub::CATALOGUE_FORMAT_VERSION_MIN));}Ok(c)diff --git a/engine/crates/xerj-api/tests/hub_api.rs b/engine/crates/xerj-api/tests/hub_api.rsindex 49ffbe07b..30bad679d 100644--- a/engine/crates/xerj-api/tests/hub_api.rs+++ b/engine/crates/xerj-api/tests/hub_api.rs@@ -93,6 +93,7 @@ fn entry(slug: &str, spdx: &str, class: LicenceClass) -> CatalogueEntry {domain: "DEV".into(),topics: vec!["c".into()],ask_for: format!("fixture corpus {slug}"),+ upstream: String::new(),pin_sha: PIN_SHA.into(),pin_date: PIN_DATE.into(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-common/src/hub.rs b/engine/crates/xerj-common/src/hub.rsindex c4b2e9792..5fafeadbc 100644--- a/engine/crates/xerj-common/src/hub.rs+++ b/engine/crates/xerj-common/src/hub.rs@@ -18,9 +18,19 @@use serde::{Deserialize, Serialize};use std::fmt::Write as _;-/// Bump when the on-the-wire catalogue shape changes. The client refuses a-/// catalogue whose `format_version` it does not equal.-pub const CATALOGUE_FORMAT_VERSION: u32 = 1;+/// The catalogue shape this build writes. 2 adds optional `upstream`+/// (`owner/name` of the pack's git remote) on each entry. Readers still load+/// format 1 (that field is absent and reads as empty) so a cached catalogue+/// from before this field keeps working. A newer shape is refused.+/// See [`catalogue_format_supported`].+pub const CATALOGUE_FORMAT_VERSION: u32 = 2;+/// Oldest catalogue shape this build still loads.+pub const CATALOGUE_FORMAT_VERSION_MIN: u32 = 1;++/// True for a catalogue `format_version` this build can read.+pub fn catalogue_format_supported(version: u32) -> bool {+ (CATALOGUE_FORMAT_VERSION_MIN..=CATALOGUE_FORMAT_VERSION).contains(&version)+}/// Bump when the pack archive layout changes. `format_version: 1` means the/// snapshot layout (engine `create_snapshot`/`restore_snapshot`).@@ -135,6 +145,12 @@ pub struct CatalogueEntry {pub topics: Vec<String>,/// The registry's one-line "an agent working on ___ queries this for ___".pub ask_for: String,+ /// Upstream git repo as `owner/name` (the manifest's `repos[].url`, GitHub+ /// form). Several repos are joined with ", " (at most four). Empty on a+ /// catalogue written before this field, or when the manifest recorded no+ /// `owner/name` remote — never guessed from the slug.+ #[serde(default, skip_serializing_if = "String::is_empty")]+ pub upstream: String,/// Full 40-hex commit the pack was built at.pub pin_sha: String,/// YYYY-MM-DD the pin was taken.@@ -481,6 +497,7 @@ mod tests {domain: "NET".into(),topics: vec!["compression".into()],ask_for: "fast block compression".into(),+ upstream: "lz4/lz4".into(),pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".into(),pin_date: "2026-10-02".into(),refresh_cadence_days: 7,@@ -511,12 +528,34 @@ mod tests {let back: Catalogue = serde_json::from_str(&s).unwrap();assert_eq!(back.corpora.len(), 1);assert_eq!(back.corpora[0].slug, "lz4");+ assert_eq!(back.corpora[0].upstream, "lz4/lz4");assert_eq!(back.corpora[0].licence_class,LicenceClass::AdaptWithAttribution);}+ /// A catalogue written before `upstream` existed has no such key. It still+ /// loads, and the field is empty rather than invented from the slug.+ #[test]+ fn a_catalogue_entry_without_upstream_loads_as_empty() {+ let mut cat = Catalogue {+ format_version: CATALOGUE_FORMAT_VERSION_MIN,+ generated_at: "2026-10-14T00:00:00Z".into(),+ corpora: vec![sample_entry()],+ };+ cat.corpora[0].upstream.clear();+ let mut v = serde_json::to_value(&cat).unwrap();+ v["corpora"][0].as_object_mut().unwrap().remove("upstream");+ let back: Catalogue = serde_json::from_value(v).unwrap();+ assert_eq!(back.corpora[0].upstream, "");+ assert_eq!(back.corpora[0].slug, "lz4");+ assert!(catalogue_format_supported(CATALOGUE_FORMAT_VERSION_MIN));+ assert!(catalogue_format_supported(CATALOGUE_FORMAT_VERSION));+ assert!(!catalogue_format_supported(0));+ assert!(!catalogue_format_supported(CATALOGUE_FORMAT_VERSION + 1));+ }+#[test]fn unknown_field_is_refused() {// A catalogue entry with a field this version does not know is rejected,diff --git a/engine/crates/xerj-hub/src/ask.rs b/engine/crates/xerj-hub/src/ask.rsindex b4f6da5fc..5ef88df8f 100644--- a/engine/crates/xerj-hub/src/ask.rs+++ b/engine/crates/xerj-hub/src/ask.rs@@ -847,6 +847,62 @@ fn printable_slug(s: &str) -> String {/// Step 2. `None` means `out` already holds the verdict (an exit code and its/// message / candidate lines). A tight group comes back as two or three slugs,+fn catalogue_entry<'a>(cat: &'a Catalogue, slug: &str) -> Option<&'a CatalogueEntry> {+ cat.corpora.iter().find(|e| e.slug == slug)+}++/// Candidate lines for a decision: each pack's upstream and description, plus+/// a note when the question named a different project of the same stem.+fn candidate_text(question: &str, cat: &Catalogue, slugs: &[String]) -> String {+ slugs+ .iter()+ .map(|s| match catalogue_entry(cat, s) {+ Some(e) => {+ let mut line = identity_line(&e.slug, &e.upstream, &e.ask_for);+ if let Some(token) =+ different_project_token(question, &e.slug, &e.upstream, &e.ask_for)+ {+ line.push('\n');+ line.push_str(&collision_sentence(&token, &e.slug, &e.upstream));+ }+ line+ }+ None => printable(s),+ })+ .collect::<Vec<_>>()+ .join("\n")+}++/// `(query token, slug, upstream)` of the first listed corpus the question+/// names as a different project.+fn first_collision(+ question: &str,+ cat: &Catalogue,+ slugs: &[String],+) -> Option<(String, String, String)> {+ for s in slugs {+ let Some(e) = catalogue_entry(cat, s) else {+ continue;+ };+ if let Some(token) = different_project_token(question, &e.slug, &e.upstream, &e.ask_for) {+ return Some((token, e.slug.clone(), e.upstream.clone()));+ }+ }+ None+}++fn collision_hint(token: &str, slug: &str, upstream: &str) -> String {+ let who = if upstream.is_empty() {+ printable(slug)+ } else {+ printable(upstream)+ };+ format!(+ "\"{}\" is a different project than this pack ({who}); re-run with --corpus <slug> only if that is the project you meant",+ printable(token)+ )+}+/// best first; one confident corpus comes back as a one-element list.fn select_corpus(args: &AskArgs,@@ -929,22 +985,29 @@ fn select_corpus(};let routing = cap_ambiguous(routing);match routing {- Routing::Confident(slug) => Some(vec![slug]),- Routing::Group(slugs) => Some(slugs),+ Routing::Confident(slug) => {+ let slugs = vec![slug];+ if let Some((token, s, up)) = first_collision(&args.question, catalogue, &slugs) {+ out.stdout = candidate_text(&args.question, catalogue, &slugs);+ out.stderr.push(collision_hint(&token, &s, &up));+ out.exit = EXIT_NEEDS_DECISION;+ None+ } else {+ Some(slugs)+ }+ }+ Routing::Group(slugs) => {+ if let Some((token, s, up)) = first_collision(&args.question, catalogue, &slugs) {+ out.stdout = candidate_text(&args.question, catalogue, &slugs);+ out.stderr.push(collision_hint(&token, &s, &up));+ out.exit = EXIT_NEEDS_DECISION;+ None+ } else {+ Some(slugs)+ }+ }Routing::Ambiguous(slugs) => {- let lines: Vec<String> = slugs- .iter()- .map(|s| {- let ask_for = catalogue- .corpora- .iter()- .find(|e| &e.slug == s)- .map(|e| printable(&e.ask_for))- .unwrap_or_default();- format!("{} — {ask_for}", printable(s))- })- .collect();- out.stdout = lines.join("\n");+ out.stdout = candidate_text(&args.question, catalogue, &slugs);out.stderr.push("more than one corpus matches; re-run with --corpus <slug>".to_string());out.exit = EXIT_NEEDS_DECISION;@@ -953,13 +1016,29 @@ fn select_corpus(Routing::NoCorpus => {out.stderr.push(NO_CORPUS_MESSAGE.to_string());// Nothing was credible, but something may have been close: name the- // nearest few so the asker can pick one with `--corpus` instead of- // starting over from "no".+ // nearest few, with what each pack is, so the asker can pick one+ // with `--corpus` instead of starting over from "no".let closest: Vec<String> = ranked.iter().filter(|(slug, _)| known(slug)).take(CLOSEST_SHOWN)- .map(|(slug, _)| printable(slug))+ .map(|(slug, _)| match catalogue_entry(catalogue, slug) {+ Some(e) => {+ let mut line = identity_line(&e.slug, &e.upstream, &e.ask_for);+ if let Some(token) = different_project_token(+ &args.question,+ &e.slug,+ &e.upstream,+ &e.ask_for,+ ) {+ line.push_str(" (");+ line.push_str(&collision_sentence(&token, &e.slug, &e.upstream));+ line.push(')');+ }+ line+ }+ None => printable(slug),+ }).collect();if !closest.is_empty() {out.stderr.push(format!(@@ -1785,12 +1864,26 @@ pub struct FoundCorpus {pub licence_spdx: String,pub licence_class: LicenceClass,pub pack_bytes: u64,+ /// `owner/name` of the upstream repo. Empty when the catalogue has none.+ #[serde(default, skip_serializing_if = "String::is_empty")]+ pub upstream: String,+ /// A query token that names a different project sharing only this slug's+ /// stem (`fabric.js` against `danielmiessler/fabric`). Empty when the+ /// names agree.+ #[serde(default, skip_serializing_if = "String::is_empty")]+ pub different_project: String,}/// What [`find`] decided: the router's verdict and the candidates, best first.#[derive(Debug, Clone, serde::Serialize)]pub struct FindOutcome {- /// `confident` | `ambiguous` | `no_corpus` — the verdict `xerj ask` would act on.+ /// `confident` | `ambiguous` | `no_corpus` | `different_project`.+ ///+ /// `different_project` means the router would have answered from a slug+ /// whose name the question only shares: the question names another project+ /// (`fabric.js`) and this pack is a different one (`danielmiessler/fabric`).+ /// `xerj ask` does not answer that as a silent match; it prints the pack's+ /// identity and exits 4.pub routing: &'static str,pub corpora: Vec<FoundCorpus>,}@@ -1838,26 +1931,47 @@ pub fn find(.map(|(_, score)| *score).unwrap_or(0.0)};- let corpora = order+ let corpora: Vec<FoundCorpus> = order.iter().filter_map(|slug| catalogue.corpora.iter().find(|e| &e.slug == slug)).filter(|e| e.licence_class != LicenceClass::Excluded).take(k)- .map(|e| FoundCorpus {- slug: e.slug.clone(),- score: score_of(&e.slug),- cached: cached.contains(&e.slug),- ask_for: e.ask_for.clone(),- domain: e.domain.clone(),- pin_sha: e.pin_sha.clone(),- pin_date: e.pin_date.clone(),- licence_spdx: e.licence_spdx.clone(),- licence_class: e.licence_class,- pack_bytes: e.pack_bytes,+ .map(|e| {+ let different_project =+ different_project_token(question, &e.slug, &e.upstream, &e.ask_for)+ .unwrap_or_default();+ FoundCorpus {+ slug: e.slug.clone(),+ score: score_of(&e.slug),+ cached: cached.contains(&e.slug),+ ask_for: e.ask_for.clone(),+ domain: e.domain.clone(),+ pin_sha: e.pin_sha.clone(),+ pin_date: e.pin_date.clone(),+ licence_spdx: e.licence_spdx.clone(),+ licence_class: e.licence_class,+ pack_bytes: e.pack_bytes,+ upstream: e.upstream.clone(),+ different_project,+ }}).collect();+ let verdict_slugs: Vec<&str> = match &routing {+ Routing::Confident(slug) => vec![slug.as_str()],+ Routing::Group(slugs) | Routing::Ambiguous(slugs) => {+ slugs.iter().map(String::as_str).collect()+ }+ Routing::NoCorpus => Vec::new(),+ };+ // A confident (or grouped) name match that is actually a different project+ // is not reported as confident: the caller must see the mismatch.+ let collided = matches!(routing, Routing::Confident(_) | Routing::Group(_))+ && corpora+ .iter()+ .any(|c| !c.different_project.is_empty() && verdict_slugs.contains(&c.slug.as_str()));FindOutcome {routing: match routing {+ _ if collided => "different_project",Routing::Confident(_) | Routing::Group(_) => "confident",Routing::Ambiguous(_) => "ambiguous",Routing::NoCorpus => "no_corpus",@@ -1899,34 +2013,152 @@ pub fn run_find(question: &str, k: usize) -> Result<FindOutcome, String> {Ok(find(question, k, &cwd, &hub_home, &catalogue))}+/// One result line: slug, upstream `owner/name` when the catalogue has one, and+/// the one-line description. Catalogue strings are made [`printable`].+pub fn identity_line(slug: &str, upstream: &str, ask_for: &str) -> String {+ let slug = printable(slug);+ let ask = printable(ask_for);+ let upstream = printable(upstream);+ if upstream.is_empty() {+ format!("{slug} — {ask}")+ } else {+ format!("{slug} — {upstream} — {ask}")+ }+}++/// The sentence that says a query token is not this pack.+pub fn collision_sentence(token: &str, slug: &str, upstream: &str) -> String {+ let token = printable(token);+ let who = if upstream.is_empty() {+ printable(slug)+ } else {+ printable(upstream)+ };+ format!("note: \"{token}\" is a different project than this pack ({who})")+}+/// Render a [`FindOutcome`] as text, one corpus per line (catalogue strings-/// made [`printable`]).+/// made [`printable`]). A same-name different project gets its own note line+/// instead of reading as a plain match.pub fn render_find(found: &FindOutcome) -> String {if found.corpora.is_empty() {return NO_CORPUS_MESSAGE.to_string();}- let mut lines: Vec<String> = found- .corpora- .iter()- .map(|c| {- format!(- "{} — {} (score {:.2}, pin {}, {}{})",- printable(&c.slug),- printable(&c.ask_for),- c.score,- printable(&c.pin_date),- printable(&c.licence_spdx),- if c.cached { ", cached" } else { "" }- )- })- .collect();- lines.push(format!(- "routing: {}; ask one with xerj_hub_ask {{question, corpus}}",- found.routing- ));+ let mut lines: Vec<String> = Vec::new();+ for c in &found.corpora {+ lines.push(format!(+ "{} (score {:.2}, pin {}, {}{})",+ identity_line(&c.slug, &c.upstream, &c.ask_for),+ c.score,+ printable(&c.pin_date),+ printable(&c.licence_spdx),+ if c.cached { ", cached" } else { "" }+ ));+ if !c.different_project.is_empty() {+ lines.push(collision_sentence(+ &c.different_project,+ &c.slug,+ &c.upstream,+ ));+ }+ }+ if found.routing == "different_project" {+ lines.push(+ "routing: different_project; this slug is a different project than the query names. \+ ask it with xerj_hub_ask {question, corpus} only if that pack is the one you meant"+ .to_string(),+ );+ } else {+ lines.push(format!(+ "routing: {}; ask one with xerj_hub_ask {{question, corpus}}",+ found.routing+ ));+ }lines.join("\n")}+/// Suffixes that make a token a different project's name (`fabric.js`), not a+/// file in the corpus (`lz4.h`, `lz4.c`).+const PROJECT_SUFFIXES: &[&str] = &[+ ".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".py", ".rb", ".go", ".php", ".rs",+];+/// Glued forms of the same idea (`fabricjs`), only for suffixes short enough to+/// be a language tag and long enough that the stem is still a name.+const GLUED_PROJECT_SUFFIXES: &[&str] = &["js", "py"];++/// The query token that names a different project than `slug`, when the token's+/// stem is exactly the slug and neither the upstream repo nor the one-line+/// description contains the token. `None` when the question names this pack,+/// or names nothing that only shares the slug.+pub fn different_project_token(+ question: &str,+ slug: &str,+ upstream: &str,+ ask_for: &str,+) -> Option<String> {+ let slug_l = slug.to_ascii_lowercase();+ if slug_l.len() < 2 {+ return None;+ }+ let upstream_l = upstream.to_ascii_lowercase();+ let ask_l = ask_for.to_ascii_lowercase();+ for token in query_name_tokens(question) {+ if token == slug_l {+ continue;+ }+ if !upstream_l.is_empty() && upstream_l.contains(&token) {+ continue;+ }+ if ask_l.contains(&token) {+ continue;+ }+ if project_stem(&token).as_deref() == Some(slug_l.as_str()) {+ return Some(token);+ }+ }+ None+}++fn query_name_tokens(question: &str) -> Vec<String> {+ let lower = question.to_ascii_lowercase();+ let mut out = Vec::new();+ let mut seen = std::collections::HashSet::new();+ for word in lower.split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')))+ {+ let word = word.trim_matches(|c| c == '.' || c == '-');+ // Same floor as the router's named-slug tokens (2): a one-letter+ // fragment is not a project name.+ if word.len() < 2 {+ continue;+ }+ if seen.insert(word.to_string()) {+ out.push(word.to_string());+ }+ }+ out+}++fn project_stem(token: &str) -> Option<String> {+ for suffix in PROJECT_SUFFIXES {+ if let Some(stem) = token.strip_suffix(suffix) {+ if stem.len() >= 2 && !stem.contains('.') && !stem.contains('/') {+ return Some(stem.to_string());+ }+ }+ }+ for suffix in GLUED_PROJECT_SUFFIXES {+ if token.len() <= suffix.len() + 2 {+ continue;+ }+ if let Some(stem) = token.strip_suffix(suffix) {+ if stem.chars().all(|c| c.is_ascii_alphanumeric()) {+ return Some(stem.to_string());+ }+ }+ }+ None+}+#[cfg(test)]mod tests {use super::*;@@ -1948,6 +2180,7 @@ mod tests {domain: "TEST".to_string(),topics: vec![],ask_for: format!("an agent working on {slug} queries this for its internals"),+ upstream: String::new(),pin_sha: "0".repeat(40),pin_date: "2026-10-01".to_string(),refresh_cadence_days: 7,@@ -3879,6 +4112,7 @@ mod tests {let text = render_find(&found);assert!(text.starts_with("lz4 — "), "{text}");assert!(text.contains("cached"), "{text}");+ assert!(text.contains("routing: "), "{text}");// k is clamped to 1..=FIND_MAX_K.let one = find("lz4 block decoder", 0, rig.cwd.path(), rig.hub.path(), &cat);@@ -3891,4 +4125,105 @@ mod tests {assert_eq!(render_find(&none), NO_CORPUS_MESSAGE);}}++ /// Issue #2179: a query that names a different project of the same stem+ /// (`fabric.js`) must not read as a silent match of the slug (`fabric`,+ /// which is danielmiessler/fabric). The result names the upstream repo and+ /// the one-line description, and says the names differ. An exact slug, a+ /// header file (`lz4.h`), and a description that already contains the+ /// token are not that case. `candle charts` is the slug plus another word:+ /// not a suffixed project name, so the line itself has to say what the+ /// pack is.+ #[test]+ fn a_same_stem_different_project_is_named_instead_of_matched_silently() {+ let rig = Rig::new();+ let mut fabric = entry(+ "fabric",+ 1,+ LicenceClass::AdaptWithAttribution,+ &["fabric", "prompt"],+ );+ fabric.upstream = "danielmiessler/fabric".into();+ fabric.ask_for =+ "An agent building a reusable prompt library queries this for prompt patterns.".into();+ let cat = catalogue(with_fillers(vec![fabric]));++ // One token, "fabric.js": the sketch sees "fabric" and would otherwise+ // answer from that pack. The stem is a different project, so it must not.+ let found = find("fabric.js", 5, rig.cwd.path(), rig.hub.path(), &cat);+ assert_eq!(found.routing, "different_project", "{found:?}");+ assert_eq!(found.corpora[0].slug, "fabric");+ assert_eq!(found.corpora[0].different_project, "fabric.js");+ assert_eq!(found.corpora[0].upstream, "danielmiessler/fabric");+ let text = render_find(&found);+ assert!(text.contains("danielmiessler/fabric"), "{text}");+ assert!(text.contains("prompt library"), "{text}");+ assert!(+ text.contains("\"fabric.js\" is a different project"),+ "{text}"+ );+ assert!(text.contains("routing: different_project"), "{text}");++ let out = rig.run(+ &args("fabric.js", None),+ &cat,+ None,+ &FakeBackend::new(true),+ );+ assert_eq!(out.exit, EXIT_NEEDS_DECISION, "{out:?}");+ assert!(+ out.answer.is_none(),+ "must not answer the wrong project: {out:?}"+ );+ let all = format!("{}\n{}", out.stdout, out.stderr.join("\n"));+ assert!(all.contains("danielmiessler/fabric"), "{all}");+ assert!(all.contains("fabric.js"), "{all}");+ assert!(all.contains("--corpus"), "{all}");++ let exact = find(+ "how does fabric format a prompt",+ 5,+ rig.cwd.path(),+ rig.hub.path(),+ &cat,+ );+ assert_eq!(exact.routing, "confident", "{exact:?}");+ assert!(exact.corpora[0].different_project.is_empty(), "{exact:?}");+ assert!(render_find(&exact).contains("danielmiessler/fabric"));++ assert!(different_project_token("read lz4.h", "lz4", "lz4/lz4", "block format").is_none());+ assert_eq!(+ different_project_token(+ "fabricjs canvas",+ "fabric",+ "danielmiessler/fabric",+ "prompt library"+ )+ .as_deref(),+ Some("fabricjs")+ );+ assert!(different_project_token(+ "see fabric.js",+ "fabric",+ "danielmiessler/fabric",+ "the fabric.js canvas library"+ )+ .is_none());++ let mut candle = entry(+ "candle",+ 1,+ LicenceClass::AdaptWithAttribution,+ &["candle", "tensor"],+ );+ candle.upstream = "huggingface/candle".into();+ candle.ask_for =+ "A Rust engine adding embeddings queries this for a tensor framework.".into();+ let cat2 = catalogue(with_fillers(vec![candle]));+ let charts = find("candle charts", 5, rig.cwd.path(), rig.hub.path(), &cat2);+ assert_ne!(charts.routing, "different_project", "{charts:?}");+ let line = render_find(&charts);+ assert!(line.contains("huggingface/candle"), "{line}");+ assert!(line.contains("tensor framework"), "{line}");+ }}diff --git a/engine/crates/xerj-hub/src/build_catalogue.rs b/engine/crates/xerj-hub/src/build_catalogue.rsindex a437c76f7..07f70af65 100644--- a/engine/crates/xerj-hub/src/build_catalogue.rs+++ b/engine/crates/xerj-hub/src/build_catalogue.rs@@ -607,6 +607,12 @@ fn record_entries(pack_sha256: pack.sha256,dependency_names: Vec::new(),quality_grade: None,+ upstream: crate::catalogue::upstream_list(+ pack.manifest+ .sources+ .iter()+ .map(|s| (s.url.as_str(), s.repo.as_str())),+ ),term_sketch: crate::sketch::build(terms),});}@@ -874,6 +880,7 @@ mod tests {assert_eq!(e.pack_sha256, hex(&Sha256::digest(&raw)));assert_eq!(e.quality_grade, None);assert!(e.ask_for.contains("block compression"));+ assert_eq!(e.upstream, "x/lz4");for term in ["LZ4_compress_default","lz4_compress_default",diff --git a/engine/crates/xerj-hub/src/catalogue.rs b/engine/crates/xerj-hub/src/catalogue.rsindex fb1bc6900..74fc34d5a 100644--- a/engine/crates/xerj-hub/src/catalogue.rs+++ b/engine/crates/xerj-hub/src/catalogue.rs@@ -31,7 +31,8 @@ use anyhow::{anyhow, bail, Context, Result};use serde::Deserialize;use xerj_autoindex::harvest::sign::{sign_bytes, verify_bytes};use xerj_common::hub::{- hub_pubkey, Catalogue, CatalogueEntry, LicenceClass, CATALOGUE_FORMAT_VERSION,+ catalogue_format_supported, hub_pubkey, Catalogue, CatalogueEntry, LicenceClass,+ CATALOGUE_FORMAT_VERSION,};/// Path of the catalogue under the hub's API root.@@ -108,12 +109,72 @@ struct Manifest {#[derive(Deserialize)]struct ManifestRepo {+ #[serde(default)]+ repo: String,+ #[serde(default)]+ url: String,#[serde(default)]sha: String,#[serde(default)]review: Option<ManifestReview>,}+/// `owner/name` from a git remote, when the URL (or the `repo` field itself)+/// records one. A bare repo name (`fabric`) is not an owner, so it is not+/// returned: the slug already says that, and inventing an owner is worse than+/// leaving the field empty.+pub fn upstream_id(url: &str, repo: &str) -> Option<String> {+ let repo = repo.trim().trim_end_matches(".git");+ if let Some(id) = owner_name(repo) {+ return Some(id);+ }+ let url = url.trim().trim_end_matches('/').trim_end_matches(".git");+ let path = url+ .strip_prefix("https://github.com/")+ .or_else(|| url.strip_prefix("http://github.com/"))+ .or_else(|| url.strip_prefix("git://github.com/"))+ .or_else(|| url.strip_prefix("ssh://[email protected]/"))+ .or_else(|| url.split_once("[email protected]:").map(|(_, p)| p));+ path.and_then(owner_name)+}++/// The first two non-empty path segments, when both look like a GitHub+/// owner and a repo name (no spaces, no scheme).+fn owner_name(path: &str) -> Option<String> {+ let parts: Vec<&str> = path+ .split('/')+ .map(str::trim)+ .filter(|p| !p.is_empty())+ .take(2)+ .collect();+ if parts.len() != 2 {+ return None;+ }+ if parts.iter().any(|p| p.contains(' ') || p.contains(':')) {+ return None;+ }+ // A path that is only `owner/name` — `owner_name` is also called on a+ // longer URL path after the host is stripped, and extra segments (a+ // `/tree/main` suffix) are dropped by `take(2)`.+ Some(format!("{}/{}", parts[0], parts[1]))+}++/// `owner/name` values from `(url, repo)` pairs, first-seen order, at most four.+pub fn upstream_list<'a>(pairs: impl Iterator<Item = (&'a str, &'a str)>) -> String {+ let mut out = Vec::new();+ for (url, repo) in pairs {+ if let Some(id) = upstream_id(url, repo) {+ if !out.contains(&id) {+ out.push(id);+ }+ }+ if out.len() == 4 {+ break;+ }+ }+ out.join(", ")+}+#[derive(Deserialize)]struct ManifestReview {#[serde(default)]@@ -230,12 +291,14 @@ fn build_entry(input: &CorpusBuildInput) -> Result<CatalogueEntry> {.unwrap_or_else(|| input.ask_for.clone());let term_sketch = crate::sketch::build(input.terms.iter().cloned());+ let upstream = upstream_list(m.repos.iter().map(|r| (r.url.as_str(), r.repo.as_str())));Ok(CatalogueEntry {slug: m.corpus,domain: input.domain.clone(),topics: m.topics,ask_for,+ upstream,pin_sha,pin_date,refresh_cadence_days: input.refresh_cadence_days,@@ -390,13 +453,14 @@ fn read_cached(cache_file: &Path) -> Option<Catalogue> {parse_catalogue(&raw).ok()}-/// Parse catalogue bytes and refuse a format version this client does not equal.+/// Parse catalogue bytes and refuse a format version this client does not read.fn parse_catalogue(bytes: &[u8]) -> Result<Catalogue> {let cat: Catalogue = serde_json::from_slice(bytes).context("parse catalogue JSON")?;- if cat.format_version != CATALOGUE_FORMAT_VERSION {+ if !catalogue_format_supported(cat.format_version) {bail!(- "catalogue format_version {} is not the supported {}",+ "catalogue format_version {} is not a supported version ({}..={})",cat.format_version,+ xerj_common::hub::CATALOGUE_FORMAT_VERSION_MIN,CATALOGUE_FORMAT_VERSION);}@@ -835,6 +899,76 @@ mod tests {assert_eq!(e.pin_sha, "1111111111111111111111111111111111111111");}+ #[test]+ fn upstream_is_the_github_owner_and_repo_not_the_slug() {+ let tmp = tempfile::tempdir().unwrap();+ let p = tmp.path().join("fabric.json");+ let json = serde_json::json!({+ "corpus": "fabric",+ "cloned_at": "2026-10-13T00:00:00Z",+ "repos": [{+ "repo": "fabric",+ "url": "https://github.com/danielmiessler/fabric.git",+ "sha": "177b3a9d284895c907d6eb3a64937d174add4cd5",+ "review": {"spdx": "MIT", "use": "adapt-with-attribution", "at": "2026-10-13"}+ }]+ });+ fs::write(&p, serde_json::to_vec(&json).unwrap()).unwrap();+ let input = CorpusBuildInput {+ manifest_path: p,+ domain: "AI".into(),+ refresh_cadence_days: 7,+ pack_bytes: 1,+ pack_url: "u".into(),+ pack_sha256: "e".repeat(64),+ sig_url: "s".into(),+ quality_grade: None,+ terms: vec![],+ pin_sha: String::new(),+ ask_for: "prompt patterns".into(),+ };+ let (seed, _public) = keypair();+ let (cat, _sig) = build_catalogue(&[input], "2026-10-14T00:00:00Z", &seed).unwrap();+ assert_eq!(cat.format_version, CATALOGUE_FORMAT_VERSION);+ assert_eq!(cat.corpora[0].upstream, "danielmiessler/fabric");+ // A non-github URL with no owner/name is left empty, not copied from the slug.+ assert_eq!(upstream_id("https://example.test/fabric", "fabric"), None);+ assert_eq!(+ upstream_id("[email protected]:KhronosGroup/WebGL.git", "WebGL").as_deref(),+ Some("KhronosGroup/WebGL")+ );+ assert_eq!(+ upstream_list(+ [+ ("https://github.com/huggingface/candle.git", "candle"),+ ("https://github.com/huggingface/candle", "candle"),+ ("https://github.com/other/extra", "extra"),+ ]+ .into_iter()+ ),+ "huggingface/candle, other/extra"+ );+ }++ /// Format 1 has no `upstream` key. It still loads. A future version does not.+ #[test]+ fn format_1_without_upstream_loads_and_a_newer_version_is_refused() {+ let tmp = tempfile::tempdir().unwrap();+ let (bytes, _sig, _public) = fixture_bytes(tmp.path());+ let mut v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();+ v["format_version"] = serde_json::json!(1);+ for entry in v["corpora"].as_array_mut().unwrap() {+ entry.as_object_mut().unwrap().remove("upstream");+ }+ let parsed = parse_catalogue(&serde_json::to_vec(&v).unwrap()).unwrap();+ assert_eq!(parsed.format_version, 1);+ assert!(parsed.corpora.iter().all(|e| e.upstream.is_empty()));++ v["format_version"] = serde_json::json!(CATALOGUE_FORMAT_VERSION + 1);+ let err = parse_catalogue(&serde_json::to_vec(&v).unwrap()).unwrap_err();+ assert!(err.to_string().contains("format_version"), "{err:#}");+ }+/// Build a catalogue, return `(bytes, sig, pubkey)` ready to seed a cache.fn fixture_bytes(dir: &Path) -> (Vec<u8>, String, String) {let inputs = two_inputs(dir);diff --git a/engine/crates/xerj-hub/src/cli.rs b/engine/crates/xerj-hub/src/cli.rsindex 1439f55d3..6c4f5ddee 100644--- a/engine/crates/xerj-hub/src/cli.rs+++ b/engine/crates/xerj-hub/src/cli.rs@@ -65,6 +65,11 @@ const HUB_USAGE: &str = "usage: xerj hub <command>this build has no hub public key to verify with (nothing is downloaded),1 for anything else (an excluded licence and a pack over the size limitare refused here too).+ find \"<question>\" [--k N] [--json]+ Which corpora could answer, best first. Each line names the upstream+ repo (owner/name, when the catalogue has it) and the one-line description+ of what that pack is. A query that names a different project of the same+ slug (fabric.js) exits 4 instead of matching silently.feedback <slug> --usefulness <1-5> --found <yes|partial|no> [...]Rate a corpus after using it: one structured review (no query text) forits Agent score on hub.xerj.org (see `xerj hub feedback -h`).@@ -89,6 +94,7 @@ fn hub_verb(arg: &str) -> bool {| "build-catalogue"| "sync"| "pull"+ | "find"| "feedback"| "status"| "stop"@@ -112,6 +118,7 @@ pub fn run_hub_cli(args: &[String]) -> i32 {"build-catalogue" => crate::build_catalogue::run(rest),"sync" => crate::sync::run_sync(rest),"pull" => run_pull(rest),+ "find" => run_find_cmd(rest),"feedback" => crate::feedback::run_feedback(rest),"status" => run_status(rest),"stop" => run_stop(rest),@@ -1316,6 +1323,85 @@ fn no_extra_args(cmd: &str, args: &[String]) -> Option<i32> {}}+/// `xerj hub find "<question>" [--k N] [--json]`. Same decision as+/// [`ask::run_find`] (and the MCP `xerj_hub_find` tool).+fn run_find_cmd(args: &[String]) -> i32 {+ let mut k = ask::FIND_DEFAULT_K;+ let mut as_json = false;+ let mut words: Vec<&str> = Vec::new();+ let mut i = 0;+ while i < args.len() {+ match args[i].as_str() {+ "-h" | "--help" => {+ println!("{HUB_USAGE}");+ return 0;+ }+ "--json" => as_json = true,+ "--k" => {+ i += 1;+ let Some(v) = args.get(i) else {+ eprintln!("xerj hub find: --k needs a value\n\n{HUB_USAGE}");+ return ask::EXIT_USAGE;+ };+ match v.parse::<usize>() {+ Ok(n) if (1..=ask::FIND_MAX_K).contains(&n) => k = n,+ _ => {+ eprintln!(+ "xerj hub find: --k must be an integer from 1 to {}, got '{v}'\n\n{HUB_USAGE}",+ ask::FIND_MAX_K+ );+ return ask::EXIT_USAGE;+ }+ }+ }+ f if f.starts_with('-') => {+ eprintln!("xerj hub find: unknown flag '{f}'\n\n{HUB_USAGE}");+ return ask::EXIT_USAGE;+ }+ word => words.push(word),+ }+ i += 1;+ }+ let question = words.join(" ");+ if question.trim().is_empty() {+ eprintln!("xerj hub find: a question is required\n\n{HUB_USAGE}");+ return ask::EXIT_USAGE;+ }+ match ask::run_find(question.trim(), k) {+ Ok(found) => {+ let text = if as_json {+ match serde_json::to_string(&found) {+ Ok(s) => s,+ Err(e) => {+ eprintln!("xerj hub find: {e}");+ return 1;+ }+ }+ } else {+ ask::render_find(&found)+ };+ if found.corpora.is_empty() {+ eprintln!("{text}");+ } else {+ println!("{text}");+ }+ match found.routing {+ "different_project" | "ambiguous" => ask::EXIT_NEEDS_DECISION,+ "no_corpus" => ask::EXIT_NO_ANSWER,+ _ => 0,+ }+ }+ Err(e) => {+ eprintln!("xerj hub find: {e}");+ if xerj_common::hub::hub_key_configured() {+ 1+ } else {+ ask::EXIT_NO_KEY+ }+ }+ }+}+/// `xerj hub config hosted on|off|auto`.fn run_config(args: &[String]) -> i32 {config_in(&hub_home(), args)@@ -2220,7 +2306,13 @@ mod tests {#[test]fn hub_usage_lists_the_new_commands() {- for word in ["status", "stop", "config hosted", "feedback <slug>"] {+ for word in [+ "status",+ "stop",+ "config hosted",+ "feedback <slug>",+ "find \"<question>\"",+ ] {assert!(HUB_USAGE.contains(word), "{word}");}assert_eq!(run_hub_cli(&strs(&["status", "extra"])), 2);@@ -2261,6 +2353,7 @@ mod tests {domain: "TEST".to_string(),topics: vec![],ask_for: "x".to_string(),+ upstream: String::new(),pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".to_string(),pin_date: "2026-10-02".to_string(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/src/lib.rs b/engine/crates/xerj-hub/src/lib.rsindex 772bf3f78..912662890 100644--- a/engine/crates/xerj-hub/src/lib.rs+++ b/engine/crates/xerj-hub/src/lib.rs@@ -30,10 +30,10 @@ pub mod sync;mod test_support;pub use xerj_common::hub::{- render, Answer, AnswerMode, AnsweredFrom, Catalogue, CatalogueEntry, DefRecord, Hit,- LicenceClass, PackIndex, PackManifest, PackSource, Sketch, AUTO_PULL_MAX_BYTES,- CATALOGUE_FORMAT_VERSION, DEFAULT_HUB_URL, DISK_BUDGET_BYTES, HUB_PUBKEY_HEX,- PACK_FORMAT_VERSION,+ catalogue_format_supported, render, Answer, AnswerMode, AnsweredFrom, Catalogue,+ CatalogueEntry, DefRecord, Hit, LicenceClass, PackIndex, PackManifest, PackSource, Sketch,+ AUTO_PULL_MAX_BYTES, CATALOGUE_FORMAT_VERSION, CATALOGUE_FORMAT_VERSION_MIN, DEFAULT_HUB_URL,+ DISK_BUDGET_BYTES, HUB_PUBKEY_HEX, PACK_FORMAT_VERSION,};#[cfg(test)]diff --git a/engine/crates/xerj-hub/src/lockfile.rs b/engine/crates/xerj-hub/src/lockfile.rsindex cbc1f2c0e..91c6eb4ee 100644--- a/engine/crates/xerj-hub/src/lockfile.rs+++ b/engine/crates/xerj-hub/src/lockfile.rs@@ -651,6 +651,7 @@ mod tests {domain: "NET".to_string(),topics: vec!["compression".to_string()],ask_for: "x".to_string(),+ upstream: String::new(),pin_sha: pin.to_string(),pin_date: "2026-10-02".to_string(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/src/router.rs b/engine/crates/xerj-hub/src/router.rsindex ca9d900f3..efbf6641a 100644--- a/engine/crates/xerj-hub/src/router.rs+++ b/engine/crates/xerj-hub/src/router.rs@@ -875,6 +875,7 @@ mod tests {domain: "TEST".to_string(),topics: Vec::new(),ask_for: String::new(),+ upstream: String::new(),pin_sha: "0".repeat(40),pin_date: "2026-01-01".to_string(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/src/sync/tests.rs b/engine/crates/xerj-hub/src/sync/tests.rsindex a83b22175..7885047bc 100644--- a/engine/crates/xerj-hub/src/sync/tests.rs+++ b/engine/crates/xerj-hub/src/sync/tests.rs@@ -66,6 +66,7 @@ fn entry(slug: &str, pin8: &str, pin_date: &str) -> CatalogueEntry {domain: "NET".to_string(),topics: vec![],ask_for: "x".to_string(),+ upstream: String::new(),pin_sha: full_sha(pin8),pin_date: pin_date.to_string(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/tests/ask_flow.rs b/engine/crates/xerj-hub/tests/ask_flow.rsindex af2de4162..8b752fe45 100644--- a/engine/crates/xerj-hub/tests/ask_flow.rs+++ b/engine/crates/xerj-hub/tests/ask_flow.rs@@ -422,6 +422,7 @@ fn publish(src_archive: &Path, src_sig: &Path, dir: &Path, hub_home: &Path) -> Pdomain: "DEV".into(),topics: vec!["c".into()],ask_for: "a tiny C fixture corpus".into(),+ upstream: String::new(),pin_sha: PIN_SHA.into(),pin_date: PIN_DATE.into(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/tests/identity_routing.rs b/engine/crates/xerj-hub/tests/identity_routing.rsindex ec2c8862a..88fe7be7c 100644--- a/engine/crates/xerj-hub/tests/identity_routing.rs+++ b/engine/crates/xerj-hub/tests/identity_routing.rs@@ -55,6 +55,7 @@ fn catalogue() -> Catalogue {domain: row["domain"].as_str().unwrap_or("").to_string(),topics,ask_for,+ upstream: String::new(),pin_sha: "0".repeat(40),pin_date: "2026-10-21".to_string(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/tests/pack_roundtrip.rs b/engine/crates/xerj-hub/tests/pack_roundtrip.rsindex 23972d706..478ad0798 100644--- a/engine/crates/xerj-hub/tests/pack_roundtrip.rs+++ b/engine/crates/xerj-hub/tests/pack_roundtrip.rs@@ -259,6 +259,7 @@ fn write_catalogue_full(domain: "DEV".into(),topics: vec!["c".into()],ask_for: "a tiny C fixture corpus".into(),+ upstream: String::new(),pin_sha: PIN_SHA.into(),pin_date: PIN_DATE.into(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/tests/routing.rs b/engine/crates/xerj-hub/tests/routing.rsindex ce1e54ec3..5f23ec978 100644--- a/engine/crates/xerj-hub/tests/routing.rs+++ b/engine/crates/xerj-hub/tests/routing.rs@@ -140,6 +140,7 @@ fn make_catalogue() -> Catalogue {domain: "TEST".to_string(),topics: Vec::new(),ask_for: (*description).to_string(),+ upstream: String::new(),pin_sha: "0".repeat(40),pin_date: "2026-01-01".to_string(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-hub/tests/sync_e2e.rs b/engine/crates/xerj-hub/tests/sync_e2e.rsindex 3d3f2eabc..a519d37d0 100644--- a/engine/crates/xerj-hub/tests/sync_e2e.rs+++ b/engine/crates/xerj-hub/tests/sync_e2e.rs@@ -278,6 +278,7 @@ fn catalogue_entry(domain: "DEV".into(),topics: vec!["c".into()],ask_for: "a tiny C fixture corpus".into(),+ upstream: String::new(),pin_sha: pin_sha.into(),pin_date: pin_date.into(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-mcp/src/hub.rs b/engine/crates/xerj-mcp/src/hub.rsindex cdb79d54c..1dde1f928 100644--- a/engine/crates/xerj-mcp/src/hub.rs+++ b/engine/crates/xerj-mcp/src/hub.rs@@ -51,13 +51,15 @@ pub fn specs() -> Vec<Value> {json!({"name": "xerj_hub_find","description": concat!(- "Discover which hub corpora can answer a question. Returns candidate corpora ",- "with their pin, licence and whether the pack is already on this machine — not ",- "passages; ask one with xerj_hub_ask. Routed exactly as `xerj ask` routes: your ",- "project's dependencies, the signed catalogue's term sketches, and the exact ",- "symbol a definition question names (offline: reads the catalogue `xerj hub sync` ",- "cached and never fetches it; this build has no hosted search). ",- "Excluded-licence corpora are never listed."),+ "Discover which hub corpora can answer a question. Each candidate names the ",+ "upstream repo (owner/name) and a one-line description of what that pack is, plus ",+ "its pin, licence and whether the pack is already on this machine — not passages; ",+ "ask one with xerj_hub_ask. A query that names a different project of the same slug ",+ "(fabric.js vs the pack's repo) is reported as a different project instead of a ",+ "silent match. Routed exactly as `xerj ask` routes: your project's dependencies, ",+ "the signed catalogue's term sketches, and the exact symbol a definition question ",+ "names (offline: reads the catalogue `xerj hub sync` cached and never fetches it; ",+ "this build has no hosted search). Excluded-licence corpora are never listed."),"inputSchema": {"type": "object","properties": {diff --git a/engine/crates/xerj-mcp/tests/hub_tools.rs b/engine/crates/xerj-mcp/tests/hub_tools.rsindex 88def1661..a90094a2c 100644--- a/engine/crates/xerj-mcp/tests/hub_tools.rs+++ b/engine/crates/xerj-mcp/tests/hub_tools.rs@@ -26,6 +26,7 @@ fn entry(slug: &str, spdx: &str, class: LicenceClass, terms: &[&str]) -> Catalogdomain: "DEV".into(),topics: vec![],ask_for: format!("fixture corpus {slug}"),+ upstream: String::new(),pin_sha: PIN_SHA.into(),pin_date: PIN_DATE.into(),refresh_cadence_days: 7,diff --git a/engine/crates/xerj-server/src/grpc_hub.rs b/engine/crates/xerj-server/src/grpc_hub.rsindex 205ebb59f..6f8944a6c 100644--- a/engine/crates/xerj-server/src/grpc_hub.rs+++ b/engine/crates/xerj-server/src/grpc_hub.rs@@ -655,6 +655,7 @@ mod tests {domain: "DEV".into(),topics: vec![],ask_for: format!("fixture corpus {slug}"),+ upstream: String::new(),pin_sha: PIN_SHA.into(),pin_date: PIN_DATE.into(),refresh_cadence_days: 7,diff --git a/landing/docs/agents/schemas/mcp-tools.json b/landing/docs/agents/schemas/mcp-tools.jsonindex 7bd3e65cc..70b9f55d8 100644--- a/landing/docs/agents/schemas/mcp-tools.json+++ b/landing/docs/agents/schemas/mcp-tools.json@@ -865,7 +865,7 @@},{"name": "xerj_hub_find",- "description": "Discover which hub corpora can answer a question. Returns candidate corpora with their pin, licence and whether the pack is already on this machine — not passages; ask one with xerj_hub_ask. Routed exactly as `xerj ask` routes: your project's dependencies, the signed catalogue's term sketches, and the exact symbol a definition question names (offline: reads the catalogue `xerj hub sync` cached and never fetches it; this build has no hosted search). Excluded-licence corpora are never listed.",+ "description": "Discover which hub corpora can answer a question. Each candidate names the upstream repo (owner/name) and a one-line description of what that pack is, plus its pin, licence and whether the pack is already on this machine — not passages; ask one with xerj_hub_ask. A query that names a different project of the same slug (fabric.js vs the pack's repo) is reported as a different project instead of a silent match. Routed exactly as `xerj ask` routes: your project's dependencies, the signed catalogue's term sketches, and the exact symbol a definition question names (offline: reads the catalogue `xerj hub sync` cached and never fetches it; this build has no hosted search). Excluded-licence corpora are never listed.","inputSchema": {"type": "object","properties": {