AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(hub): xerj-hub crate + shared Corpus Hub v2 types (step 1)
xerj-orgb86ab203ea80
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 67a239699..20a31b6e3 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -7613,6 +7613,17 @@ dependencies = ["zstd",]+[[package]]+name = "xerj-hub"+version = "1.0.0-rc.95"+dependencies = [+ "anyhow",+ "serde",+ "serde_json",+ "tempfile",+ "xerj-common",+]+[[package]]name = "xerj-logs"version = "1.0.0-rc.95"diff --git a/engine/Cargo.toml b/engine/Cargo.tomlindex 3d73b7d5d..d15f54ec4 100644--- a/engine/Cargo.toml+++ b/engine/Cargo.toml@@ -18,6 +18,7 @@ members = ["crates/xerj-cluster","crates/xerj-console-api","crates/xerj-autoindex",+ "crates/xerj-hub","tests/es-compat-yaml",]diff --git a/engine/crates/xerj-common/src/hub.rs b/engine/crates/xerj-common/src/hub.rsnew file mode 100644index 000000000..2c84d3076--- /dev/null+++ b/engine/crates/xerj-common/src/hub.rs@@ -0,0 +1,509 @@+//! Corpus Hub v2 — the types shared across crates.+//!+//! The hub serves PRE-INDEXED packs an agent pulls and queries, instead of a+//! JSON manifest it must clone and index locally (discussion #1374,+//! `docs/CORPUS_HUB_V2_IMPLEMENTATION.md`). Every type that crosses a crate+//! boundary or the network lives here so `xerj-hub`, `xerj-api`, `xerj-mcp`+//! and the CLI all speak one vocabulary.+//!+//! `xerj-common` depends on no other xerj crate, so this module holds only the+//! data shapes and the one renderer that turns an [`Answer`] into the text every+//! surface prints. The behaviour (download, verify, restore, route) is in+//! `xerj-hub`.+//!+//! Anything deserialized from the network carries `#[serde(deny_unknown_fields)]`+//! so a catalogue or a pack manifest from a future format version is refused+//! rather than half-read.++use serde::{Deserialize, Serialize};+use std::fmt::Write as _;++/// Bump when the on-the-wire catalogue shape changes. The client refuses a+/// catalogue whose `format_version` it does not equal.+pub const CATALOGUE_FORMAT_VERSION: u32 = 1;++/// Bump when the pack archive layout changes. `format_version: 1` means the+/// snapshot layout (engine `create_snapshot`/`restore_snapshot`).+pub const PACK_FORMAT_VERSION: u32 = 1;++/// Where the catalogue and the hosted query API live.+pub const DEFAULT_HUB_URL: &str = "https://hub.xerj.org";++/// Packs at or below this size are pulled automatically; larger ones are+/// answered through the hosted route until the user runs `xerj hub pull`.+pub const AUTO_PULL_MAX_BYTES: u64 = 50 * 1024 * 1024;++/// The local pack cache is pruned (least-recently-used) to stay under this.+pub const DISK_BUDGET_BYTES: u64 = 4 * 1024 * 1024 * 1024;++/// Hex of the ed25519 public key the hub signs catalogues and packs with.+/// Filled when the hub signing key is generated (Step 2 / CI). Overridable by+/// the `XERJ_HUB_PUBKEY` environment variable for tests only.+pub const HUB_PUBKEY_HEX: &str = "";++/// What a per-source licence review allows, straight from the registry's+/// `review.use`. Governs whether a pack is pulled silently, with a warning, or+/// never.+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]+#[serde(rename_all = "kebab-case")]+pub enum LicenceClass {+ /// Public-domain, OGL, permissive — pull silently.+ AdaptWithAttribution,+ /// GPL/AGPL/SSPL/BUSL and the like — pull with a "read, do not copy" warning.+ ApproachOnly,+ /// Closed or non-redistributable — never pulled.+ Excluded,+}++/// One corpus as the catalogue lists it. The client picks a corpus from this+/// without downloading anything.+#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct CatalogueEntry {+ pub slug: String,+ /// SEC, DB, NET, CRYPTO, WEB, API, OPS, LAW, HEALTH, AI, WRITE.+ pub domain: String,+ pub topics: Vec<String>,+ /// The registry's one-line "an agent working on ___ queries this for ___".+ pub ask_for: String,+ /// Full 40-hex commit the pack was built at.+ pub pin_sha: String,+ /// YYYY-MM-DD the pin was taken.+ pub pin_date: String,+ /// How often this corpus is re-pinned (7 for code, 1 for feeds).+ pub refresh_cadence_days: u32,+ pub licence_spdx: String,+ pub licence_class: LicenceClass,+ pub pack_bytes: u64,+ pub pack_url: String,+ pub pack_sha256: String,+ pub sig_url: String,+ /// crate / npm / go-module / pypi names this corpus answers for; used by the+ /// project-dependency router. Empty when none are recorded — never guessed.+ pub dependency_names: Vec<String>,+ /// e.g. "G7 4/5", or None. Never invented.+ pub quality_grade: Option<String>,+ pub term_sketch: Sketch,+}++/// A Bloom filter over a corpus's identifiers and vocabulary, used to route a+/// question to candidate corpora offline. See `xerj-hub`'s `sketch` module for+/// the hashing.+#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct Sketch {+ pub m_bits: u32,+ pub k_hashes: u32,+ pub bits_base64: String,+}++/// The whole catalogue: one small signed file, refreshed in the background.+#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct Catalogue {+ pub format_version: u32,+ /// RFC 3339.+ pub generated_at: String,+ pub corpora: Vec<CatalogueEntry>,+}++/// `pack.json` inside a pack archive.+#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct PackManifest {+ pub format_version: u32,+ pub slug: String,+ pub pin_sha: String,+ pub pin_date: String,+ /// RFC 3339 the pack was built.+ pub built_at: String,+ /// `xerj --version` of the builder.+ pub builder_version: String,+ /// `xc-<slug>-b<build>`.+ pub index_prefix: String,+ /// Every index in the snapshot, the docs index and the defs index included.+ pub indices: Vec<PackIndex>,+ /// The name used with `create_snapshot`.+ pub snapshot_name: String,+ pub licence_spdx: String,+ pub licence_class: LicenceClass,+ /// Copied from the manifest's `repos[]`.+ pub sources: Vec<PackSource>,+ pub defs_count: u64,+ pub files_count: u64,+}++#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct PackIndex {+ pub name: String,+ pub docs: u64,+ pub bytes: u64,+}++#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct PackSource {+ pub repo: String,+ pub url: String,+ pub sha: String,+ pub licence: String,+}++/// One line of `defs.jsonl` and one document of the `-defs` index: a single+/// definition, with the literal value for constants and macros.+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]+#[serde(deny_unknown_fields)]+pub struct DefRecord {+ pub symbol: String,+ /// As `extract/code.rs` names it: function, const, define, struct, …+ pub kind: String,+ pub lang: String,+ /// Path relative to the corpus root, forward slashes.+ pub file: String,+ /// 1-based line the NAME sits on.+ pub line: u32,+ /// The declaration, signature only, at most 400 chars.+ pub text: String,+ /// The literal value for constants/macros/defines; None otherwise.+ pub value: Option<String>,+}++/// Where an answer came from.+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]+#[serde(rename_all = "snake_case")]+pub enum AnsweredFrom {+ /// A pack already on this machine.+ LocalPack,+ /// The hub's hosted query node.+ Hosted,+ /// A corpus indexed locally the old way (not a hub pack).+ LocalIndex,+}++/// Which shape of question this answer is.+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]+#[serde(rename_all = "snake_case")]+pub enum AnswerMode {+ /// One definition, with its value.+ Def,+ /// Ranked passages for a how-does-it-work question.+ How,+ /// The file list with per-path licence.+ Files,+}++/// One retrieved definition or passage.+#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct Hit {+ pub file: String,+ pub line: u32,+ pub kind: String,+ pub text: String,+ pub value: Option<String>,+ pub score: f32,+ /// Set for an approach-only source, naming the licence.+ pub licence_warning: Option<String>,+}++/// What every surface returns: the CLI (as `--json`), the MCP tools and the+/// hosted HTTP endpoints all build this and render it with [`render`].+#[derive(Serialize, Deserialize, Clone, Debug)]+#[serde(deny_unknown_fields)]+pub struct Answer {+ pub corpus: String,+ pub pin_sha: String,+ pub pin_date: String,+ pub age_days: u32,+ pub licence_spdx: String,+ pub licence_class: LicenceClass,+ pub answered_from: AnsweredFrom,+ pub mode: AnswerMode,+ pub hits: Vec<Hit>,+ pub truncated: bool,+ /// One line each, e.g. "pin is 23 days old; the question asks for the current source".+ pub warnings: Vec<String>,+}++impl AnsweredFrom {+ /// The words printed on the `answered from:` line.+ pub fn label(self) -> &'static str {+ match self {+ AnsweredFrom::LocalPack => "local pack",+ AnsweredFrom::Hosted => "hosted hub",+ AnsweredFrom::LocalIndex => "local index",+ }+ }+}++/// Render an [`Answer`] to the text every surface prints. The format is fixed by+/// `docs/CORPUS_HUB_V2_IMPLEMENTATION.md` §1:+///+/// ```text+/// <corpus> pin <pin_date> (<age_days> day[s] old) <licence_spdx> <file>:<line>+/// <text>+/// [value: <value>] # only in def mode when value is Some+/// answered from: <local pack|hosted hub|local index>+/// [warning: …] # one line per warning+/// ```+///+/// For several hits the first lines repeat per hit; the `answered from:` line is+/// printed once at the end. With no hits a single line states the corpus and pin.+pub fn render(answer: &Answer) -> String {+ let age = if answer.age_days == 1 {+ "1 day old".to_string()+ } else {+ format!("{} days old", answer.age_days)+ };+ let header_prefix = format!(+ "{} pin {} ({}) {}",+ answer.corpus, answer.pin_date, age, answer.licence_spdx+ );++ let mut out = String::new();+ if answer.hits.is_empty() {+ let _ = write!(+ out,+ "{header_prefix}\nno match in {} (pin {})",+ answer.corpus, answer.pin_date+ );+ } else {+ for hit in &answer.hits {+ let _ = write!(+ out,+ "{header_prefix} {}:{}\n{}",+ hit.file, hit.line, hit.text+ );+ if answer.mode == AnswerMode::Def {+ if let Some(v) = &hit.value {+ let _ = write!(out, "\nvalue: {v}");+ }+ }+ if let Some(w) = &hit.licence_warning {+ let _ = write!(out, "\n {w}");+ }+ out.push('\n');+ }+ out.pop(); // drop the trailing newline before the footer+ }++ let _ = write!(out, "\nanswered from: {}", answer.answered_from.label());+ for w in &answer.warnings {+ let _ = write!(out, "\nwarning: {w}");+ }+ out+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn licence_class_is_kebab_case() {+ assert_eq!(+ serde_json::to_value(LicenceClass::AdaptWithAttribution).unwrap(),+ json!("adapt-with-attribution")+ );+ assert_eq!(+ serde_json::to_value(LicenceClass::ApproachOnly).unwrap(),+ json!("approach-only")+ );+ assert_eq!(+ serde_json::to_value(LicenceClass::Excluded).unwrap(),+ json!("excluded")+ );+ let c: LicenceClass = serde_json::from_value(json!("approach-only")).unwrap();+ assert_eq!(c, LicenceClass::ApproachOnly);+ }++ #[test]+ fn answered_from_and_mode_are_snake_case() {+ assert_eq!(+ serde_json::to_value(AnsweredFrom::LocalPack).unwrap(),+ json!("local_pack")+ );+ assert_eq!(+ serde_json::to_value(AnsweredFrom::Hosted).unwrap(),+ json!("hosted")+ );+ assert_eq!(+ serde_json::to_value(AnsweredFrom::LocalIndex).unwrap(),+ json!("local_index")+ );+ assert_eq!(serde_json::to_value(AnswerMode::Def).unwrap(), json!("def"));+ assert_eq!(serde_json::to_value(AnswerMode::How).unwrap(), json!("how"));+ assert_eq!(+ serde_json::to_value(AnswerMode::Files).unwrap(),+ json!("files")+ );+ }++ fn sample_entry() -> CatalogueEntry {+ CatalogueEntry {+ slug: "lz4".into(),+ domain: "NET".into(),+ topics: vec!["compression".into()],+ ask_for: "fast block compression".into(),+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".into(),+ pin_date: "2026-10-02".into(),+ refresh_cadence_days: 7,+ licence_spdx: "BSD-2-Clause".into(),+ licence_class: LicenceClass::AdaptWithAttribution,+ pack_bytes: 2_000_000,+ pack_url: "https://example/lz4.hubpack.tar.zst".into(),+ pack_sha256: "deadbeef".into(),+ sig_url: "https://example/lz4.hubpack.tar.zst.sig".into(),+ dependency_names: vec!["lz4".into()],+ quality_grade: None,+ term_sketch: Sketch {+ m_bits: 64,+ k_hashes: 7,+ bits_base64: "AAAAAAAAAAA=".into(),+ },+ }+ }++ #[test]+ fn catalogue_round_trips() {+ let cat = Catalogue {+ format_version: CATALOGUE_FORMAT_VERSION,+ generated_at: "2026-10-14T00:00:00Z".into(),+ corpora: vec![sample_entry()],+ };+ let s = serde_json::to_string(&cat).unwrap();+ let back: Catalogue = serde_json::from_str(&s).unwrap();+ assert_eq!(back.corpora.len(), 1);+ assert_eq!(back.corpora[0].slug, "lz4");+ assert_eq!(+ back.corpora[0].licence_class,+ LicenceClass::AdaptWithAttribution+ );+ }++ #[test]+ fn unknown_field_is_refused() {+ // A catalogue entry with a field this version does not know is rejected,+ // not half-read.+ let mut v = serde_json::to_value(sample_entry()).unwrap();+ v["surprise"] = json!(1);+ let r: Result<CatalogueEntry, _> = serde_json::from_value(v);+ assert!(+ r.is_err(),+ "deny_unknown_fields should reject an extra field"+ );+ }++ #[test]+ fn def_record_round_trips_with_and_without_value() {+ let with = DefRecord {+ symbol: "LZ4_VERSION_RELEASE".into(),+ kind: "define".into(),+ lang: "c".into(),+ file: "lib/lz4.h".into(),+ line: 133,+ text: "#define LZ4_VERSION_RELEASE 0".into(),+ value: Some("0".into()),+ };+ let back: DefRecord = serde_json::from_str(&serde_json::to_string(&with).unwrap()).unwrap();+ assert_eq!(back, with);+ let without = DefRecord {+ value: None,+ ..with+ };+ let back: DefRecord =+ serde_json::from_str(&serde_json::to_string(&without).unwrap()).unwrap();+ assert_eq!(back.value, None);+ }++ fn answer_with(+ mode: AnswerMode,+ hits: Vec<Hit>,+ from: AnsweredFrom,+ warnings: Vec<String>,+ ) -> Answer {+ Answer {+ corpus: "lz4".into(),+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".into(),+ pin_date: "2026-10-02".into(),+ age_days: 1,+ licence_spdx: "BSD-2-Clause".into(),+ licence_class: LicenceClass::AdaptWithAttribution,+ answered_from: from,+ mode,+ hits,+ truncated: false,+ warnings,+ }+ }++ #[test]+ fn render_one_def_hit() {+ let a = answer_with(+ AnswerMode::Def,+ vec![Hit {+ file: "lib/lz4.h".into(),+ line: 133,+ kind: "define".into(),+ text: "#define LZ4_VERSION_RELEASE 0".into(),+ value: Some("0".into()),+ score: 14.5,+ licence_warning: None,+ }],+ AnsweredFrom::LocalPack,+ vec![],+ );+ let expected = "lz4 pin 2026-10-02 (1 day old) BSD-2-Clause lib/lz4.h:133\n\+ #define LZ4_VERSION_RELEASE 0\n\+ value: 0\n\+ answered from: local pack";+ assert_eq!(render(&a), expected);+ }++ #[test]+ fn render_three_hits_how_mode_with_warning() {+ let hit = |f: &str, l: u32| Hit {+ file: f.into(),+ line: l,+ kind: "function".into(),+ text: format!("fn at {f}"),+ value: None,+ score: 1.0,+ licence_warning: None,+ };+ let a = answer_with(+ AnswerMode::How,+ vec![hit("a.rs", 1), hit("b.rs", 2), hit("c.rs", 3)],+ AnsweredFrom::Hosted,+ vec!["pin is 23 days old; the question asks for the current source".into()],+ );+ let expected = "lz4 pin 2026-10-02 (1 day old) BSD-2-Clause a.rs:1\n\+ fn at a.rs\n\+ lz4 pin 2026-10-02 (1 day old) BSD-2-Clause b.rs:2\n\+ fn at b.rs\n\+ lz4 pin 2026-10-02 (1 day old) BSD-2-Clause c.rs:3\n\+ fn at c.rs\n\+ answered from: hosted hub\n\+ warning: pin is 23 days old; the question asks for the current source";+ assert_eq!(render(&a), expected);+ }++ #[test]+ fn render_no_hits() {+ let a = answer_with(AnswerMode::Def, vec![], AnsweredFrom::LocalPack, vec![]);+ let expected = "lz4 pin 2026-10-02 (1 day old) BSD-2-Clause\n\+ no match in lz4 (pin 2026-10-02)\n\+ answered from: local pack";+ assert_eq!(render(&a), expected);+ }++ #[test]+ fn age_days_pluralization() {+ let a = answer_with(AnswerMode::Def, vec![], AnsweredFrom::LocalPack, vec![]);+ assert!(render(&a).contains("(1 day old)"));+ let mut a2 = a;+ a2.age_days = 5;+ assert!(render(&a2).contains("(5 days old)"));+ }+}diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rsindex d28e176b8..741c09415 100644--- a/engine/crates/xerj-common/src/lib.rs+++ b/engine/crates/xerj-common/src/lib.rs@@ -29,6 +29,8 @@//! same-machine CLI/MCP commands (`xerj mcp`, `xerj autoindex`, `xerj init`)//! - [`xccode`] — the reference-coding semantics shared by `xerj code`,//! `xerj corpus`, and the `xerj_code_search` MCP tool (issue #977)+//! - [`hub`] — Corpus Hub v2 shared types: the catalogue, pack manifest,+//! definition record, and the [`hub::Answer`] every surface renders (#1374)pub mod calibration;pub mod config;@@ -36,6 +38,7 @@ pub mod error;pub mod feedback;pub mod field_coercion;pub mod fsio;+pub mod hub;pub mod localauth;pub mod metrics;pub mod net;diff --git a/engine/crates/xerj-hub/Cargo.toml b/engine/crates/xerj-hub/Cargo.tomlnew file mode 100644index 000000000..d239db94a--- /dev/null+++ b/engine/crates/xerj-hub/Cargo.toml@@ -0,0 +1,16 @@+[package]+name = "xerj-hub"+version.workspace = true+edition.workspace = true+license.workspace = true+authors.workspace = true+description = "Corpus Hub v2 client: pull pre-indexed packs, route a question to a corpus, answer locally or through the hosted hub"++[dependencies]+xerj-common = { path = "../xerj-common" }+serde.workspace = true+serde_json = { workspace = true, features = ["preserve_order"] }+anyhow.workspace = true++[dev-dependencies]+tempfile.workspace = truediff --git a/engine/crates/xerj-hub/src/lib.rs b/engine/crates/xerj-hub/src/lib.rsnew file mode 100644index 000000000..476f32a84--- /dev/null+++ b/engine/crates/xerj-hub/src/lib.rs@@ -0,0 +1,35 @@+//! Corpus Hub v2 client (discussion #1374, `docs/CORPUS_HUB_V2_IMPLEMENTATION.md`).+//!+//! The hub serves PRE-INDEXED packs. This crate pulls one, restores it into a+//! local node with no re-indexing, routes a question to the right corpus, and+//! answers from the local pack when it is cached or through the hosted hub when+//! it is not — so an agent gets an answer in one step instead of the nine the v1+//! clone-and-index path needed.+//!+//! The data shapes that cross crates live in [`xerj_common::hub`]; this crate+//! re-exports them so callers write `xerj_hub::Answer`, not the full path.+//!+//! Modules are added step by step per the implementation plan:+//! pack build/pull (Step 2), definitions (Step 3), catalogue + sketch (Step 4),+//! router + ask (Step 5). This file carries only the shared re-exports until+//! those land.++pub use xerj_common::hub::{+ render, Answer, AnswerMode, AnsweredFrom, Catalogue, CatalogueEntry, DefRecord, Hit,+ LicenceClass, PackIndex, PackManifest, PackSource, Sketch, AUTO_PULL_MAX_BYTES,+ CATALOGUE_FORMAT_VERSION, DEFAULT_HUB_URL, DISK_BUDGET_BYTES, HUB_PUBKEY_HEX,+ PACK_FORMAT_VERSION,+};++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn reexports_resolve() {+ // The crate compiles and the shared types are reachable under xerj_hub::.+ assert_eq!(PACK_FORMAT_VERSION, 1);+ assert_eq!(AnsweredFrom::LocalPack.label(), "local pack");+ let _ = LicenceClass::AdaptWithAttribution;+ }+}