Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(hub): xerj-hub crate + shared Corpus Hub v2 types (step 1)

xerj-orgb86ab203ea80
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 67a239699..20a31b6e3 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7613,6 +7613,17 @@ dependencies = [
"zstd",
]
+[[package]]
+name = "xerj-hub"
+version = "1.0.0-rc.95"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "xerj-common",
+]
+
[[package]]
name = "xerj-logs"
version = "1.0.0-rc.95"
diff --git a/engine/Cargo.toml b/engine/Cargo.toml
index 3d73b7d5d..d15f54ec4 100644
--- a/engine/Cargo.toml
+++ b/engine/Cargo.toml
@@ -18,6 +18,7 @@ members = [
"crates/xerj-cluster",
"crates/xerj-console-api",
"crates/xerj-autoindex",
+ "crates/xerj-hub",
"tests/es-compat-yaml",
]
diff --git a/engine/crates/xerj-common/src/hub.rs b/engine/crates/xerj-common/src/hub.rs
new file mode 100644
index 000000000..2c84d3076
--- /dev/null
+++ b/engine/crates/xerj-common/src/hub.rs
@@ -0,0 +1,509 @@
+//! Corpus Hub v2 — the types shared across crates.
+//!
+//! The hub serves PRE-INDEXED packs an agent pulls and queries, instead of a
+//! JSON manifest it must clone and index locally (discussion #1374,
+//! `docs/CORPUS_HUB_V2_IMPLEMENTATION.md`). Every type that crosses a crate
+//! boundary or the network lives here so `xerj-hub`, `xerj-api`, `xerj-mcp`
+//! and the CLI all speak one vocabulary.
+//!
+//! `xerj-common` depends on no other xerj crate, so this module holds only the
+//! data shapes and the one renderer that turns an [`Answer`] into the text every
+//! surface prints. The behaviour (download, verify, restore, route) is in
+//! `xerj-hub`.
+//!
+//! Anything deserialized from the network carries `#[serde(deny_unknown_fields)]`
+//! so a catalogue or a pack manifest from a future format version is refused
+//! rather than half-read.
+
+use serde::{Deserialize, Serialize};
+use std::fmt::Write as _;
+
+/// Bump when the on-the-wire catalogue shape changes. The client refuses a
+/// catalogue whose `format_version` it does not equal.
+pub const CATALOGUE_FORMAT_VERSION: u32 = 1;
+
+/// Bump when the pack archive layout changes. `format_version: 1` means the
+/// snapshot layout (engine `create_snapshot`/`restore_snapshot`).
+pub const PACK_FORMAT_VERSION: u32 = 1;
+
+/// Where the catalogue and the hosted query API live.
+pub const DEFAULT_HUB_URL: &str = "https://hub.xerj.org";
+
+/// Packs at or below this size are pulled automatically; larger ones are
+/// answered through the hosted route until the user runs `xerj hub pull`.
+pub const AUTO_PULL_MAX_BYTES: u64 = 50 * 1024 * 1024;
+
+/// The local pack cache is pruned (least-recently-used) to stay under this.
+pub const DISK_BUDGET_BYTES: u64 = 4 * 1024 * 1024 * 1024;
+
+/// Hex of the ed25519 public key the hub signs catalogues and packs with.
+/// Filled when the hub signing key is generated (Step 2 / CI). Overridable by
+/// the `XERJ_HUB_PUBKEY` environment variable for tests only.
+pub const HUB_PUBKEY_HEX: &str = "";
+
+/// What a per-source licence review allows, straight from the registry's
+/// `review.use`. Governs whether a pack is pulled silently, with a warning, or
+/// never.
+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub enum LicenceClass {
+ /// Public-domain, OGL, permissive — pull silently.
+ AdaptWithAttribution,
+ /// GPL/AGPL/SSPL/BUSL and the like — pull with a "read, do not copy" warning.
+ ApproachOnly,
+ /// Closed or non-redistributable — never pulled.
+ Excluded,
+}
+
+/// One corpus as the catalogue lists it. The client picks a corpus from this
+/// without downloading anything.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct CatalogueEntry {
+ pub slug: String,
+ /// SEC, DB, NET, CRYPTO, WEB, API, OPS, LAW, HEALTH, AI, WRITE.
+ pub domain: String,
+ pub topics: Vec<String>,
+ /// The registry's one-line "an agent working on ___ queries this for ___".
+ pub ask_for: String,
+ /// Full 40-hex commit the pack was built at.
+ pub pin_sha: String,
+ /// YYYY-MM-DD the pin was taken.
+ pub pin_date: String,
+ /// How often this corpus is re-pinned (7 for code, 1 for feeds).
+ pub refresh_cadence_days: u32,
+ pub licence_spdx: String,
+ pub licence_class: LicenceClass,
+ pub pack_bytes: u64,
+ pub pack_url: String,
+ pub pack_sha256: String,
+ pub sig_url: String,
+ /// crate / npm / go-module / pypi names this corpus answers for; used by the
+ /// project-dependency router. Empty when none are recorded — never guessed.
+ pub dependency_names: Vec<String>,
+ /// e.g. "G7 4/5", or None. Never invented.
+ pub quality_grade: Option<String>,
+ pub term_sketch: Sketch,
+}
+
+/// A Bloom filter over a corpus's identifiers and vocabulary, used to route a
+/// question to candidate corpora offline. See `xerj-hub`'s `sketch` module for
+/// the hashing.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct Sketch {
+ pub m_bits: u32,
+ pub k_hashes: u32,
+ pub bits_base64: String,
+}
+
+/// The whole catalogue: one small signed file, refreshed in the background.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct Catalogue {
+ pub format_version: u32,
+ /// RFC 3339.
+ pub generated_at: String,
+ pub corpora: Vec<CatalogueEntry>,
+}
+
+/// `pack.json` inside a pack archive.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct PackManifest {
+ pub format_version: u32,
+ pub slug: String,
+ pub pin_sha: String,
+ pub pin_date: String,
+ /// RFC 3339 the pack was built.
+ pub built_at: String,
+ /// `xerj --version` of the builder.
+ pub builder_version: String,
+ /// `xc-<slug>-b<build>`.
+ pub index_prefix: String,
+ /// Every index in the snapshot, the docs index and the defs index included.
+ pub indices: Vec<PackIndex>,
+ /// The name used with `create_snapshot`.
+ pub snapshot_name: String,
+ pub licence_spdx: String,
+ pub licence_class: LicenceClass,
+ /// Copied from the manifest's `repos[]`.
+ pub sources: Vec<PackSource>,
+ pub defs_count: u64,
+ pub files_count: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct PackIndex {
+ pub name: String,
+ pub docs: u64,
+ pub bytes: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct PackSource {
+ pub repo: String,
+ pub url: String,
+ pub sha: String,
+ pub licence: String,
+}
+
+/// One line of `defs.jsonl` and one document of the `-defs` index: a single
+/// definition, with the literal value for constants and macros.
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct DefRecord {
+ pub symbol: String,
+ /// As `extract/code.rs` names it: function, const, define, struct, …
+ pub kind: String,
+ pub lang: String,
+ /// Path relative to the corpus root, forward slashes.
+ pub file: String,
+ /// 1-based line the NAME sits on.
+ pub line: u32,
+ /// The declaration, signature only, at most 400 chars.
+ pub text: String,
+ /// The literal value for constants/macros/defines; None otherwise.
+ pub value: Option<String>,
+}
+
+/// Where an answer came from.
+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum AnsweredFrom {
+ /// A pack already on this machine.
+ LocalPack,
+ /// The hub's hosted query node.
+ Hosted,
+ /// A corpus indexed locally the old way (not a hub pack).
+ LocalIndex,
+}
+
+/// Which shape of question this answer is.
+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum AnswerMode {
+ /// One definition, with its value.
+ Def,
+ /// Ranked passages for a how-does-it-work question.
+ How,
+ /// The file list with per-path licence.
+ Files,
+}
+
+/// One retrieved definition or passage.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct Hit {
+ pub file: String,
+ pub line: u32,
+ pub kind: String,
+ pub text: String,
+ pub value: Option<String>,
+ pub score: f32,
+ /// Set for an approach-only source, naming the licence.
+ pub licence_warning: Option<String>,
+}
+
+/// What every surface returns: the CLI (as `--json`), the MCP tools and the
+/// hosted HTTP endpoints all build this and render it with [`render`].
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct Answer {
+ pub corpus: String,
+ pub pin_sha: String,
+ pub pin_date: String,
+ pub age_days: u32,
+ pub licence_spdx: String,
+ pub licence_class: LicenceClass,
+ pub answered_from: AnsweredFrom,
+ pub mode: AnswerMode,
+ pub hits: Vec<Hit>,
+ pub truncated: bool,
+ /// One line each, e.g. "pin is 23 days old; the question asks for the current source".
+ pub warnings: Vec<String>,
+}
+
+impl AnsweredFrom {
+ /// The words printed on the `answered from:` line.
+ pub fn label(self) -> &'static str {
+ match self {
+ AnsweredFrom::LocalPack => "local pack",
+ AnsweredFrom::Hosted => "hosted hub",
+ AnsweredFrom::LocalIndex => "local index",
+ }
+ }
+}
+
+/// Render an [`Answer`] to the text every surface prints. The format is fixed by
+/// `docs/CORPUS_HUB_V2_IMPLEMENTATION.md` §1:
+///
+/// ```text
+/// <corpus> pin <pin_date> (<age_days> day[s] old) <licence_spdx> <file>:<line>
+/// <text>
+/// [value: <value>] # only in def mode when value is Some
+/// answered from: <local pack|hosted hub|local index>
+/// [warning: …] # one line per warning
+/// ```
+///
+/// For several hits the first lines repeat per hit; the `answered from:` line is
+/// printed once at the end. With no hits a single line states the corpus and pin.
+pub fn render(answer: &Answer) -> String {
+ let age = if answer.age_days == 1 {
+ "1 day old".to_string()
+ } else {
+ format!("{} days old", answer.age_days)
+ };
+ let header_prefix = format!(
+ "{} pin {} ({}) {}",
+ answer.corpus, answer.pin_date, age, answer.licence_spdx
+ );
+
+ let mut out = String::new();
+ if answer.hits.is_empty() {
+ let _ = write!(
+ out,
+ "{header_prefix}\nno match in {} (pin {})",
+ answer.corpus, answer.pin_date
+ );
+ } else {
+ for hit in &answer.hits {
+ let _ = write!(
+ out,
+ "{header_prefix} {}:{}\n{}",
+ hit.file, hit.line, hit.text
+ );
+ if answer.mode == AnswerMode::Def {
+ if let Some(v) = &hit.value {
+ let _ = write!(out, "\nvalue: {v}");
+ }
+ }
+ if let Some(w) = &hit.licence_warning {
+ let _ = write!(out, "\n {w}");
+ }
+ out.push('\n');
+ }
+ out.pop(); // drop the trailing newline before the footer
+ }
+
+ let _ = write!(out, "\nanswered from: {}", answer.answered_from.label());
+ for w in &answer.warnings {
+ let _ = write!(out, "\nwarning: {w}");
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn licence_class_is_kebab_case() {
+ assert_eq!(
+ serde_json::to_value(LicenceClass::AdaptWithAttribution).unwrap(),
+ json!("adapt-with-attribution")
+ );
+ assert_eq!(
+ serde_json::to_value(LicenceClass::ApproachOnly).unwrap(),
+ json!("approach-only")
+ );
+ assert_eq!(
+ serde_json::to_value(LicenceClass::Excluded).unwrap(),
+ json!("excluded")
+ );
+ let c: LicenceClass = serde_json::from_value(json!("approach-only")).unwrap();
+ assert_eq!(c, LicenceClass::ApproachOnly);
+ }
+
+ #[test]
+ fn answered_from_and_mode_are_snake_case() {
+ assert_eq!(
+ serde_json::to_value(AnsweredFrom::LocalPack).unwrap(),
+ json!("local_pack")
+ );
+ assert_eq!(
+ serde_json::to_value(AnsweredFrom::Hosted).unwrap(),
+ json!("hosted")
+ );
+ assert_eq!(
+ serde_json::to_value(AnsweredFrom::LocalIndex).unwrap(),
+ json!("local_index")
+ );
+ assert_eq!(serde_json::to_value(AnswerMode::Def).unwrap(), json!("def"));
+ assert_eq!(serde_json::to_value(AnswerMode::How).unwrap(), json!("how"));
+ assert_eq!(
+ serde_json::to_value(AnswerMode::Files).unwrap(),
+ json!("files")
+ );
+ }
+
+ fn sample_entry() -> CatalogueEntry {
+ CatalogueEntry {
+ slug: "lz4".into(),
+ domain: "NET".into(),
+ topics: vec!["compression".into()],
+ ask_for: "fast block compression".into(),
+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".into(),
+ pin_date: "2026-10-02".into(),
+ refresh_cadence_days: 7,
+ licence_spdx: "BSD-2-Clause".into(),
+ licence_class: LicenceClass::AdaptWithAttribution,
+ pack_bytes: 2_000_000,
+ pack_url: "https://example/lz4.hubpack.tar.zst".into(),
+ pack_sha256: "deadbeef".into(),
+ sig_url: "https://example/lz4.hubpack.tar.zst.sig".into(),
+ dependency_names: vec!["lz4".into()],
+ quality_grade: None,
+ term_sketch: Sketch {
+ m_bits: 64,
+ k_hashes: 7,
+ bits_base64: "AAAAAAAAAAA=".into(),
+ },
+ }
+ }
+
+ #[test]
+ fn catalogue_round_trips() {
+ let cat = Catalogue {
+ format_version: CATALOGUE_FORMAT_VERSION,
+ generated_at: "2026-10-14T00:00:00Z".into(),
+ corpora: vec![sample_entry()],
+ };
+ let s = serde_json::to_string(&cat).unwrap();
+ let back: Catalogue = serde_json::from_str(&s).unwrap();
+ assert_eq!(back.corpora.len(), 1);
+ assert_eq!(back.corpora[0].slug, "lz4");
+ assert_eq!(
+ back.corpora[0].licence_class,
+ LicenceClass::AdaptWithAttribution
+ );
+ }
+
+ #[test]
+ fn unknown_field_is_refused() {
+ // A catalogue entry with a field this version does not know is rejected,
+ // not half-read.
+ let mut v = serde_json::to_value(sample_entry()).unwrap();
+ v["surprise"] = json!(1);
+ let r: Result<CatalogueEntry, _> = serde_json::from_value(v);
+ assert!(
+ r.is_err(),
+ "deny_unknown_fields should reject an extra field"
+ );
+ }
+
+ #[test]
+ fn def_record_round_trips_with_and_without_value() {
+ let with = DefRecord {
+ symbol: "LZ4_VERSION_RELEASE".into(),
+ kind: "define".into(),
+ lang: "c".into(),
+ file: "lib/lz4.h".into(),
+ line: 133,
+ text: "#define LZ4_VERSION_RELEASE 0".into(),
+ value: Some("0".into()),
+ };
+ let back: DefRecord = serde_json::from_str(&serde_json::to_string(&with).unwrap()).unwrap();
+ assert_eq!(back, with);
+ let without = DefRecord {
+ value: None,
+ ..with
+ };
+ let back: DefRecord =
+ serde_json::from_str(&serde_json::to_string(&without).unwrap()).unwrap();
+ assert_eq!(back.value, None);
+ }
+
+ fn answer_with(
+ mode: AnswerMode,
+ hits: Vec<Hit>,
+ from: AnsweredFrom,
+ warnings: Vec<String>,
+ ) -> Answer {
+ Answer {
+ corpus: "lz4".into(),
+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".into(),
+ pin_date: "2026-10-02".into(),
+ age_days: 1,
+ licence_spdx: "BSD-2-Clause".into(),
+ licence_class: LicenceClass::AdaptWithAttribution,
+ answered_from: from,
+ mode,
+ hits,
+ truncated: false,
+ warnings,
+ }
+ }
+
+ #[test]
+ fn render_one_def_hit() {
+ let a = answer_with(
+ AnswerMode::Def,
+ vec![Hit {
+ file: "lib/lz4.h".into(),
+ line: 133,
+ kind: "define".into(),
+ text: "#define LZ4_VERSION_RELEASE 0".into(),
+ value: Some("0".into()),
+ score: 14.5,
+ licence_warning: None,
+ }],
+ AnsweredFrom::LocalPack,
+ vec![],
+ );
+ let expected = "lz4 pin 2026-10-02 (1 day old) BSD-2-Clause lib/lz4.h:133\n\
+ #define LZ4_VERSION_RELEASE 0\n\
+ value: 0\n\
+ answered from: local pack";
+ assert_eq!(render(&a), expected);
+ }
+
+ #[test]
+ fn render_three_hits_how_mode_with_warning() {
+ let hit = |f: &str, l: u32| Hit {
+ file: f.into(),
+ line: l,
+ kind: "function".into(),
+ text: format!("fn at {f}"),
+ value: None,
+ score: 1.0,
+ licence_warning: None,
+ };
+ let a = answer_with(
+ AnswerMode::How,
+ vec![hit("a.rs", 1), hit("b.rs", 2), hit("c.rs", 3)],
+ AnsweredFrom::Hosted,
+ vec!["pin is 23 days old; the question asks for the current source".into()],
+ );
+ let expected = "lz4 pin 2026-10-02 (1 day old) BSD-2-Clause a.rs:1\n\
+ fn at a.rs\n\
+ lz4 pin 2026-10-02 (1 day old) BSD-2-Clause b.rs:2\n\
+ fn at b.rs\n\
+ lz4 pin 2026-10-02 (1 day old) BSD-2-Clause c.rs:3\n\
+ fn at c.rs\n\
+ answered from: hosted hub\n\
+ warning: pin is 23 days old; the question asks for the current source";
+ assert_eq!(render(&a), expected);
+ }
+
+ #[test]
+ fn render_no_hits() {
+ let a = answer_with(AnswerMode::Def, vec![], AnsweredFrom::LocalPack, vec![]);
+ let expected = "lz4 pin 2026-10-02 (1 day old) BSD-2-Clause\n\
+ no match in lz4 (pin 2026-10-02)\n\
+ answered from: local pack";
+ assert_eq!(render(&a), expected);
+ }
+
+ #[test]
+ fn age_days_pluralization() {
+ let a = answer_with(AnswerMode::Def, vec![], AnsweredFrom::LocalPack, vec![]);
+ assert!(render(&a).contains("(1 day old)"));
+ let mut a2 = a;
+ a2.age_days = 5;
+ assert!(render(&a2).contains("(5 days old)"));
+ }
+}
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index d28e176b8..741c09415 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -29,6 +29,8 @@
//! same-machine CLI/MCP commands (`xerj mcp`, `xerj autoindex`, `xerj init`)
//! - [`xccode`] — the reference-coding semantics shared by `xerj code`,
//! `xerj corpus`, and the `xerj_code_search` MCP tool (issue #977)
+//! - [`hub`] — Corpus Hub v2 shared types: the catalogue, pack manifest,
+//! definition record, and the [`hub::Answer`] every surface renders (#1374)
pub mod calibration;
pub mod config;
@@ -36,6 +38,7 @@ pub mod error;
pub mod feedback;
pub mod field_coercion;
pub mod fsio;
+pub mod hub;
pub mod localauth;
pub mod metrics;
pub mod net;
diff --git a/engine/crates/xerj-hub/Cargo.toml b/engine/crates/xerj-hub/Cargo.toml
new file mode 100644
index 000000000..d239db94a
--- /dev/null
+++ b/engine/crates/xerj-hub/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "xerj-hub"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+description = "Corpus Hub v2 client: pull pre-indexed packs, route a question to a corpus, answer locally or through the hosted hub"
+
+[dependencies]
+xerj-common = { path = "../xerj-common" }
+serde.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
+anyhow.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
diff --git a/engine/crates/xerj-hub/src/lib.rs b/engine/crates/xerj-hub/src/lib.rs
new file mode 100644
index 000000000..476f32a84
--- /dev/null
+++ b/engine/crates/xerj-hub/src/lib.rs
@@ -0,0 +1,35 @@
+//! Corpus Hub v2 client (discussion #1374, `docs/CORPUS_HUB_V2_IMPLEMENTATION.md`).
+//!
+//! The hub serves PRE-INDEXED packs. This crate pulls one, restores it into a
+//! local node with no re-indexing, routes a question to the right corpus, and
+//! answers from the local pack when it is cached or through the hosted hub when
+//! it is not — so an agent gets an answer in one step instead of the nine the v1
+//! clone-and-index path needed.
+//!
+//! The data shapes that cross crates live in [`xerj_common::hub`]; this crate
+//! re-exports them so callers write `xerj_hub::Answer`, not the full path.
+//!
+//! Modules are added step by step per the implementation plan:
+//! pack build/pull (Step 2), definitions (Step 3), catalogue + sketch (Step 4),
+//! router + ask (Step 5). This file carries only the shared re-exports until
+//! those land.
+
+pub use xerj_common::hub::{
+ render, Answer, AnswerMode, AnsweredFrom, Catalogue, CatalogueEntry, DefRecord, Hit,
+ LicenceClass, PackIndex, PackManifest, PackSource, Sketch, AUTO_PULL_MAX_BYTES,
+ CATALOGUE_FORMAT_VERSION, DEFAULT_HUB_URL, DISK_BUDGET_BYTES, HUB_PUBKEY_HEX,
+ PACK_FORMAT_VERSION,
+};
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn reexports_resolve() {
+ // The crate compiles and the shared types are reachable under xerj_hub::.
+ assert_eq!(PACK_FORMAT_VERSION, 1);
+ assert_eq!(AnsweredFrom::LocalPack.label(), "local pack");
+ let _ = LicenceClass::AdaptWithAttribution;
+ }
+}