AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(hub): catalogue builder + client loader (step 4)
xerj-org280083efa491
diff --git a/engine/crates/xerj-hub/src/catalogue.rs b/engine/crates/xerj-hub/src/catalogue.rsindex 8129acc1d..9861b7747 100644--- a/engine/crates/xerj-hub/src/catalogue.rs+++ b/engine/crates/xerj-hub/src/catalogue.rs@@ -1,3 +1,850 @@-//! Corpus Hub v2 — the catalogue module. Filled by the implementing agent per-//! docs/CORPUS_HUB_V2_IMPLEMENTATION.md. This stub reserves the module so the-//! crate wires up before the behaviour lands.+//! Corpus Hub v2 — the catalogue (`docs/CORPUS_HUB_V2_IMPLEMENTATION.md`, step 4).+//!+//! The catalogue is one small, signed JSON file listing every corpus the hub+//! serves. An agent reads it to pick a corpus — slug, domain, licence, pin,+//! pack URL and a Bloom [`term sketch`](crate::sketch) for offline routing —+//! without downloading a single pack.+//!+//! This module holds the two ends of that file:+//!+//! - [`build_catalogue`] — the CI-side builder. It turns one+//! [`CorpusBuildInput`] per corpus into a [`CatalogueEntry`], assembles the+//! [`Catalogue`], serialises it deterministically ([`canonical_json`]) and+//! signs those exact bytes. It deliberately takes its per-corpus facts as a+//! struct instead of opening pack archives, so it stays decoupled from the+//! pack module built in parallel.+//! - [`load_catalogue`] — the client-side loader. It serves a cached copy while+//! it is young, otherwise refreshes over HTTP with `If-None-Match` and+//! **verifies the detached signature before letting the fetched bytes replace+//! the cache**. On any network or verification failure it keeps the last good+//! cached copy; `offline` never touches the network.+//!+//! The signing primitives are shared with the pack format+//! (`xerj_autoindex::harvest::sign`): a raw ed25519 signature over the exact+//! bytes, as lower-case hex in a sibling `.sig` file.++use std::fs;+use std::path::{Path, PathBuf};+use std::time::Duration;++use anyhow::{anyhow, bail, Context, Result};+use serde::Deserialize;+use xerj_autoindex::harvest::sign::{sign_bytes, verify_bytes};+use xerj_common::hub::{+ Catalogue, CatalogueEntry, LicenceClass, CATALOGUE_FORMAT_VERSION, HUB_PUBKEY_HEX,+};++/// Path of the catalogue under the hub's API root.+const CATALOGUE_API_PATH: &str = "api/v1/catalogue.json";+/// The cached catalogue file name inside the cache dir.+const CACHE_FILE: &str = "catalogue.json";+/// Where the last response's ETag is stored for the conditional GET.+const ETAG_FILE: &str = "catalogue.etag";+/// A cached catalogue younger than this is used without touching the network.+const FRESH_FOR: Duration = Duration::from_secs(24 * 60 * 60);++// ── builder input ───────────────────────────────────────────────────────────++/// One corpus's inputs to [`build_catalogue`]: the path to its registry+/// manifest (`tools/xerj-code/hub/<slug>.json`) plus the pack facts and the+/// terms for its sketch.+///+/// Everything the catalogue entry needs that the manifest does not yet carry is+/// supplied by the caller here; the manifest's own field always wins when it+/// has one.+pub struct CorpusBuildInput {+ /// Registry manifest; read for slug, licence (spdx + class), the pin sha+ /// fallback and pin date, and — when future manifests carry them — topics,+ /// the `ask_for` sentence and `dependency_names`.+ pub manifest_path: PathBuf,+ /// SEC / DB / NET / … — the backlog row's domain, which the caller supplies+ /// (the manifest does not record it).+ pub domain: String,+ pub refresh_cadence_days: u32,+ pub pack_bytes: u64,+ pub pack_url: String,+ pub pack_sha256: String,+ pub sig_url: String,+ pub quality_grade: Option<String>,+ /// Identifiers / vocabulary for the Bloom sketch. The caller assembles these+ /// (def symbols, path components, `ask_for`/`topics` words) per the plan;+ /// the builder only hashes them.+ pub terms: Vec<String>,+ /// Pin sha override. When empty, the manifest's first repo sha is used — the+ /// manifests store the authoritative pin there, so that is the common path.+ pub pin_sha: String,+ /// The backlog row's `ask_for` sentence, used only when the manifest does+ /// not carry one of its own.+ pub ask_for: String,+}++// ── manifest shapes (only the fields we read) ───────────────────────────────+//+// No `deny_unknown_fields`: the real hub manifests carry many fields we do not+// need (`url`, `files`, `bytes`, the licence hint, review `by`/`note`, …), and+// serde skips them by default. New richer fields (`topics`, `ask_for`,+// `dependency_names`, `pin_date`) are read when present and default otherwise.++#[derive(Deserialize)]+struct Manifest {+ corpus: String,+ #[serde(default)]+ cloned_at: Option<String>,+ #[serde(default)]+ repos: Vec<ManifestRepo>,+ #[serde(default)]+ topics: Vec<String>,+ #[serde(default)]+ ask_for: Option<String>,+ /// Per the plan's note, the `ask_for` sentence rides in a `domain` field in+ /// the backlog-derived manifests; used as a secondary source for `ask_for`.+ #[serde(default)]+ domain: Option<String>,+ #[serde(default)]+ pin_date: Option<String>,+ #[serde(default)]+ dependency_names: Vec<String>,+}++#[derive(Deserialize)]+struct ManifestRepo {+ #[serde(default)]+ sha: String,+ #[serde(default)]+ review: Option<ManifestReview>,+}++#[derive(Deserialize)]+struct ManifestReview {+ #[serde(default)]+ spdx: String,+ #[serde(rename = "use", default)]+ use_: String,+ /// `YYYY-MM-DD` the review (and thus the pin) was taken.+ #[serde(default)]+ at: Option<String>,+}++// ── builder ─────────────────────────────────────────────────────────────────++/// The exact bytes the catalogue is signed over, and the bytes CI must write to+/// `catalogue.json`. Compact UTF-8 JSON with the corpora sorted by slug, so the+/// same inputs produce a byte-identical file and the detached signature+/// reproduces across builds. The client verifies the signature over the raw+/// bytes it fetches, so the only contract is that the publisher writes *these*+/// bytes.+pub fn canonical_json(catalogue: &Catalogue) -> Result<Vec<u8>> {+ serde_json::to_vec(catalogue).context("serialize catalogue")+}++/// Build the catalogue and its detached signature (hex) over the exact JSON+/// bytes. One [`CatalogueEntry`] per input; the entries are sorted by slug so+/// the output does not depend on the order CI enumerates the registry in.+pub fn build_catalogue(+ inputs: &[CorpusBuildInput],+ generated_at: &str,+ sign_seed_hex: &str,+) -> Result<(Catalogue, String)> {+ let mut corpora = Vec::with_capacity(inputs.len());+ for input in inputs {+ corpora.push(build_entry(input)?);+ }+ corpora.sort_by(|a, b| a.slug.cmp(&b.slug));++ let catalogue = Catalogue {+ format_version: CATALOGUE_FORMAT_VERSION,+ generated_at: generated_at.to_string(),+ corpora,+ };++ let bytes = canonical_json(&catalogue)?;+ let signature = sign_bytes(&bytes, sign_seed_hex).context("sign catalogue bytes")?;+ Ok((catalogue, signature))+}++fn build_entry(input: &CorpusBuildInput) -> Result<CatalogueEntry> {+ let path = &input.manifest_path;+ let raw =+ fs::read_to_string(path).with_context(|| format!("read manifest {}", path.display()))?;+ let m: Manifest =+ serde_json::from_str(&raw).with_context(|| format!("parse manifest {}", path.display()))?;++ // Licence and the pin-sha fallback come from the repos; take them in a block+ // so the borrow ends before the owned manifest fields are moved out below.+ let (licence_spdx, licence_class, repo_sha, review_at) = {+ let first = m+ .repos+ .first()+ .ok_or_else(|| anyhow!("manifest {} lists no repos", path.display()))?;+ let review = first.review.as_ref().ok_or_else(|| {+ anyhow!(+ "manifest {}: first repo has no licence review",+ path.display()+ )+ })?;+ let spdx = review.spdx.trim().to_string();+ if spdx.is_empty() {+ bail!("manifest {}: empty review.spdx", path.display());+ }+ // Most restrictive class across all repos — a single GPL/mixed sub-repo+ // must not be masked by a permissive sibling.+ let class = most_restrictive_class(&m.repos, path)?;+ (spdx, class, first.sha.trim().to_string(), review.at.clone())+ };++ // Caller's pin sha wins; otherwise the manifest's first repo sha.+ let pin_sha = {+ let caller = input.pin_sha.trim();+ if caller.is_empty() {+ repo_sha+ } else {+ caller.to_string()+ }+ };+ if pin_sha.is_empty() {+ bail!(+ "manifest {}: no pin sha (caller override empty and repo sha absent)",+ path.display()+ );+ }++ // Pin date: an explicit manifest field, else the review date, else the date+ // portion of the clone timestamp.+ let pin_date = m+ .pin_date+ .or(review_at)+ .or_else(|| {+ m.cloned_at+ .as_deref()+ .and_then(|c| c.get(0..10).map(str::to_string))+ })+ .unwrap_or_default();++ // ask_for: the manifest's own field, else its `domain` sentence, else the+ // caller's fallback.+ let ask_for = non_empty(m.ask_for.as_deref())+ .or_else(|| non_empty(m.domain.as_deref()))+ .unwrap_or_else(|| input.ask_for.clone());++ let term_sketch = crate::sketch::build(input.terms.iter().cloned());++ Ok(CatalogueEntry {+ slug: m.corpus,+ domain: input.domain.clone(),+ topics: m.topics,+ ask_for,+ pin_sha,+ pin_date,+ refresh_cadence_days: input.refresh_cadence_days,+ licence_spdx,+ licence_class,+ pack_bytes: input.pack_bytes,+ pack_url: input.pack_url.clone(),+ pack_sha256: input.pack_sha256.clone(),+ sig_url: input.sig_url.clone(),+ dependency_names: m.dependency_names,+ quality_grade: input.quality_grade.clone(),+ term_sketch,+ })+}++/// `Some(trimmed)` when the string is present and non-blank, else `None`.+fn non_empty(s: Option<&str>) -> Option<String> {+ s.map(str::trim)+ .filter(|t| !t.is_empty())+ .map(str::to_string)+}++/// Map a registry `review.use` to a [`LicenceClass`]. `mixed` is read as+/// `approach-only`: part of the repo is copyable and part is not, and the+/// licence-safe (restrictive-first) reading is "read, do not copy".+fn parse_class(use_: &str) -> Option<LicenceClass> {+ match use_.trim() {+ "adapt-with-attribution" => Some(LicenceClass::AdaptWithAttribution),+ "approach-only" => Some(LicenceClass::ApproachOnly),+ "excluded" => Some(LicenceClass::Excluded),+ "mixed" => Some(LicenceClass::ApproachOnly),+ _ => None,+ }+}++/// Ordering for "most restrictive wins": excluded > approach-only > adapt.+fn restrictiveness(c: LicenceClass) -> u8 {+ match c {+ LicenceClass::AdaptWithAttribution => 0,+ LicenceClass::ApproachOnly => 1,+ LicenceClass::Excluded => 2,+ }+}++fn most_restrictive_class(repos: &[ManifestRepo], path: &Path) -> Result<LicenceClass> {+ let mut worst: Option<LicenceClass> = None;+ for repo in repos {+ let review = repo+ .review+ .as_ref()+ .ok_or_else(|| anyhow!("manifest {}: a repo has no licence review", path.display()))?;+ let class = parse_class(&review.use_).ok_or_else(|| {+ anyhow!(+ "manifest {}: unknown licence use '{}'",+ path.display(),+ review.use_+ )+ })?;+ worst = Some(match worst {+ Some(w) if restrictiveness(w) >= restrictiveness(class) => w,+ _ => class,+ });+ }+ worst.ok_or_else(|| anyhow!("manifest {}: no repos to classify", path.display()))+}++// ── client loader ───────────────────────────────────────────────────────────++/// Load the catalogue.+///+/// A cached `<cache_dir>/catalogue.json` younger than 24 h is returned as-is (it+/// was signature-verified when it was written). Otherwise the catalogue is+/// fetched from `<hub_url>/api/v1/catalogue.json` with `If-None-Match` (from+/// `<cache_dir>/catalogue.etag`), its detached signature+/// (`…/catalogue.json.sig`) is verified against the pubkey **before** the bytes+/// are allowed to replace the cache, and only a verified copy is cached and+/// returned. On any network, signature or parse failure the previous cached+/// copy is kept and returned; if there is none, an error is returned.+/// `offline == true` never fetches.+pub fn load_catalogue(hub_url: &str, cache_dir: &Path, offline: bool) -> Result<Catalogue> {+ let cache_file = cache_dir.join(CACHE_FILE);+ let etag_file = cache_dir.join(ETAG_FILE);++ if offline {+ return match read_cached(&cache_file) {+ Some(cat) => {+ warn_if_old(&cat);+ Ok(cat)+ }+ None => bail!(+ "offline and no usable cached catalogue at {}",+ cache_file.display()+ ),+ };+ }++ // A young cache is authoritative without a network round-trip.+ if cache_is_fresh(&cache_file) {+ if let Some(cat) = read_cached(&cache_file) {+ return Ok(cat);+ }+ }++ match fetch_and_store(hub_url, &cache_file, &etag_file) {+ Ok(cat) => Ok(cat),+ // Keep the last good copy on any failure, exactly as the plan requires.+ Err(e) => match read_cached(&cache_file) {+ Some(cat) => {+ eprintln!("xerj hub: catalogue refresh failed ({e:#}); keeping the cached copy");+ warn_if_old(&cat);+ Ok(cat)+ }+ None => Err(e.context("fetch catalogue, and no cached copy to fall back to")),+ },+ }+}++/// Read and validate the cached catalogue, or `None` if it is missing,+/// unreadable, unparsable or a format version this client does not speak.+fn read_cached(cache_file: &Path) -> Option<Catalogue> {+ let raw = fs::read(cache_file).ok()?;+ parse_catalogue(&raw).ok()+}++/// Parse catalogue bytes and refuse a format version this client does not equal.+fn parse_catalogue(bytes: &[u8]) -> Result<Catalogue> {+ let cat: Catalogue = serde_json::from_slice(bytes).context("parse catalogue JSON")?;+ if cat.format_version != CATALOGUE_FORMAT_VERSION {+ bail!(+ "catalogue format_version {} is not the supported {}",+ cat.format_version,+ CATALOGUE_FORMAT_VERSION+ );+ }+ Ok(cat)+}++/// True when the cache file exists and its mtime is within [`FRESH_FOR`].+fn cache_is_fresh(cache_file: &Path) -> bool {+ let Ok(modified) = fs::metadata(cache_file).and_then(|m| m.modified()) else {+ return false;+ };+ match modified.elapsed() {+ Ok(elapsed) => elapsed < FRESH_FOR,+ // mtime in the future (clock skew): treat as fresh rather than hammer.+ Err(_) => true,+ }+}++/// The pubkey to verify with: `XERJ_HUB_PUBKEY` (tests), else the compiled-in+/// [`HUB_PUBKEY_HEX`]. `None` when neither is set — the real key lands later, so+/// an empty configured key means "skip verification with a warning", never a+/// hard failure.+fn configured_pubkey() -> Option<String> {+ std::env::var("XERJ_HUB_PUBKEY")+ .ok()+ .and_then(|s| non_empty(Some(&s)))+ .or_else(|| non_empty(Some(HUB_PUBKEY_HEX)))+}++/// Warn when the cached copy being served is a day or more old.+fn warn_if_old(cat: &Catalogue) {+ if let Ok(generated) = chrono::DateTime::parse_from_rfc3339(&cat.generated_at) {+ let days = chrono::Utc::now()+ .signed_duration_since(generated.with_timezone(&chrono::Utc))+ .num_days();+ if days >= 1 {+ eprintln!(+ "xerj hub: catalogue is {days} day{} old",+ if days == 1 { "" } else { "s" }+ );+ }+ }+}++/// Fetch, verify and cache the catalogue. The signature guards the replacement:+/// the fetched bytes are only written to the cache after they verify (or after a+/// deliberate, warned skip when no key is configured).+fn fetch_and_store(hub_url: &str, cache_file: &Path, etag_file: &Path) -> Result<Catalogue> {+ let base = hub_url.trim_end_matches('/');+ let cat_url = format!("{base}/{CATALOGUE_API_PATH}");+ let sig_url = format!("{cat_url}.sig");++ let client = reqwest::blocking::Client::builder()+ .timeout(Duration::from_secs(30))+ .connect_timeout(Duration::from_secs(10))+ .redirect(reqwest::redirect::Policy::limited(5))+ .build()+ .context("build http client")?;++ // Conditional GET: a stored ETag lets the hub answer 304 and save bandwidth.+ let prev_etag = fs::read_to_string(etag_file)+ .ok()+ .and_then(|s| non_empty(Some(&s)));+ let mut req = client.get(&cat_url);+ if let Some(etag) = &prev_etag {+ req = req.header(reqwest::header::IF_NONE_MATCH, etag.as_str());+ }+ let resp = req.send().with_context(|| format!("GET {cat_url}"))?;++ if resp.status() == reqwest::StatusCode::NOT_MODIFIED {+ // The cached bytes are still current; re-touch them so the next 24 h are+ // served from cache without re-asking.+ let raw =+ fs::read(cache_file).context("hub returned 304 Not Modified but the cache is gone")?;+ let cat = parse_catalogue(&raw)?;+ write_atomic(cache_file, &raw)?;+ return Ok(cat);+ }+ if !resp.status().is_success() {+ bail!("GET {cat_url} returned HTTP {}", resp.status());+ }++ // Capture the new ETag before the body consumes the response.+ let new_etag = resp+ .headers()+ .get(reqwest::header::ETAG)+ .and_then(|v| v.to_str().ok())+ .map(str::to_string);+ let body = resp+ .bytes()+ .with_context(|| format!("read body of {cat_url}"))?+ .to_vec();++ let sig_resp = client+ .get(&sig_url)+ .send()+ .with_context(|| format!("GET {sig_url}"))?;+ if !sig_resp.status().is_success() {+ bail!("GET {sig_url} returned HTTP {}", sig_resp.status());+ }+ let sig_hex = sig_resp+ .text()+ .with_context(|| format!("read body of {sig_url}"))?+ .trim()+ .to_string();++ // Verify BEFORE these bytes are allowed anywhere near the cache.+ match configured_pubkey() {+ Some(key) => verify_bytes(&body, &sig_hex, &key)+ .context("catalogue signature did not verify; refusing to replace the cache")?,+ None => eprintln!(+ "xerj hub: no hub public key configured (HUB_PUBKEY_HEX empty, XERJ_HUB_PUBKEY \+ unset) — caching the catalogue WITHOUT verifying its signature"+ ),+ }++ // Only trusted bytes get parsed and cached.+ let cat = parse_catalogue(&body)?;+ if let Some(parent) = cache_file.parent() {+ fs::create_dir_all(parent)+ .with_context(|| format!("create cache dir {}", parent.display()))?;+ }+ write_atomic(cache_file, &body)?;+ let _ = fs::write(sig_sibling(cache_file), &sig_hex);+ match &new_etag {+ Some(etag) => {+ let _ = fs::write(etag_file, etag);+ }+ // A fresh body with no ETag: drop a stale one so we do not send a+ // mismatched If-None-Match next time.+ None => {+ let _ = fs::remove_file(etag_file);+ }+ }+ Ok(cat)+}++/// `<cache_file>.sig` (e.g. `catalogue.json.sig`).+fn sig_sibling(cache_file: &Path) -> PathBuf {+ let mut name = cache_file.file_name().unwrap_or_default().to_os_string();+ name.push(".sig");+ cache_file.with_file_name(name)+}++/// Write via a sibling temp file and rename, so a crash mid-write never leaves a+/// half-written catalogue under the final name.+fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {+ let mut tmp_name = path.file_name().unwrap_or_default().to_os_string();+ tmp_name.push(".tmp");+ let tmp = path.with_file_name(tmp_name);+ fs::write(&tmp, bytes).with_context(|| format!("write {}", tmp.display()))?;+ fs::rename(&tmp, path).with_context(|| format!("rename into {}", path.display()))?;+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::io::{Read, Write};+ use std::sync::Mutex;+ use std::time::SystemTime;++ use xerj_autoindex::harvest::sign::generate_keypair;++ /// `XERJ_HUB_PUBKEY` is process-global; serialise the tests that set it.+ static ENV_LOCK: Mutex<()> = Mutex::new(());++ /// A `.key`/`.pub` keypair for the tests (hex seed, hex public key).+ fn keypair() -> (String, String) {+ generate_keypair().unwrap()+ }++ fn write_manifest(dir: &Path, slug: &str, sha: &str, spdx: &str, use_: &str) -> PathBuf {+ let p = dir.join(format!("{slug}.json"));+ let json = serde_json::json!({+ "corpus": slug,+ "cloned_at": "2026-10-02T00:49:45Z",+ "repos": [{+ "repo": slug,+ "url": format!("https://example.test/{slug}"),+ "licence": "hint only",+ "sha": sha,+ "files": 1,+ "bytes": 1,+ "review": {+ "spdx": spdx,+ "use": use_,+ "by": "xerj-org",+ "at": "2026-10-02",+ "note": "test fixture"+ }+ }]+ });+ fs::write(&p, serde_json::to_vec_pretty(&json).unwrap()).unwrap();+ p+ }++ fn two_inputs(dir: &Path) -> Vec<CorpusBuildInput> {+ let alpha = write_manifest(+ dir,+ "alpha",+ "1111111111111111111111111111111111111111",+ "MIT",+ "adapt-with-attribution",+ );+ let beta = write_manifest(+ dir,+ "beta",+ "2222222222222222222222222222222222222222",+ "GPL-3.0-only",+ "approach-only",+ );+ vec![+ CorpusBuildInput {+ manifest_path: alpha,+ domain: "DB".into(),+ refresh_cadence_days: 7,+ pack_bytes: 111,+ pack_url: "https://example.test/alpha.hubpack.tar.zst".into(),+ pack_sha256: "a".repeat(64),+ sig_url: "https://example.test/alpha.hubpack.tar.zst.sig".into(),+ quality_grade: Some("G7 4/5".into()),+ terms: vec!["alpha_sym".into(), "db".into(), "shared".into()],+ pin_sha: String::new(),+ ask_for: "an agent working on storage queries this for alpha".into(),+ },+ CorpusBuildInput {+ manifest_path: beta,+ domain: "SEC".into(),+ refresh_cadence_days: 1,+ pack_bytes: 222,+ pack_url: "https://example.test/beta.hubpack.tar.zst".into(),+ pack_sha256: "b".repeat(64),+ sig_url: "https://example.test/beta.hubpack.tar.zst.sig".into(),+ quality_grade: None,+ terms: vec!["beta_sym".into(), "sec".into(), "shared".into()],+ pin_sha: String::new(),+ ask_for: "an agent working on security queries this for beta".into(),+ },+ ]+ }++ fn entry<'a>(cat: &'a Catalogue, slug: &str) -> &'a CatalogueEntry {+ cat.corpora.iter().find(|e| e.slug == slug).unwrap()+ }++ #[test]+ fn build_catalogue_two_inputs() {+ let tmp = tempfile::tempdir().unwrap();+ let inputs = two_inputs(tmp.path());+ let (seed, public) = keypair();++ let (cat, sig) =+ build_catalogue(&inputs, "2026-10-14T00:00:00Z", &seed).expect("build catalogue");++ assert_eq!(cat.format_version, CATALOGUE_FORMAT_VERSION);+ assert_eq!(cat.corpora.len(), 2);+ // sorted by slug, regardless of input order+ assert_eq!(cat.corpora[0].slug, "alpha");+ assert_eq!(cat.corpora[1].slug, "beta");++ let a = entry(&cat, "alpha");+ assert_eq!(a.domain, "DB");+ assert_eq!(a.licence_spdx, "MIT");+ assert_eq!(a.licence_class, LicenceClass::AdaptWithAttribution);+ assert_eq!(a.pack_bytes, 111);+ // caller pin_sha was empty → manifest's first repo sha+ assert_eq!(a.pin_sha, "1111111111111111111111111111111111111111");+ assert_eq!(a.pin_date, "2026-10-02");+ assert_eq!(a.quality_grade.as_deref(), Some("G7 4/5"));+ // caller fallback, since the fixture manifest has no ask_for field+ assert!(a.ask_for.contains("alpha"));+ assert!(a.term_sketch.m_bits >= 8 * 1024);++ let b = entry(&cat, "beta");+ assert_eq!(b.licence_spdx, "GPL-3.0-only");+ assert_eq!(b.licence_class, LicenceClass::ApproachOnly);+ assert_eq!(b.pack_bytes, 222);+ assert_eq!(b.quality_grade, None);++ assert!(!sig.is_empty(), "signature must be present");+ // the detached signature verifies over the exact canonical bytes+ let bytes = canonical_json(&cat).unwrap();+ verify_bytes(&bytes, &sig, &public).expect("signature verifies");+ }++ #[test]+ fn signature_verify_rejects_tampered_bytes() {+ let tmp = tempfile::tempdir().unwrap();+ let inputs = two_inputs(tmp.path());+ let (seed, public) = keypair();+ let (cat, sig) = build_catalogue(&inputs, "2026-10-14T00:00:00Z", &seed).unwrap();++ let mut bytes = canonical_json(&cat).unwrap();+ // flipping any one byte must break verification+ let mid = bytes.len() / 2;+ bytes[mid] ^= 0x01;+ assert!(+ verify_bytes(&bytes, &sig, &public).is_err(),+ "a tampered catalogue must not verify"+ );+ }++ #[test]+ fn mixed_licence_reads_as_approach_only() {+ // a `mixed` review.use is the licence-unsafe case; it must map to the+ // restrictive class, not the permissive one.+ let tmp = tempfile::tempdir().unwrap();+ let path = write_manifest(+ tmp.path(),+ "lz4",+ "0774d05537f9762f838f7ab541b7765f1a729cb5",+ "BSD-2-Clause",+ "mixed",+ );+ let input = CorpusBuildInput {+ manifest_path: path,+ domain: "NET".into(),+ refresh_cadence_days: 7,+ pack_bytes: 1,+ pack_url: "u".into(),+ pack_sha256: "c".repeat(64),+ sig_url: "s".into(),+ quality_grade: None,+ terms: vec![],+ pin_sha: String::new(),+ ask_for: "fast compression".into(),+ };+ let (seed, _public) = keypair();+ let (cat, _sig) = build_catalogue(&[input], "2026-10-14T00:00:00Z", &seed).unwrap();+ assert_eq!(cat.corpora[0].licence_class, LicenceClass::ApproachOnly);+ }++ /// Build a catalogue, return `(bytes, sig, pubkey)` ready to seed a cache.+ fn fixture_bytes(dir: &Path) -> (Vec<u8>, String, String) {+ let inputs = two_inputs(dir);+ let (seed, public) = keypair();+ let (cat, sig) = build_catalogue(&inputs, "2026-10-14T00:00:00Z", &seed).unwrap();+ (canonical_json(&cat).unwrap(), sig, public)+ }++ fn seed_cache(cache_dir: &Path, bytes: &[u8], sig: &str) {+ fs::write(cache_dir.join(CACHE_FILE), bytes).unwrap();+ fs::write(sig_sibling(&cache_dir.join(CACHE_FILE)), sig).unwrap();+ }++ fn age_file(path: &Path, secs: u64) {+ let when = SystemTime::now()+ .checked_sub(Duration::from_secs(secs))+ .unwrap();+ std::fs::OpenOptions::new()+ .write(true)+ .open(path)+ .unwrap()+ .set_modified(when)+ .unwrap();+ }++ #[test]+ fn load_offline_uses_fresh_cache_without_network() {+ let _guard = ENV_LOCK.lock().unwrap();+ let fx = tempfile::tempdir().unwrap();+ let (bytes, sig, public) = fixture_bytes(fx.path());++ let cache = tempfile::tempdir().unwrap();+ seed_cache(cache.path(), &bytes, &sig);++ std::env::set_var("XERJ_HUB_PUBKEY", &public);+ // the hub URL is unreachable on purpose — offline must never dial it+ let cat = load_catalogue("http://127.0.0.1:1", cache.path(), true).expect("offline load");+ std::env::remove_var("XERJ_HUB_PUBKEY");++ assert_eq!(cat.corpora.len(), 2);+ assert_eq!(cat.corpora[0].slug, "alpha");+ }++ /// A one-shot localhost hub that answers exactly two GETs (catalogue, then+ /// signature) and then exits, so the test can join it. Each response says+ /// `Connection: close`, forcing a fresh TCP connection per GET.+ fn spawn_hub(cat_body: Vec<u8>, sig_body: String) -> (String, std::thread::JoinHandle<()>) {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ let url = format!("http://{addr}");+ let handle = std::thread::spawn(move || {+ for _ in 0..2 {+ let Ok((mut stream, _)) = listener.accept() else {+ break;+ };+ let mut buf = [0u8; 2048];+ let n = stream.read(&mut buf).unwrap_or(0);+ let req = String::from_utf8_lossy(&buf[..n]);+ let is_sig = req.lines().next().unwrap_or("").contains(".sig");+ let body: Vec<u8> = if is_sig {+ sig_body.clone().into_bytes()+ } else {+ cat_body.clone()+ };+ let header = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\+ Content-Length: {}\r\nETag: \"test-etag\"\r\nConnection: close\r\n\r\n",+ body.len()+ );+ let _ = stream.write_all(header.as_bytes());+ let _ = stream.write_all(&body);+ let _ = stream.flush();+ }+ });+ (url, handle)+ }++ #[test]+ fn load_rejects_tampered_network_replacement_and_keeps_cache() {+ let _guard = ENV_LOCK.lock().unwrap();+ let fx = tempfile::tempdir().unwrap();+ let (good_bytes, good_sig, public) = fixture_bytes(fx.path());++ // seed a VALID cache, then age it so the loader must refresh.+ let cache = tempfile::tempdir().unwrap();+ seed_cache(cache.path(), &good_bytes, &good_sig);+ age_file(&cache.path().join(CACHE_FILE), 48 * 3600);++ // the hub serves a TAMPERED catalogue with the (now mismatched) good sig+ let mut tampered = good_bytes.clone();+ tampered[good_bytes.len() / 2] ^= 0x01;+ let (url, handle) = spawn_hub(tampered, good_sig.clone());++ std::env::set_var("XERJ_HUB_PUBKEY", &public);+ let cat = load_catalogue(&url, cache.path(), false).expect("falls back to cached copy");+ std::env::remove_var("XERJ_HUB_PUBKEY");+ handle.join().unwrap();++ // the signature guarded the replacement: the GOOD cached copy is served,+ // not the tampered network bytes…+ let served = canonical_json(&cat).unwrap();+ assert_eq!(served, good_bytes, "tampered replacement must be rejected");+ // …and the on-disk cache is untouched.+ let on_disk = fs::read(cache.path().join(CACHE_FILE)).unwrap();+ assert_eq!(on_disk, good_bytes, "cache file must not be overwritten");+ }++ #[test]+ fn load_accepts_verified_network_replacement_into_empty_cache() {+ let _guard = ENV_LOCK.lock().unwrap();+ let fx = tempfile::tempdir().unwrap();+ let (good_bytes, good_sig, public) = fixture_bytes(fx.path());++ // empty cache → the loader must fetch+ let cache = tempfile::tempdir().unwrap();+ let (url, handle) = spawn_hub(good_bytes.clone(), good_sig.clone());++ std::env::set_var("XERJ_HUB_PUBKEY", &public);+ let cat = load_catalogue(&url, cache.path(), false).expect("verified fetch");+ std::env::remove_var("XERJ_HUB_PUBKEY");+ handle.join().unwrap();++ assert_eq!(cat.corpora.len(), 2);+ // the verified bytes are now cached, byte-for-byte+ let on_disk = fs::read(cache.path().join(CACHE_FILE)).unwrap();+ assert_eq!(on_disk, good_bytes);+ }++ #[test]+ fn load_without_a_key_skips_verification_rather_than_failing() {+ let _guard = ENV_LOCK.lock().unwrap();+ // no XERJ_HUB_PUBKEY, and HUB_PUBKEY_HEX is empty on this branch, so the+ // loader must cache the fetched copy WITHOUT verifying — not hard-fail.+ std::env::remove_var("XERJ_HUB_PUBKEY");+ assert!(+ HUB_PUBKEY_HEX.is_empty(),+ "precondition: no compiled-in key"+ );++ let fx = tempfile::tempdir().unwrap();+ let (good_bytes, _good_sig, _public) = fixture_bytes(fx.path());+ let cache = tempfile::tempdir().unwrap();+ // the sig is irrelevant here: with no key we never check it+ let (url, handle) = spawn_hub(good_bytes.clone(), "00".repeat(64));++ let cat = load_catalogue(&url, cache.path(), false).expect("unverified fetch is allowed");+ handle.join().unwrap();++ assert_eq!(cat.corpora.len(), 2);+ }+}