AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(hub): remediate the PR #1403 adversarial-review findings in /_hub/v1
xerj-org6aeacecd97f8
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 8b83bb323..f4df0b160 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -10205,6 +10205,82 @@ fn search_actor_class(size: usize, index: &str) -> &'static str {}}+/// How a search the node issues to ITSELF differs from one that arrived over+/// HTTP. Only [`crate::hub_api`]'s in-process caller sets it, around the one+/// internal request it makes; a request off the wire cannot carry it, because it+/// is a task-local and not a header, a parameter or a body field.+///+/// Both effects exist because the hosted hub's routes are public and a pack's+/// contents are not the operator's: neither may be a lever on the node.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub(crate) struct InProcessSearch {+ /// Do not append this search to the audit ring. The ring is a bounded piece+ /// of operator evidence shared by the whole node (4,096 entries); a public+ /// route that appended one entry per request would let a crowd of strangers+ /// age every earlier entry out of it. The HTTP access log still records the+ /// request.+ pub skip_audit: bool,+ /// A fan-out over several indices skips an index whose search fails — it+ /// is counted in `_shards.failed` and logged — instead of failing the whole+ /// request. One pack with an unusable mapping must not take `search` down+ /// for every other pack. When EVERY index fails the first failure is still+ /// returned, so an outage never reads as "no results".+ pub skip_failed_indices: bool,+}++tokio::task_local! {+ static IN_PROCESS_SEARCH: InProcessSearch;+}++/// Run `fut` — the whole internal request, handler included — with `mode`+/// installed. Same shape as `xerj_engine::index_guard::scoped`, and for the same+/// reason: the handler reads it on the task that drives `fut`.+pub(crate) async fn scoped_in_process_search<F>(mode: InProcessSearch, fut: F) -> F::Output+where+ F: std::future::Future,+{+ IN_PROCESS_SEARCH.scope(mode, fut).await+}++/// The mode the current request runs under; the default (a request off the+/// wire, or engine-internal work) changes nothing.+fn in_process_search() -> InProcessSearch {+ IN_PROCESS_SEARCH.try_with(|m| *m).unwrap_or_default()+}++/// Bookkeeping for [`InProcessSearch::skip_failed_indices`]: how many indices a+/// fan-out tried, how many failed, and the first failure to report if all did.+#[derive(Default)]+struct SkippedIndices {+ searched: u32,+ failed: u32,+ first_failure: Option<Response>,+}++impl SkippedIndices {+ /// One more index was attempted.+ fn attempt(&mut self) {+ self.searched += 1;+ }++ /// An attempted index failed with `why`; keep the first such response.+ fn skip(&mut self, why: impl FnOnce() -> Response) {+ self.failed += 1;+ if self.first_failure.is_none() {+ self.first_failure = Some(why());+ }+ }++ /// The failure to answer with when nothing at all could be searched.+ fn all_failed(self) -> Option<Response> {+ if self.failed > 0 && self.failed == self.searched {+ self.first_failure+ } else {+ None+ }+ }+}+async fn search_impl(state: AppState,index: String,@@ -10213,6 +10289,9 @@ async fn search_impl(body: EsSearchJson,) -> Response {let started = Instant::now();+ // Read once, before the first `.await`: what the in-process hub caller asked+ // of this search. A request off the wire gets the default and is unaffected.+ let in_process = in_process_search();// Closed-index handling is deferred to after index resolution below so// `ignore_unavailable` / `expand_wildcards` are honored per ES semantics.@@ -12546,6 +12625,8 @@ async fn search_impl(// and excess searches queue on that semaphore deadline-aware exactly as// they did when this loop issued them one at a time.let mut spawned_searches = Vec::with_capacity(index_names.len());+ // Only used when an in-process caller asked to have failing indices skipped.+ let mut skipped = SkippedIndices::default();for idx_name in &index_names {let idx = match state.engine.get_index(idx_name) {Ok(i) => i,@@ -12554,6 +12635,19 @@ async fn search_impl(// Silently skip; behaves like an empty index.continue;}+ if in_process.skip_failed_indices {+ // An index that is listed but cannot be opened is one bad+ // index, not a reason to refuse the rest.+ tracing::warn!(+ index = %idx_name,+ error = ?e.to_string().chars().take(300).collect::<String>(),+ "in-process search: index unavailable, skipping it"+ );+ skipped.attempt();+ skipped.skip(|| ApiError::new(xerj_common::XerjError::from(e)).into_response());+ shards_failed_count += 1;+ continue;+ }let mut api_err = ApiError::new(xerj_common::XerjError::from(e));// Byte-parity `reason` ("no such index [name]") is fixed by// ES-compat tests and set unconditionally in `error.rs` — this@@ -12651,6 +12745,7 @@ async fn search_impl(}idx.search(&req_clone).await}));+ skipped.attempt();spawned_searches.push((idx_name, search_task));}@@ -12663,9 +12758,29 @@ async fn search_impl(match search_result {Err(join_err) => {+ if in_process.skip_failed_indices {+ tracing::warn!(index = %idx_name, "in-process search: index search panicked, skipping it");+ skipped.skip(|| search_task_panic_response(join_err, idx_name));+ shards_failed_count += 1;+ continue;+ }return search_task_panic_response(join_err, idx_name);}- Ok(Err(e)) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),+ Ok(Err(e)) => {+ if in_process.skip_failed_indices {+ // The engine's own words can quote the query; bound and+ // escape them before they reach the log.+ tracing::warn!(+ index = %idx_name,+ error = ?e.to_string().chars().take(300).collect::<String>(),+ "in-process search: index search failed, skipping it"+ );+ skipped.skip(|| ApiError::new(xerj_common::XerjError::from(e)).into_response());+ shards_failed_count += 1;+ continue;+ }+ return ApiError::new(xerj_common::XerjError::from(e)).into_response();+ }Ok(Ok(result)) => {// A Painless resource limit tripped somewhere in scoring or// aggregation. Those paths have no error channel, so they@@ -12750,6 +12865,13 @@ async fn search_impl(}}+ // Skipping a bad index is isolation, not a way to hide an outage: when every+ // index that was tried failed there is nothing to answer from, and the first+ // failure is the answer.+ if let Some(failure) = skipped.all_failed() {+ return failure;+ }+// ── Post-merge `auto_date_histogram` re-coordination ────────────────────// Each per-shard run chose its own interval against its local min/max,// so a two-shard search with one doc per shard can end up with two@@ -13315,29 +13437,35 @@ async fn search_impl(// "anonymous" until issue #329 — the API-key operations had been passing a// real `principal.label()` since #201, so the plumbing existed and this one// site simply did not use it.- state.engine.audit.append_as(- // #1109: the machine/user split is decided HERE, server-side, from- // what the request itself is — never from anything a client can set- // (the spoofable `X-Xerj-Internal` header was considered and- // rejected during #1107, and that decision stands). See- // `search_actor_class` for exactly which requests are machine.- search_actor_class(body.size, index.as_str()),- "search",- principal.label(),- index.as_str(),- "ok",- // `size=` is the requested page size. A `size:0` search asks for no- // hits — it is a counting probe (autoindex's verification reads back- // exact group counts that way), not someone looking for documents,- // and `xerj gain` needs exactly that distinction to report a hit- // rate that is about people (#1105).- &format!(- "took={}ms hits={} size={}",- took_ms,- merged_hits.len(),- body.size- ),- );+ //+ // Not for a search the hosted hub issues to itself (`InProcessSearch`): that+ // is a public read, and one audit entry per request would let anyone age the+ // operator's evidence out of the ring.+ if !in_process.skip_audit {+ state.engine.audit.append_as(+ // #1109: the machine/user split is decided HERE, server-side, from+ // what the request itself is — never from anything a client can set+ // (the spoofable `X-Xerj-Internal` header was considered and+ // rejected during #1107, and that decision stands). See+ // `search_actor_class` for exactly which requests are machine.+ search_actor_class(body.size, index.as_str()),+ "search",+ principal.label(),+ index.as_str(),+ "ok",+ // `size=` is the requested page size. A `size:0` search asks for no+ // hits — it is a counting probe (autoindex's verification reads back+ // exact group counts that way), not someone looking for documents,+ // and `xerj gain` needs exactly that distinction to report a hit+ // rate that is about people (#1105).+ &format!(+ "took={}ms hits={} size={}",+ took_ms,+ merged_hits.len(),+ body.size+ ),+ );+ }// ES semantics: when the sort has any non-score key, `max_score` is null// (scores aren't tracked during field-sorted execution unlessdiff --git a/engine/crates/xerj-api/src/hub_api.rs b/engine/crates/xerj-api/src/hub_api.rsindex b1cf4422b..a31b22a8b 100644--- a/engine/crates/xerj-api/src/hub_api.rs+++ b/engine/crates/xerj-api/src/hub_api.rs@@ -65,30 +65,79 @@//! * the private router offers three read routes and nothing else, which is a//! smaller surface than "the admin key against the whole API".//!-//! Three further guards bound what a hosted query can touch, even if a bug let-//! an unexpected name through: [`InProcessNode`] refuses any index expression-//! that is not made of `xc-` names in a conservative character set, every call-//! runs under an engine-level [`index_guard`] that hides every index outside-//! `xc-*`, and the corpus name is checked against the catalogue before any-//! filesystem path is built from it.+//! Four further guards bound what a hosted query can touch, even if a bug let+//! an unexpected name through:+//!+//! * [`InProcessNode`] refuses any index expression that is not made of `xc-`+//! names in a conservative character set;+//! * every call runs under an engine-level [`index_guard`] that hides every+//! index outside `xc-*` AND every index the caller's own request could not+//! see. `HubVisibility` is the INTERSECTION of the hub namespace and the rule+//! `authz_middleware` installed for the caller, captured before the work leaves+//! the request's task: a key scoped to `tenant-a-*` is refused a corpus here+//! exactly as it is by `_search`, and a hosted route is never a way to widen a+//! grant;+//! * the ledger's `index_prefix` is pack-supplied text, so it is checked against+//! the exact shape `xc-<slug>-b<digits>` for the corpus being asked about+//! ([`valid_build_prefix`]) before any query is built from it, and a `def` or+//! `code` request is then pinned to that build (`InProcessNode::pin_to_build`):+//! the node refuses an index expression outside it, whatever the shared+//! pipeline goes on to compute from its own second read of the ledger;+//! * the corpus name is checked against the catalogue before any filesystem path+//! is built from it.+//!+//! The licence is decided the same way on every route: the STRICTER of the+//! catalogue's class and the corpus's own ledger's ([`effective_licence`]); either+//! can refuse (`excluded`), and either can require the read-do-not-copy warning+//! (`approach-only`).//!//! The `XcHttp` methods are synchronous and the handlers async, so each query//! runs on the blocking pool (`run_query`) under a global concurrency cap//! ([`MAX_CONCURRENT_QUERIES`], `503` + `Retry-After` when full) and a wall//! clock timeout ([`QUERY_TIMEOUT`], `504`).//!+//! ## What the in-process call changes about the search it makes+//!+//! The hub's call sets `es_compat::InProcessSearch`, a task-local that a request+//! off the wire cannot carry, and nothing else does:+//!+//! * **It is not audited.** The node's `_search` handler appends every search to+//! the audit ring — 4,096 entries shared by the whole node — so one entry per+//! public request would let a crowd of strangers age the operator's evidence out+//! of it. The HTTP access log is the record of hub reads. (The audit LAYER in+//! `router.rs` is skipped for the public routes for the same reason.)+//! * **A failing index is skipped, not fatal.** `search` fans out over every pack+//! on the node, and the engine used to fail the whole request on the first index+//! that errored: one pack with an unusable mapping took `search` down for every+//! other pack. The index is skipped (logged, counted in `_shards.failed`); when+//! EVERY index fails the first failure is the answer, so an outage is a `503`+//! and never an empty list. `search` also sends `lenient`, which removes the+//! commonest cause (a pack that maps `body` as a number) at the source.+//! * **An oversized response shrinks the window.** One in-process response is+//! capped at 16 MiB (`MAX_INTERNAL_RESPONSE_BYTES`). `code` fetches whole records, ten+//! times the `k` asked for, so two very large records overflow it even at+//! `k=1`; the search is retried with half the window until it fits, and the+//! answer says it was built from fewer candidates (`truncated: true`). A+//! mapping read that overflows is a different matter — see `search` below.+//!//! ## Known limits, stated so they are not rediscovered//!-//! * Every hosted search appends an entry to the node's audit ring (the-//! `_search` handler does so unconditionally). On a dedicated hub node that is-//! harmless; on a node that also holds data worth auditing, a flood of-//! rate-limited-but-distinct clients can age out older entries.//! * `Answer.answered_from` is always `hosted` here: that is where the answer//! came from as far as the client is concerned, even though the server read a//! local pack.//! * The shared query pipeline's JSON output does not report which arms ran for//! `mode=semantic|hybrid` (its "BM25 only" note is prose-only), so those-//! modes carry a warning saying so instead of a silent fallback.+//! modes carry a warning saying what each one DOES instead: `semantic` over a+//! corpus with no `semantic_text` mapping returns no hits (it does not fall+//! back), `hybrid` answers from BM25 alone.+//! * `search` reads the node-wide mappings to learn which of `body`, `defs` and+//! `symbol_text` exist (this engine collapses a multi-token query to zero hits+//! when a named field is mapped nowhere). If that read fails — typically+//! because the mappings of every pack together are larger than the response+//! cap — `search` is a `503`, not `{"corpora":[]}`, and the failure is+//! remembered for a few seconds (`SEARCH_FIELDS_FAILURE_TTL`) so it is not recomputed by every+//! request. A caller that runs under a visibility rule of its own reads (and+//! shares) nothing from that cache.//! * `search` queries `xc-*`, not the plan's `xc-*-docs,xc-*-defs`. This engine//! resolves index-name wildcards only as `*`, `prefix*` or `*suffix`//! (`glob_match_simple`, `es_compat.rs`), so a `*` in the middle of a name@@ -103,7 +152,8 @@use std::collections::{HashMap, VecDeque};use std::net::IpAddr;use std::path::{Path as FsPath, PathBuf};-use std::sync::{Arc, Mutex};+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex, OnceLock};use std::time::{Duration, Instant, SystemTime};use axum::body::{Body, Bytes};@@ -178,6 +228,19 @@ const SEARCH_FIELDS: [&str; 3] = ["body", "defs", "symbol_text"];const SEARCH_POOL: usize = 100;/// How long a resolved `search` field list is reused.const SEARCH_FIELDS_TTL: Duration = Duration::from_secs(60);+/// How long a FAILED read of the `search` field list is remembered. Short, so a+/// transient fault clears itself quickly; not zero, because the usual cause is a+/// node-wide mapping too large to read (see [`HubShared::search_fields_blocking`])+/// and that read costs seconds every time it is repeated.+const SEARCH_FIELDS_FAILURE_TTL: Duration = Duration::from_secs(15);+/// The `size` an Elasticsearch-style search has when its body names none.+const DEFAULT_SEARCH_SIZE: u64 = 10;+/// How many times, in all, one hosted request may halve a search window because+/// the response did not fit the cap. Each try is a whole search that is thrown+/// away, so the budget is per request and not per search: a pipeline that+/// searches three times (`hybrid`) gets no more tries than one that searches+/// once. Seven halvings take the largest window (200) down to one.+const MAX_WINDOW_SHRINKS: usize = 7;/// Longest excerpt carried by a `search` best hit built from a docs record.const EXCERPT_CHARS: usize = 240;/// `_source` fields `search` asks the engine for — everything a best hit needs,@@ -212,21 +275,76 @@ pub struct HubShared {limiter: RateLimiter,slots: Arc<tokio::sync::Semaphore>,catalogue: Mutex<Option<CachedCatalogue>>,- search_fields: Mutex<Option<(Instant, Vec<&'static str>)>>,+ /// What the last read of the node-wide mappings found, and when. Shared by+ /// callers who run under no visibility rule of their own; see+ /// [`HubShared::search_fields_blocking`].+ search_fields: Mutex<Option<(Instant, FieldsRead)>>,+ tuning: Tuning,+ /// How many times the node-wide mappings were read for `search`.+ mapping_reads: AtomicU64,+}++/// The outcome of reading the node-wide mappings for `search`.+enum FieldsRead {+ Resolved(Vec<&'static str>),+ /// The read failed. The detail is for the log; the client gets the fixed 503.+ Failed(String),+}++/// Limits a test may lower so that a failure which needs 16 MiB in production+/// needs a few KiB in a test. Production always uses [`Tuning::default`]; there+/// is no way to set this over HTTP or from configuration.+#[doc(hidden)]+#[derive(Clone, Copy, Debug)]+pub struct Tuning {+ /// Cap on any single in-process response, in bytes.+ pub response_cap_bytes: usize,+ /// How long a failed read of the `search` field list is remembered.+ pub fields_failure_ttl: Duration,+ /// Called with the ledger root and the slug right after a `def`/`code`+ /// request's ledger passed the build-prefix check and before the shared+ /// pipeline reads it again: the instant at which a `xerj hub pull` landing+ /// mid-request would change what that second read finds.+ pub after_ledger_check: Option<fn(&FsPath, &str)>,+}++impl Default for Tuning {+ fn default() -> Self {+ Self {+ response_cap_bytes: MAX_INTERNAL_RESPONSE_BYTES,+ fields_failure_ttl: SEARCH_FIELDS_FAILURE_TTL,+ after_ledger_check: None,+ }+ }}impl HubShared {/// State for one router, over the node's [`AppState`].pub fn new(app: AppState) -> Arc<Self> {+ Self::with_tuning(app, Tuning::default())+ }++ /// Test seam: [`HubShared::new`] with lowered limits. Not reachable over+ /// HTTP.+ #[doc(hidden)]+ pub fn with_tuning(app: AppState, tuning: Tuning) -> Arc<Self> {Arc::new(Self {app,limiter: RateLimiter::new(RATE_LIMIT, RATE_WINDOW, MAX_TRACKED_CLIENTS),slots: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_QUERIES)),catalogue: Mutex::new(None),search_fields: Mutex::new(None),+ tuning,+ mapping_reads: AtomicU64::new(0),})}+ /// Test seam: how many times `search` has read the node-wide mappings.+ #[doc(hidden)]+ pub fn mapping_reads_for_test(&self) -> u64 {+ self.mapping_reads.load(Ordering::Relaxed)+ }+/// Test seam: take `n` query slots, as `n` slow queries would. Not/// reachable over HTTP.#[doc(hidden)]@@ -289,13 +407,15 @@ impl HubShared {}}- /// `Ok` when `slug` is a corpus the catalogue lists and permits serving,- /// else the response to send instead.+ /// The catalogue's rule for `slug` when it lists the corpus and permits+ /// serving it, else the response to send instead.////// Everything that is not a servable corpus — a malformed name, a name the/// catalogue does not list, a listed name with the `excluded` licence class- /// — is the same `404`: the caller learns nothing about which it was.- async fn resolve_corpus(self: &Arc<Self>, slug: &str) -> Result<(), Response> {+ /// — is the same `404`: the caller learns nothing about which it was. The+ /// rule is the catalogue's half of the licence decision; the corpus's own+ /// ledger is the other half, and [`corpus_facts`] takes the stricter.+ async fn resolve_corpus(self: &Arc<Self>, slug: &str) -> Result<CorpusRule, Response> {if !valid_slug(slug) {return Err(unknown_corpus());}@@ -315,7 +435,7 @@ impl HubShared {}};match index.corpora.get(slug) {- Some(rule) if rule.licence_class != LicenceClass::Excluded => Ok(()),+ Some(rule) if rule.licence_class != LicenceClass::Excluded => Ok(rule.clone()),_ => Err(unknown_corpus()),}}@@ -329,32 +449,64 @@ impl HubShared {/// union — the same gate the CLI applies — and the answer is reused for/// [`SEARCH_FIELDS_TTL`], since reading the mappings of every pack is far/// more expensive than a search. Blocking.- fn search_fields_blocking(&self, node: &InProcessNode) -> Vec<&'static str> {- {- let cache = self.search_fields.lock().unwrap_or_else(|e| e.into_inner());- if let Some((at, fields)) = cache.as_ref() {- if at.elapsed() < SEARCH_FIELDS_TTL {- return fields.clone();+ ///+ /// `Err` is a read that FAILED, and `search` answers it with the fixed 503:+ /// an unreadable mapping is an outage, not "no results". The usual cause is+ /// a node-wide mapping bigger than the cap on one in-process response+ /// (16 MiB: a few hundred wide indices will do it), and reading it costs+ /// seconds each time, so a failure is remembered for+ /// [`Tuning::fields_failure_ttl`] — short, so a transient fault clears+ /// itself — and the read is single-flight: the lock is held while the+ /// mappings are read, so a crowd of requests arriving together waits for one+ /// read instead of making sixteen.+ ///+ /// A caller running under a visibility rule of its own sees only its slice+ /// of the node, so what it reads is not shared and what others read is not+ /// reused for it: it neither reads nor writes the cache.+ fn search_fields_blocking(&self, node: &InProcessNode) -> Result<Vec<&'static str>, String> {+ if node.is_guarded() {+ return self.read_search_fields(node).map(Option::unwrap_or_default);+ }+ let mut cache = self.search_fields.lock().unwrap_or_else(|e| e.into_inner());+ if let Some((at, read)) = cache.as_ref() {+ match read {+ FieldsRead::Resolved(fields) if at.elapsed() < SEARCH_FIELDS_TTL => {+ return Ok(fields.clone());+ }+ FieldsRead::Failed(why) if at.elapsed() < self.tuning.fields_failure_ttl => {+ return Err(why.clone());}+ _ => {}}}- let path = format!("/{SEARCH_INDEX_EXPR}/_mapping");- let fields = match node.get_mapping(&path) {- Ok(mapping) => mapped_search_fields(&mapping),+ match self.read_search_fields(node) {+ Ok(Some(fields)) => {+ *cache = Some((Instant::now(), FieldsRead::Resolved(fields.clone())));+ Ok(fields)+ }// No `xc-*` index yet: nothing to search, and not an error. Not// remembered, so the first pack restored is searchable on the next// request rather than a minute later.- Err(e) if e.contains("__404__") => return Vec::new(),- // A failed read is not remembered either: one transient fault must- // not blank `search` for the whole TTL.- Err(e) => {- tracing::warn!(error = ?truncate_chars(&e, 500), "hub search: cannot read mappings");- return Vec::new();+ Ok(None) => Ok(Vec::new()),+ Err(why) => {+ *cache = Some((Instant::now(), FieldsRead::Failed(why.clone())));+ Err(why)}- };- *self.search_fields.lock().unwrap_or_else(|e| e.into_inner()) =- Some((Instant::now(), fields.clone()));- fields+ }+ }++ /// One read of the node-wide mappings, as the caller sees them. `Ok(None)`:+ /// there is no `xc-*` index at all.+ fn read_search_fields(+ &self,+ node: &InProcessNode,+ ) -> Result<Option<Vec<&'static str>>, String> {+ self.mapping_reads.fetch_add(1, Ordering::Relaxed);+ match node.get_mapping(&format!("/{SEARCH_INDEX_EXPR}/_mapping")) {+ Ok(mapping) => Ok(Some(mapped_search_fields(&mapping))),+ Err(e) if e.contains("__404__") => Ok(None),+ Err(e) => Err(format!("cannot read the node-wide mappings: {e}")),+ }}}@@ -614,17 +766,30 @@ async fn rate_limit(State(shared): State<Arc<HubShared>>, req: Request, next: Ne// The in-process XcHttp// ─────────────────────────────────────────────────────────────────────────────-/// Engine-level visibility for hosted queries: `xc-*` indices only.+/// Engine-level visibility for hosted queries: `xc-*` indices only, and only+/// those the caller could already see.////// Installed around every internal call. The engine consults it at the/// funnel every name passes through (`Engine::get_index`, `list_indices`), so/// an alias, a wildcard or a bug upstream of this module still cannot reach an/// index outside the hub namespace.-struct HubVisibility;+///+/// It is the INTERSECTION of two rules and a replacement for neither. The hub+/// namespace is one. The other is `outer`: the rule the caller's own request was+/// running under, which `authz_middleware` installs for every principal that is+/// not the superuser (an API key's grants). A key scoped to `tenant-a-*` is+/// refused `xc-mini-c-b1-docs` by the ordinary API, and must be refused it here+/// too — a hosted route is a different way to ask the engine, not a different+/// set of rights. With no `outer` the caller is unrestricted (the superuser, an+/// open-mode node, or a `hub.public` request, which skips authorization) and+/// only the namespace applies.+struct HubVisibility {+ outer: Option<Arc<dyn IndexVisibility>>,+}impl IndexVisibility for HubVisibility {fn visible(&self, index: &str) -> bool {- index.starts_with("xc-")+ index.starts_with("xc-") && self.outer.as_ref().is_none_or(|rule| rule.visible(index))}}@@ -662,18 +827,114 @@ fn hub_index_expr_ok(expr: &str) -> bool {pub struct InProcessNode {router: Router,rt: tokio::runtime::Handle,- /// The caller's `Authorization` header, forwarded only so the audit entry a- /// search writes names the right principal. The private router has no+ /// The caller's `Authorization` header, forwarded so the internal request+ /// carries the principal it came from. The private router has no/// authentication layer; nothing here grants anything.auth: Option<HeaderValue>,+ /// Cap on any single response read back from the engine, in bytes.+ cap: usize,+ /// The visibility rule the caller's own request was running under, captured+ /// on the request's task before the work moved to the blocking pool (a+ /// task-local does not follow a `spawn_blocking`). `None`: unrestricted.+ outer: Option<Arc<dyn IndexVisibility>>,+ /// When set, only index expressions that begin with this build prefix are+ /// sent: see [`InProcessNode::pin_to_build`].+ scope: OnceLock<String>,+ /// How many candidate records the last search that had to be cut back to fit+ /// the response cap was limited to; 0 when none was.+ cut_to: AtomicUsize,+ /// The most indices any one search of this request skipped because they+ /// failed (`_shards.failed`); 0 when none did.+ skipped: AtomicU64,+ /// Halvings of a search window this request may still spend+ /// ([`MAX_WINDOW_SHRINKS`] in all).+ shrinks_left: AtomicUsize,+}++/// Why an internal request produced no response.+enum SendError {+ /// The response was larger than the cap on one in-process response.+ TooLarge,+ /// Anything else; the text is for the log.+ Failed(String),+}++impl SendError {+ fn describe(self, cap: usize) -> String {+ match self {+ SendError::TooLarge => format!("internal response over {cap} bytes"),+ SendError::Failed(why) => why,+ }+ }}impl InProcessNode {- fn new(app: &AppState, auth: Option<HeaderValue>) -> Self {+ fn new(+ app: &AppState,+ auth: Option<HeaderValue>,+ cap: usize,+ outer: Option<Arc<dyn IndexVisibility>>,+ ) -> Self {Self {router: internal_router(app),rt: tokio::runtime::Handle::current(),auth,+ cap,+ outer,+ scope: OnceLock::new(),+ cut_to: AtomicUsize::new(0),+ skipped: AtomicU64::new(0),+ shrinks_left: AtomicUsize::new(MAX_WINDOW_SHRINKS),+ }+ }++ /// Spend one of this request's window halvings; `false` when none are left.+ fn take_shrink(&self) -> bool {+ self.shrinks_left+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1))+ .is_ok()+ }++ /// How many of the indices a search covered could not be searched and were+ /// skipped (see [`es_compat::InProcessSearch::skip_failed_indices`]).+ fn skipped_indices(&self) -> u64 {+ self.skipped.load(Ordering::Relaxed)+ }++ /// Is the caller running under a visibility rule of its own?+ fn is_guarded(&self) -> bool {+ self.outer.is_some()+ }++ /// Confine this node to ONE corpus build: from now on an index expression is+ /// sent only if every comma-separated part begins with `prefix`.+ ///+ /// `prefix` is the build prefix [`valid_build_prefix`] accepted. The `def`+ /// and `code` pipelines re-read the corpus's ledger themselves to decide+ /// what to query, so validating the ledger once, up front, would not stop a+ /// ledger that changed in between (a `xerj hub pull` landing mid-request)+ /// from aiming a query somewhere else. Pinning the node makes the validated+ /// prefix the only thing that request can address, whatever the pipeline+ /// then computes. Set once; a second call changes nothing.+ fn pin_to_build(&self, prefix: &str) {+ let _ = self.scope.set(prefix.to_string());+ }++ /// [`hub_index_expr_ok`], and inside the build this node is pinned to.+ fn expr_ok(&self, expr: &str) -> bool {+ hub_index_expr_ok(expr)+ && self.scope.get().is_none_or(|pinned| {+ expr.split(',')+ .all(|part| part.starts_with(pinned.as_str()))+ })+ }++ /// The number of candidate records the answer was cut back to because the+ /// full window did not fit the response cap, if it was.+ fn window_cut_to(&self) -> Option<usize> {+ match self.cut_to.load(Ordering::Relaxed) {+ 0 => None,+ n => Some(n),}}@@ -682,7 +943,7 @@ impl InProcessNode {method: Method,path: &str,body: Option<&Value>,- ) -> Result<(u16, Bytes), String> {+ ) -> Result<(u16, Bytes), SendError> {let mut builder = Request::builder().method(method).uri(path);if let Some(a) = &self.auth {builder = builder.header(header::AUTHORIZATION, a.clone());@@ -691,30 +952,40 @@ impl InProcessNode {Some(b) => builder.header(header::CONTENT_TYPE, "application/json").body(Body::from(- serde_json::to_vec(b).map_err(|e| e.to_string())?,+ serde_json::to_vec(b).map_err(|e| SendError::Failed(e.to_string()))?,)),None => builder.body(Body::empty()),}- .map_err(|e| format!("cannot build the internal request: {e}"))?;+ .map_err(|e| SendError::Failed(format!("cannot build the internal request: {e}")))?;let router = self.router.clone();+ let cap = self.cap;let call = async move {let resp = match router.oneshot(req).await {Ok(resp) => resp,Err(never) => match never {},};let status = resp.status().as_u16();- let bytes = axum::body::to_bytes(resp.into_body(), MAX_INTERNAL_RESPONSE_BYTES)+ let bytes = axum::body::to_bytes(resp.into_body(), cap).await- .map_err(|_| {- format!("internal response over {MAX_INTERNAL_RESPONSE_BYTES} bytes")- })?;- Ok::<_, String>((status, bytes))+ .map_err(|_| SendError::TooLarge)?;+ Ok::<_, SendError>((status, bytes))};+ // This search is the node talking to itself on behalf of a public route:+ // it is not audited, and one bad index must not fail it for the rest.+ let call = es_compat::scoped_in_process_search(+ es_compat::InProcessSearch {+ skip_audit: true,+ skip_failed_indices: true,+ },+ call,+ );+ let guard: Arc<dyn IndexVisibility> = Arc::new(HubVisibility {+ outer: self.outer.clone(),+ });// `block_on` is legal here: this runs on a blocking-pool thread, never// on an async worker. The visibility guard is a task-local, so it must// be installed inside the future this thread drives.- self.rt- .block_on(index_guard::scoped(Arc::new(HubVisibility), call))+ self.rt.block_on(index_guard::scoped(guard, call))}fn json(@@ -722,7 +993,7 @@ impl InProcessNode {method: Method,path: &str,body: Option<&Value>,- ) -> Result<(u16, Value), String> {+ ) -> Result<(u16, Value), SendError> {let (status, bytes) = self.send(method, path, body)?;let value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);Ok((status, value))@@ -734,9 +1005,12 @@ impl XcHttp for InProcessNode {let expr = path.strip_prefix('/').and_then(|p| p.strip_suffix("/_mapping"))- .filter(|e| hub_index_expr_ok(e))+ .filter(|e| self.expr_ok(e)).ok_or_else(|| "refused: not a hub mapping path".to_string())?;- match self.json(Method::GET, &format!("/{expr}/_mapping"), None)? {+ match self+ .json(Method::GET, &format!("/{expr}/_mapping"), None)+ .map_err(|e| e.describe(self.cap))?+ {(s, v) if (200..300).contains(&s) => Ok(v),(404, _) => Err("__404__".to_string()),(s, v) => Err(format!("HTTP {s}: {v}")),@@ -744,11 +1018,14 @@ impl XcHttp for InProcessNode {}fn cat_indices_json(&self, pattern: &str) -> Result<Vec<String>, String> {- if !hub_index_expr_ok(pattern) {+ if !self.expr_ok(pattern) {return Err("refused: not a hub index pattern".to_string());}let path = format!("/_cat/indices/{pattern}?format=json&h=index");- match self.json(Method::GET, &path, None)? {+ match self+ .json(Method::GET, &path, None)+ .map_err(|e| e.describe(self.cap))?+ {// A wildcard matching no index is "none", not an error.(404, _) => Ok(Vec::new()),(s, v) if (200..300).contains(&s) => v@@ -764,14 +1041,71 @@ impl XcHttp for InProcessNode {}}+ /// `POST /{index}/_search`, retried with a smaller `size` when the response+ /// does not fit the cap on one in-process response.+ ///+ /// The `code` pipeline asks for whole records (the passage is cut from the+ /// record's `body`) and for ten times the `k` it was asked for, up to 200.+ /// Two 9 MiB records therefore overflow a 16 MiB cap even at `k=1`, and a+ /// 503 for a question the corpus can answer is the wrong outcome. So an+ /// overflowing search is repeated with half the window until it fits, within+ /// a budget of [`MAX_WINDOW_SHRINKS`] halvings for the whole request (each+ /// try is a full search that is thrown away);+ /// [`InProcessNode::window_cut_to`] then reports that the answer was built+ /// from fewer candidates. A window of one that still does not fit, or an+ /// exhausted budget, is an error: there is no smaller answer to give.fn search(&self, index: &str, body: &Value) -> Result<Value, String> {- if !hub_index_expr_ok(index) {+ if !self.expr_ok(index) {return Err("refused: not a hub index expression".to_string());}- match self.json(Method::POST, &format!("/{index}/_search"), Some(body))? {- (s, v) if (200..300).contains(&s) => Ok(v),- (404, _) => Err("__404__".to_string()),- (s, v) => Err(format!("HTTP {s}: {v}")),+ let path = format!("/{index}/_search");+ let mut body = std::borrow::Cow::Borrowed(body);+ loop {+ match self.json(Method::POST, &path, Some(&body)) {+ Err(SendError::TooLarge) => {+ let size = body+ .get("size")+ .and_then(Value::as_u64)+ .unwrap_or(DEFAULT_SEARCH_SIZE);+ if size <= 1 || !self.take_shrink() {+ return Err(SendError::TooLarge.describe(self.cap));+ }+ let mut smaller = body.into_owned();+ smaller["size"] = json!((size / 2).max(1));+ body = std::borrow::Cow::Owned(smaller);+ }+ Err(e) => return Err(e.describe(self.cap)),+ Ok((s, v)) if (200..300).contains(&s) => {+ let failed = v+ .pointer("/_shards/failed")+ .and_then(Value::as_u64)+ .unwrap_or(0);+ if failed > 0 {+ self.skipped.fetch_max(failed, Ordering::Relaxed);+ }+ if matches!(body, std::borrow::Cow::Owned(_)) {+ let size = body+ .get("size")+ .and_then(Value::as_u64)+ .unwrap_or(DEFAULT_SEARCH_SIZE);+ let returned = v+ .pointer("/hits/hits")+ .and_then(Value::as_array)+ .map_or(0, Vec::len) as u64;+ // A window that came back FULL may have left matches+ // behind; one that did not returned everything there was.+ if returned >= size {+ self.cut_to.store(+ usize::try_from(size).unwrap_or(usize::MAX),+ Ordering::Relaxed,+ );+ }+ }+ return Ok(v);+ }+ Ok((404, _)) => return Err("__404__".to_string()),+ Ok((s, v)) => return Err(format!("HTTP {s}: {v}")),+ }}}}@@ -801,7 +1135,16 @@ where.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));return Err(resp);};- let node = InProcessNode::new(&shared.app, headers.get(header::AUTHORIZATION).cloned());+ // Captured HERE, on the request's own task, before the work moves to the+ // blocking pool: the caller's visibility rule is a task-local and does not+ // follow a `spawn_blocking`. Read inside the closure it would see nothing and+ // silently widen the caller to the whole `xc-*` namespace.+ let node = InProcessNode::new(+ &shared.app,+ headers.get(header::AUTHORIZATION).cloned(),+ shared.tuning.response_cap_bytes,+ index_guard::current(),+ );let job = tokio::task::spawn_blocking(move || {let _permit = permit;work(&node)@@ -839,6 +1182,27 @@ pub fn valid_slug(slug: &str) -> bool {&& bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'+' | b'-'))}+/// Is `prefix` exactly `xc-<slug>-b<digits>` — the build prefix of the corpus+/// named `slug` and of nothing else?+///+/// The state ledger is written by `xerj hub pull` from the pack's own manifest,+/// so its `index_prefix` is pack-supplied text. Every `code` and `def` query is+/// aimed at `<index_prefix>*` or `<index_prefix>-defs`, so an unchecked prefix is+/// a way to aim a hosted query at somebody else's indices: `xc-` is every hub+/// index on the node, `xc-mini-closed-b1` is another corpus's real build (the+/// excluded one, say), and a comma list is two expressions in one. The character+/// set [`hub_index_expr_ok`] allows does not stop any of those, so the shape does:+/// the corpus's OWN slug, then `-b`, then digits to the end — no wildcard, no+/// comma, no suffix, and no other corpus's name.+pub fn valid_build_prefix(prefix: &str, slug: &str) -> bool {+ valid_slug(slug)+ && prefix+ .strip_prefix("xc-")+ .and_then(|rest| rest.strip_prefix(slug))+ .and_then(|rest| rest.strip_prefix("-b"))+ .is_some_and(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()))+}+/// Collect the query string into a map, refusing a parameter this route does/// not take and a parameter given twice.fn collect_params(@@ -1090,18 +1454,27 @@ async fn def_route(Ok(k) => k,Err(why) => return bad_request(why),};- if let Err(resp) = shared.resolve_corpus(&slug).await {- return resp;- }+ let rule = match shared.resolve_corpus(&slug).await {+ Ok(rule) => rule,+ Err(resp) => return resp,+ };+ let after_check = shared.tuning.after_ledger_check;let outcome = run_query(&shared, &headers, move |node| {let root = hub_root();- let facts = match corpus_facts(&root, &slug) {+ let facts = match corpus_facts(&root, &slug, &rule) {Ok(f) => f,- Err(_) => return Outcome::NotLoaded,+ Err(LedgerError::NotLoaded) => return Outcome::NotLoaded,+ Err(LedgerError::Invalid(why)) => return Outcome::Failed(why),};if facts.licence_class == LicenceClass::Excluded {return Outcome::NotServed;}+ // Nothing this request sends may name an index outside the build the+ // ledger was just checked to describe, whatever `lookup_in` then reads.+ node.pin_to_build(&facts.index_prefix);+ if let Some(hook) = after_check {+ hook(&root, &slug);+ }// Mirror the CLI's refusals: a corpus the ledger knows but this node// holds no index for is "not loaded", not "no such definition".match node.cat_indices_json(&format!("{}*", facts.index_prefix)) {@@ -1121,6 +1494,21 @@ async fn def_route(return Outcome::Failed(w.clone());}answer.answered_from = AnsweredFrom::Hosted;+ // `lookup_in` describes the corpus from its ledger alone. This node+ // serves it under the stricter of the ledger and the catalogue, and the+ // header and every hit must say so (a `code` answer for the same corpus+ // does).+ answer.licence_class = facts.licence_class;+ answer.licence_spdx = facts.licence_spdx.clone();+ if let Some(note) = approach_only_note(&facts) {+ for hit in &mut answer.hits {+ hit.licence_warning = Some(note.clone());+ }+ }+ if let Some(window) = node.window_cut_to() {+ answer.truncated = true;+ answer.warnings.push(window_cut_warning(window));+ }Outcome::Done(Box::new(answer))}).await;@@ -1154,19 +1542,29 @@ async fn code_route(Ok(m) => m,Err(why) => return bad_request(why),};- if let Err(resp) = shared.resolve_corpus(&slug).await {- return resp;- }+ let rule = match shared.resolve_corpus(&slug).await {+ Ok(rule) => rule,+ Err(resp) => return resp,+ };let width = passage_width(max_tokens);+ let after_check = shared.tuning.after_ledger_check;let outcome = run_query(&shared, &headers, move |node| {let root = hub_root();- let facts = match corpus_facts(&root, &slug) {+ let facts = match corpus_facts(&root, &slug, &rule) {Ok(f) => f,- Err(_) => return Outcome::NotLoaded,+ Err(LedgerError::NotLoaded) => return Outcome::NotLoaded,+ Err(LedgerError::Invalid(why)) => return Outcome::Failed(why),};if facts.licence_class == LicenceClass::Excluded {return Outcome::NotServed;}+ // `run_code_query` reads the ledger again to decide what to query. Pin+ // the node to the build just validated, so nothing it computes can leave+ // that corpus.+ node.pin_to_build(&facts.index_prefix);+ if let Some(hook) = after_check {+ hook(&root, &slug);+ }let mut params = CodeParams::new(&slug, &query);params.k = k;params.mode = mode;@@ -1191,17 +1589,23 @@ async fn code_route(hit.licence_warning = Some(note.clone());}}- if mode != Mode::Bm25 {- warnings.push(- "mode semantic/hybrid falls back to BM25 when the corpus has no \- semantic_text mapping, and this response does not say which arms ran"- .to_string(),- );+ if let Some(note) = mode_warning(mode) {+ warnings.push(note.to_string());}- let truncated = match max_tokens {+ let mut truncated = match max_tokens {Some(budget) => apply_token_budget(&mut hits, budget, &mut warnings),None => false,};+ if let Some(window) = node.window_cut_to() {+ truncated = true;+ warnings.push(window_cut_warning(window));+ }+ // A miss must not read as "this code does not exist" when part+ // of the corpus could not be searched.+ let skipped = node.skipped_indices();+ if skipped > 0 {+ warnings.push(skipped_warning(skipped));+ }Outcome::Done(Box::new(facts.into_answer(AnswerMode::How,hits,@@ -1249,25 +1653,35 @@ async fn search_route(}let blocking_shared = shared.clone();let outcome = run_query(&shared, &headers, move |node| {- let fields = blocking_shared.search_fields_blocking(node);+ // A failed read of the mappings is an outage, not an empty node.+ let fields = blocking_shared.search_fields_blocking(node)?;if fields.is_empty() {return Ok(Vec::new());}// Only the top hits are read, never a total: with the fan-out over every// pack index, counting matches is work nobody asked for.+ //+ // `lenient`: the fan-out covers every pack on the node, and one pack that+ // maps `body` as a number must not fail a text query for all the others+ // (this engine answers a text query against a numeric field with a 400+ // unless the clause is lenient). An index that still fails is skipped+ // by the engine under [`es_compat::InProcessSearch`], not fatal.let body = json!({"size": SEARCH_POOL,"track_total_hits": false,"_source": SEARCH_SOURCE_FIELDS,- "query": { "multi_match": { "query": query, "fields": fields } },+ "query": { "multi_match": { "query": query, "fields": fields, "lenient": true } },});let resp = match node.search(SEARCH_INDEX_EXPR, &body) {Ok(r) => r,Err(e) if e.contains("__404__") => return Ok(Vec::new()),Err(e) => return Err(format!("search failed: {e}")),};+ let root = hub_root();let corpora = match &data.index {- Ok(index) => group_by_corpus(&resp, &index.corpora, k),+ Ok(index) => group_by_corpus(&resp, &index.corpora, k, |slug, rule| {+ servable_licence(&root, slug, rule)+ }),Err(_) => Vec::new(),};Ok(corpora)@@ -1300,6 +1714,11 @@ fn hub_root() -> PathBuf {/// pull wrote. Mirrors `xerj_hub::def`'s private `DefMeta` so a `def` answer/// and a `code` answer for the same corpus carry the same header; a test pins/// the two together.+///+/// `index_prefix` is the build prefix [`valid_build_prefix`] accepted, never+/// what the ledger merely said. `licence_class` and `licence_spdx` are the+/// effective licence ([`effective_licence`]): the stricter of the ledger's and+/// the catalogue's, not the ledger's alone.struct CorpusFacts {corpus: String,index_prefix: String,@@ -1310,20 +1729,143 @@ struct CorpusFacts {licence_class: LicenceClass,}-fn corpus_facts(root: &FsPath, slug: &str) -> Result<CorpusFacts, String> {- let st = ledger::load_state(root, slug)?;+/// Why a corpus's ledger cannot be used.+enum LedgerError {+ /// There is no readable ledger: the corpus is not loaded on this node.+ NotLoaded,+ /// A ledger that does not describe a servable pack. The detail is for the+ /// log; the client gets the fixed 503.+ Invalid(String),+}++/// The corpus's ledger, checked, and its licence reconciled with the+/// catalogue's `rule`.+///+/// The ledger is written by `xerj hub pull` from the pack's own manifest, so+/// nothing in it is trusted to say where to query: its build prefix must be+/// exactly `xc-<slug>-b<digits>` ([`valid_build_prefix`]) or the corpus is+/// refused before any query is built from it.+fn corpus_facts(root: &FsPath, slug: &str, rule: &CorpusRule) -> Result<CorpusFacts, LedgerError> {+ let st = ledger::load_state(root, slug).map_err(|_| LedgerError::NotLoaded)?;+ let index_prefix = ledger::query_prefix(&st);+ if !valid_build_prefix(&index_prefix, slug) {+ return Err(LedgerError::Invalid(format!(+ "the ledger for {slug} names the build prefix [{}], not xc-{slug}-b<digits>",+ truncate_chars(&index_prefix, 80)+ )));+ }let pin_date = st.pin_date.clone().unwrap_or_default();+ let (licence_class, licence_spdx) = effective_licence(+ (rule.licence_class, &rule.licence_spdx),+ (+ parse_licence_class(st.licence_class.as_deref()),+ st.licence_spdx.as_deref().unwrap_or_default(),+ ),+ );Ok(CorpusFacts {corpus: st.name(slug),- index_prefix: ledger::query_prefix(&st),+ index_prefix,age_days: age_days_from(&pin_date, st.indexed_at.as_deref()),pin_sha: st.pin_sha.clone().unwrap_or_default(),pin_date,- licence_spdx: st.licence_spdx.clone().unwrap_or_default(),- licence_class: parse_licence_class(st.licence_class.as_deref()),+ licence_spdx,+ licence_class,})}+/// How restrictive a class is: permissive < approach-only < excluded.+fn licence_rank(class: LicenceClass) -> u8 {+ match class {+ LicenceClass::AdaptWithAttribution => 0,+ LicenceClass::ApproachOnly => 1,+ LicenceClass::Excluded => 2,+ }+}++/// The licence a corpus is served under: the STRICTER of the catalogue's and the+/// ledger's class, each given as `(class, spdx)`.+///+/// Both must permit serving and either can refuse — the catalogue is what the+/// hub operator lists, the ledger is what this node's pack says about itself —+/// so neither may loosen the other: a catalogue entry that is servable does not+/// override a ledger that says `excluded`, and a ledger that says nothing (an+/// absent class reads as permissive) does not override a catalogue that says+/// `approach-only`. Every route takes its class from here, so `def`, `code` and+/// `search` cannot disagree about the same corpus.+///+/// The SPDX id travels with the class that decided: the stricter side's, falling+/// back to the other side's when it names none; on a tie the ledger's, because+/// that is what the pack on this node says it is.+pub fn effective_licence(+ catalogue: (LicenceClass, &str),+ ledger: (LicenceClass, &str),+) -> (LicenceClass, String) {+ let (decider, other) = if licence_rank(catalogue.0) > licence_rank(ledger.0) {+ (catalogue, ledger)+ } else {+ (ledger, catalogue)+ };+ let spdx = if decider.1.is_empty() {+ other.1+ } else {+ decider.1+ };+ (decider.0, spdx.to_string())+}++/// The effective licence for `search`, or `None` when the corpus must not be+/// served from it: the stricter class is `excluded`, or the corpus's ledger+/// cannot be used (not loaded here, or naming a build that is not its own) —+/// the same corpora `def` and `code` answer `404`/`503` for.+fn servable_licence(+ root: &FsPath,+ slug: &str,+ rule: &CorpusRule,+) -> Option<(LicenceClass, String)> {+ let facts = corpus_facts(root, slug, rule).ok()?;+ (facts.licence_class != LicenceClass::Excluded)+ .then_some((facts.licence_class, facts.licence_spdx))+}++/// What `mode` does on a hosted corpus, for the one warning that says so. The+/// shared query pipeline's JSON output does not report which arms ran, so the+/// answer cannot say; it says what each mode DOES instead, and the two modes+/// behave differently when the corpus has no `semantic_text` mapping for `body`.+fn mode_warning(mode: Mode) -> Option<&'static str> {+ match mode {+ Mode::Bm25 => None,+ Mode::Semantic => Some(+ "mode=semantic searches only the indices that map `body` as semantic_text; a corpus \+ with none returns no hits (it does not fall back to BM25), and this response does \+ not say which indices were searched",+ ),+ Mode::Hybrid => Some(+ "mode=hybrid combines BM25 with a vector search where the corpus maps `body` as \+ semantic_text and answers from BM25 alone where it does not; this response does not \+ say which arms ran",+ ),+ }+}++/// The warning for an answer from a corpus some of whose indices could not be+/// searched (the engine skipped them rather than fail the whole request).+fn skipped_warning(count: u64) -> String {+ format!(+ "{count} of this corpus's indices could not be searched and were skipped; the answer \+ is from the rest, so a missing result may be in one of them"+ )+}++/// The warning for an answer built from fewer candidate records than usual+/// because the full window did not fit the response cap.+fn window_cut_warning(window: usize) -> String {+ format!(+ "the matching records are too large to return together; the answer was built from the \+ top {window} candidate record(s) instead of the usual window, so lower-ranked matches \+ may be missing"+ )+}+impl CorpusFacts {fn into_answer(self,@@ -1620,10 +2162,26 @@ struct BestHit {}/// Group a `multi_match` reply by corpus: the best hit of each, best first-/// (ties by name), at most `want`. Hits from an index that belongs to no listed-/// corpus, and from an `excluded` one, are skipped.-fn group_by_corpus(resp: &Value, known: &HashMap<String, CorpusRule>, want: usize) -> Vec<BestHit> {+/// (ties by name), at most `want`.+///+/// A hit's index is attributed to a corpus by `known`, the WHOLE catalogue+/// (never a filtered subset: with `a` excluded and `a-b1` listed, the index+/// `xc-a-b1-b2-docs` is `a`'s, and attributing it to `a-b1` would serve an+/// excluded corpus's records under another name). What may then be served is+/// decided by `licence`, asked once per corpus: it returns the licence class and+/// SPDX id to serve the corpus under, or `None` to skip it. Hits from an index+/// that belongs to no listed corpus are skipped too.+fn group_by_corpus<F>(+ resp: &Value,+ known: &HashMap<String, CorpusRule>,+ want: usize,+ mut licence: F,+) -> Vec<BestHit>+where+ F: FnMut(&str, &CorpusRule) -> Option<(LicenceClass, String)>,+{let mut best: HashMap<&str, Hit> = HashMap::new();+ let mut licences: HashMap<&str, Option<(LicenceClass, String)>> = HashMap::new();let rows = resp.pointer("/hits/hits").and_then(Value::as_array)@@ -1639,7 +2197,14 @@ fn group_by_corpus(resp: &Value, known: &HashMap<String, CorpusRule>, want: usizlet Some(rule) = known.get(slug) else {continue;};- if rule.licence_class == LicenceClass::Excluded {+ let Some((class, spdx)) = licences+ .entry(slug)+ .or_insert_with(|| licence(slug, rule))+ .as_ref()+ else {+ continue;+ };+ if *class == LicenceClass::Excluded {continue;}let score = row.get("_score").and_then(Value::as_f64).unwrap_or(0.0) as f32;@@ -1655,8 +2220,8 @@ fn group_by_corpus(resp: &Value, known: &HashMap<String, CorpusRule>, want: usiz} else {doc_hit(src, score)};- if rule.licence_class == LicenceClass::ApproachOnly {- hit.licence_warning = Some(licence_note(&rule.licence_spdx));+ if *class == LicenceClass::ApproachOnly {+ hit.licence_warning = Some(licence_note(spdx));}best.insert(slug, hit);}@@ -1718,3 +2283,141 @@ fn doc_hit(src: &Value, score: f32) -> Hit {licence_warning: None,}}++#[cfg(test)]+mod tests {+ use super::*;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ /// The pin is what makes the validated build the ONLY thing a `def`/`code`+ /// request can address, whatever the shared pipeline computes from its own+ /// second read of a ledger that may have changed since it was checked.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn a_node_pinned_to_a_build_refuses_every_expression_outside_it() {+ let state = test_state();+ let pinned = InProcessNode::new(&state, None, 1 << 20, None);+ pinned.pin_to_build("xc-a-b1");+ let unpinned = InProcessNode::new(&state, None, 1 << 20, None);++ tokio::task::spawn_blocking(move || {+ let refused =+ |r: &Result<Value, String>| matches!(r, Err(e) if e.starts_with("refused"));+ let outside = [+ "xc-*",+ "xc-",+ "xc-b-b1*",+ "xc-b-b1-defs",+ "xc-a-b1*,xc-b-b1*",+ "xc-b-b1-docs,xc-a-b1-docs",+ "xc-a",+ "xc-a-b",+ ];+ for expr in outside {+ assert!(refused(&pinned.search(expr, &json!({}))), "search {expr}");+ assert!(+ pinned+ .cat_indices_json(expr)+ .is_err_and(|e| e.starts_with("refused")),+ "cat {expr}"+ );+ assert!(+ refused(&pinned.get_mapping(&format!("/{expr}/_mapping"))),+ "mapping {expr}"+ );+ // Without the pin the same expressions are well-formed hub names:+ // the pin, not the character set, is what refused them.+ if expr != "xc-" {+ assert!(+ !refused(&unpinned.search(expr, &json!({}))),+ "unpinned {expr} is a legal hub expression"+ );+ }+ }+ // Inside the build: well-formed, so the request reaches the engine,+ // which has no such index here (a 404, not a refusal).+ for expr in ["xc-a-b1*", "xc-a-b1-defs", "xc-a-b1-docs,xc-a-b1-defs"] {+ let r = pinned.search(expr, &json!({}));+ assert!(!refused(&r), "{expr}: {r:?}");+ assert!(pinned.cat_indices_json(expr).is_ok(), "cat {expr}");+ }+ })+ .await+ .expect("blocking task");+ }++ /// Every overflow retry is a whole search that is thrown away, so a request+ /// gets a fixed number of them however many searches its pipeline makes.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn a_request_has_a_fixed_budget_of_window_halvings() {+ let state = test_state();+ let node = InProcessNode::new(&state, None, 1 << 20, None);+ for n in 0..MAX_WINDOW_SHRINKS {+ assert!(node.take_shrink(), "halving {n} is within the budget");+ }+ assert!(!node.take_shrink(), "the budget is spent");+ assert!(!node.take_shrink(), "and stays spent");+ // Seven halvings are exactly what takes the widest window to one.+ assert_eq!(+ (0..MAX_WINDOW_SHRINKS).fold(200u64, |s, _| (s / 2).max(1)),+ 1+ );+ // A fresh request starts with a full budget.+ let next = InProcessNode::new(&state, None, 1 << 20, None);+ assert!(next.take_shrink());+ }++ #[test]+ fn effective_licence_is_the_stricter_class_with_the_deciding_spdx() {+ use LicenceClass::{AdaptWithAttribution as Adapt, ApproachOnly as Approach, Excluded};+ let el = effective_licence;+ // Agreement.+ assert_eq!(el((Adapt, "MIT"), (Adapt, "MIT")), (Adapt, "MIT".into()));+ assert_eq!(+ el((Approach, "GPL-3.0-only"), (Approach, "GPL-3.0-only")),+ (Approach, "GPL-3.0-only".into())+ );+ // Either side can be the stricter one, in every pair.+ assert_eq!(+ el((Adapt, "MIT"), (Approach, "AGPL")),+ (Approach, "AGPL".into())+ );+ assert_eq!(+ el((Approach, "GPL"), (Adapt, "MIT")),+ (Approach, "GPL".into())+ );+ assert_eq!(+ el((Adapt, "MIT"), (Excluded, "Closed")),+ (Excluded, "Closed".into())+ );+ assert_eq!(+ el((Excluded, "Closed"), (Adapt, "MIT")),+ (Excluded, "Closed".into())+ );+ assert_eq!(+ el((Approach, "GPL"), (Excluded, "Closed")),+ (Excluded, "Closed".into())+ );+ assert_eq!(+ el((Excluded, "Closed"), (Approach, "GPL")),+ (Excluded, "Closed".into())+ );+ // The stricter side names no licence: the other side's id is used.+ assert_eq!(el((Approach, "GPL"), (Adapt, "")), (Approach, "GPL".into()));+ assert_eq!(el((Adapt, "MIT"), (Approach, "")), (Approach, "MIT".into()));+ // A tie takes the ledger's id, else the catalogue's.+ assert_eq!(+ el((Adapt, "MIT"), (Adapt, "Apache-2.0")),+ (Adapt, "Apache-2.0".into())+ );+ assert_eq!(el((Adapt, "MIT"), (Adapt, "")), (Adapt, "MIT".into()));+ }+}diff --git a/engine/crates/xerj-api/tests/hub_api.rs b/engine/crates/xerj-api/tests/hub_api.rsindex 6108e07e8..0e18cedc0 100644--- a/engine/crates/xerj-api/tests/hub_api.rs+++ b/engine/crates/xerj-api/tests/hub_api.rs@@ -23,6 +23,7 @@use std::net::SocketAddr;use std::path::PathBuf;+use std::sync::atomic::{AtomicU32, Ordering};use std::sync::OnceLock;use std::time::{Duration, Instant};@@ -44,6 +45,33 @@ const ADMIN_KEY: &str = "hub-test-admin-key";const PIN_SHA: &str = "0774d05537f9762f838f7ab541b7765f1a729cb5";const PIN_DATE: &str = "2026-10-02";+/// `(slug, ledger index_prefix)`: pack-supplied build prefixes that are NOT+/// `xc-<that slug>-b<digits>`. Every one uses only characters a URL path can+/// carry, so the character-set guard alone does not stop them; each one is a way+/// to aim `<prefix>*` at somebody else's indices.+const HOSTILE_PREFIXES: [(&str, &str); 8] = [+ // `xc-*`: every hub index on the node.+ ("evil-bare", "xc-"),+ ("evil-wild", "xc-*"),+ // Another corpus's real build (the excluded one), and one no entry lists.+ ("evil-other", "xc-mini-closed-b1"),+ ("evil-orphan", "xc-orphan-b1"),+ // A comma list is two index expressions in one.+ ("evil-list", "xc-evil-list-b1,xc-mini-closed-b1"),+ // Not a build prefix: a whole index name, and a name with no build at all.+ ("evil-tail", "xc-evil-tail-b1-defs"),+ ("evil-nobuild", "xc-evil-nobuild"),+ ("evil-empty", ""),+];++/// Every slug whose ledger must be refused, `HOSTILE_PREFIXES` plus the one with+/// no `index_prefix` field.+fn hostile_slugs() -> Vec<&'static str> {+ let mut v: Vec<&'static str> = HOSTILE_PREFIXES.iter().map(|(s, _)| *s).collect();+ v.push("evil-legacy");+ v+}+// ─────────────────────────────────────────────────────────────────────────────// The shared fixture: ledgers + catalogue, written once// ─────────────────────────────────────────────────────────────────────────────@@ -56,6 +84,9 @@ struct Fixture {static FIXTURE: OnceLock<Fixture> = OnceLock::new();+/// Source of fresh client addresses for [`Hub::stranger`].+static NEXT_PEER: AtomicU32 = AtomicU32::new(1);+fn entry(slug: &str, spdx: &str, class: LicenceClass) -> CatalogueEntry {CatalogueEntry {slug: slug.into(),@@ -132,7 +163,67 @@ fn fixture() -> &'static Fixture {"MIT","adapt-with-attribution",);- let bytes = catalogue_bytes(vec![+ // Ledgers whose pack-supplied `index_prefix` is not `xc-<own slug>-b<digits>`.+ // Each is built from characters the URL guard accepts, and each points at+ // indices that belong to somebody else.+ for (slug, prefix) in HOSTILE_PREFIXES {+ led(slug, prefix, "MIT", "adapt-with-attribution");+ }+ // No `index_prefix` at all: `query_prefix` falls back to the ledger's own+ // `prefix`, which here is the bare namespace.+ {+ let state_dir = home.join("state");+ std::fs::create_dir_all(&state_dir).expect("state dir");+ std::fs::write(+ state_dir.join("evil-legacy.json"),+ format!(+ "{{\"corpus\":\"evil-legacy\",\"indexed_at\":\"2026-10-01T00:00:00Z\",\+ \"prefix\":\"xc-\",\"url\":\"http://x\",\"autoindex_exit\":0,\+ \"salvaged\":false,\"source\":\"hubpack\",\"pin_sha\":\"{PIN_SHA}\",\+ \"pin_date\":\"{PIN_DATE}\",\"licence_spdx\":\"MIT\",\+ \"licence_class\":\"adapt-with-attribution\"}}\n"+ ),+ )+ .expect("legacy ledger");+ }+ // The two sides of "either can refuse" the other way round, plus both+ // directions of "the stricter class wins".+ led(+ "cat-closed",+ "xc-cat-closed-b1",+ "MIT",+ "adapt-with-attribution",+ );+ led("gplish", "xc-gplish-b1", "MIT", "adapt-with-attribution");+ led(+ "ledger-gpl",+ "xc-ledger-gpl-b1",+ "AGPL-3.0-only",+ "approach-only",+ );+ // Corpora used by the failure-isolation tests: indices exist only on the+ // nodes of the tests that create them.+ led(+ "longbody",+ "xc-longbody-b1",+ "MIT",+ "adapt-with-attribution",+ );+ led("noindex", "xc-noindex-b1", "MIT", "adapt-with-attribution");+ led("big", "xc-big-b1", "MIT", "adapt-with-attribution");+ led(+ "bodyonly",+ "xc-bodyonly-b1",+ "MIT",+ "adapt-with-attribution",+ );+ // Well-formed until `rewrite_flip_ledger` replaces it mid-request.+ led("flip", "xc-flip-b1", "MIT", "adapt-with-attribution");+ // One corpus with an index that cannot be searched next to a good one,+ // and one whose every index cannot.+ led("mixed", "xc-mixed-b1", "MIT", "adapt-with-attribution");+ led("allbad", "xc-allbad-b1", "MIT", "adapt-with-attribution");+ let mut entries = vec![entry("mini-c", "MIT", LicenceClass::AdaptWithAttribution),entry("mini-gpl", "GPL-3.0-only", LicenceClass::ApproachOnly),entry("mini-closed", "LicenseRef-Closed", LicenceClass::Excluded),@@ -141,7 +232,25 @@ fn fixture() -> &'static Fixture {entry("badprefix", "MIT", LicenceClass::AdaptWithAttribution),// Listed, but no ledger and no indices on any test node.entry("ghost", "MIT", LicenceClass::AdaptWithAttribution),- ]);+ // The catalogue refuses it; this node's ledger would serve it.+ entry("cat-closed", "LicenseRef-Closed", LicenceClass::Excluded),+ // The catalogue says approach-only; the ledger says permissive.+ entry("gplish", "GPL-3.0-only", LicenceClass::ApproachOnly),+ // The catalogue says permissive; the ledger says approach-only.+ entry("ledger-gpl", "MIT", LicenceClass::AdaptWithAttribution),+ entry("longbody", "MIT", LicenceClass::AdaptWithAttribution),+ entry("noindex", "MIT", LicenceClass::AdaptWithAttribution),+ entry("big", "MIT", LicenceClass::AdaptWithAttribution),+ entry("bodyonly", "MIT", LicenceClass::AdaptWithAttribution),+ entry("flip", "MIT", LicenceClass::AdaptWithAttribution),+ entry("mixed", "MIT", LicenceClass::AdaptWithAttribution),+ entry("allbad", "MIT", LicenceClass::AdaptWithAttribution),+ entry("evil-legacy", "MIT", LicenceClass::AdaptWithAttribution),+ ];+ for (slug, _) in HOSTILE_PREFIXES {+ entries.push(entry(slug, "MIT", LicenceClass::AdaptWithAttribution));+ }+ let bytes = catalogue_bytes(entries);let catalogue = dir.join("catalogue.json");std::fs::write(&catalogue, &bytes).expect("catalogue file");std::env::set_var("XERJ_CODE_HOME", &home);@@ -324,6 +433,62 @@ impl Hub {self.send(req).await}+ /// A GET from a fresh client address each time, so the per-address rate+ /// limit never fires: for tests that send many requests and are about+ /// something else.+ async fn stranger(&self, path: &str) -> Reply {+ let n = NEXT_PEER.fetch_add(1, Ordering::Relaxed);+ let peer = format!("10.{}.{}.{}", (n >> 16) & 255, (n >> 8) & 255, n & 255);+ self.get_from(path, &peer, None).await+ }++ /// `stranger`, presenting `ApiKey <encoded>`.+ async fn stranger_with_key(&self, path: &str, encoded: &str) -> Reply {+ let n = NEXT_PEER.fetch_add(1, Ordering::Relaxed);+ let peer = format!("10.{}.{}.{}", (n >> 16) & 255, (n >> 8) & 255, n & 255);+ let mut req = Self::req("GET", path)+ .header(header::AUTHORIZATION, format!("ApiKey {encoded}"))+ .body(Body::empty())+ .unwrap();+ let addr: SocketAddr = format!("{peer}:40000").parse().expect("peer");+ req.extensions_mut().insert(ConnectInfo(addr));+ self.send(req).await+ }++ /// Mint an API key (as the admin) that may read only `patterns`; returns the+ /// `encoded` value to present as `ApiKey <encoded>`.+ async fn mint_read_key(&self, name: &str, patterns: &[&str]) -> String {+ let r = self+ .admin(+ "POST",+ "/_security/api_key",+ Some(&json!({+ "name": name,+ "role_descriptors": { "r": { "indices": [+ { "names": patterns, "privileges": ["read"] }+ ]}}+ })),+ )+ .await;+ assert!(r.status.is_success(), "mint {name}: {:?}", r.json());+ r.json()["encoded"]+ .as_str()+ .unwrap_or_else(|| panic!("no encoded key in {:?}", r.json()))+ .to_string()+ }++ /// A direct (non-hub) `POST /{index}/_search` presenting `ApiKey <encoded>`.+ async fn direct_search_with_key(&self, index: &str, encoded: &str) -> Reply {+ self.send(+ Self::req("POST", &format!("/{index}/_search"))+ .header(header::AUTHORIZATION, format!("ApiKey {encoded}"))+ .header(header::CONTENT_TYPE, "application/json")+ .body(Body::from(json!({"query": {"match_all": {}}}).to_string()))+ .unwrap(),+ )+ .await+ }+/// A request of any method as if it arrived from `peer`.async fn method_from(&self, method: &str, path: &str, peer: &str) -> Reply {let mut req = Self::req(method, path).body(Body::empty()).unwrap();@@ -560,13 +725,53 @@ async fn seed(h: &Hub) {.await;assert!(alias.status.is_success(), "alias: {:?}", alias.json());+ // ── corpora whose catalogue entry and node ledger disagree about the licence ──+ //+ // Each gets a docs index (`frobnicate …`) and a defs index, so every route+ // has something of theirs to find.+ for (slug, limit) in [+ ("ledger-closed", "LEDGER_CLOSED_LIMIT"),+ ("cat-closed", "CAT_CLOSED_LIMIT"),+ ("gplish", "GPLISH_LIMIT"),+ ("ledger-gpl", "LEDGER_GPL_LIMIT"),+ ] {+ h.create(&format!("xc-{slug}-b1-defs"), defs_mapping())+ .await;+ h.bulk(+ &format!("xc-{slug}-b1-defs"),+ &[def(+ limit,+ "define",+ "limits.h",+ 3,+ &format!("#define {limit} 9"),+ Some("9"),+ )],+ )+ .await;+ h.create(&format!("xc-{slug}-b1-docs"), docs_mapping())+ .await;+ h.bulk(+ &format!("xc-{slug}-b1-docs"),+ &[json!({"ax_path": format!("{slug}/s.c"),+ "body": format!("frobnicate {slug} text frobnicate"),+ "defs": format!("function {slug}_fn"),+ "symbols": [{"name": format!("{slug}_fn"), "kind": "function", "line": 1}]})],+ )+ .await;+ }+// Searchable now, not after the next refresh tick.let r = h.admin("POST","/xc-mini-c-b1-defs,xc-mini-c-b1-docs,xc-mini-gpl-b2-defs,xc-mini-gpl-b2-docs,\xc-mini-closed-b1-defs,xc-mini-closed-b1-docs,xc-orphan-b1-docs,\- xc-leaky-b1-docs,secret-notes/_refresh",+ xc-leaky-b1-docs,secret-notes,\+ xc-ledger-closed-b1-defs,xc-ledger-closed-b1-docs,\+ xc-cat-closed-b1-defs,xc-cat-closed-b1-docs,\+ xc-gplish-b1-defs,xc-gplish-b1-docs,\+ xc-ledger-gpl-b1-defs,xc-ledger-gpl-b1-docs/_refresh",None,).await;@@ -828,14 +1033,70 @@ async fn code_max_tokens_holds_the_answer_and_says_so() {}let r = h.get("/_hub/v1/code/mini-c?q=x&mode=nope").await;assert_eq!(r.status, StatusCode::BAD_REQUEST);- // The other arms are accepted; the answer warns that it cannot say which ran.- let r = h.get("/_hub/v1/code/mini-c?q=mini_hash&mode=hybrid").await;- assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());- assert!(r- .answer()- .warnings- .iter()- .any(|w| w.contains("semantic/hybrid")));+ // The other arms are accepted (what they do is pinned in+ // `semantic_and_hybrid_say_what_they_actually_do`).+ for mode in ["semantic", "hybrid"] {+ let r = h+ .get(&format!("/_hub/v1/code/mini-c?q=mini_hash&mode={mode}"))+ .await;+ assert_eq!(r.status, StatusCode::OK, "{mode}: {:?}", r.json());+ }+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn semantic_and_hybrid_say_what_they_actually_do() {+ // The fixture corpus maps `body` as plain text: it has no `semantic_text`+ // index for a vector arm to search.+ let h = hub(Opts::new(false, false)).await;+ let bm25 = h+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned")+ .await+ .answer();+ assert!(!bm25.hits.is_empty());+ assert!(bm25.warnings.is_empty(), "{:?}", bm25.warnings);++ // `semantic` over a corpus with no semantic index finds NOTHING; it does not+ // fall back to BM25 (that is what `hybrid` does). The answer must say so.+ let sem = h+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned&mode=semantic")+ .await;+ assert_eq!(sem.status, StatusCode::OK, "{:?}", sem.json());+ let sem = sem.answer();+ assert!(+ sem.hits.is_empty(),+ "the premise: semantic returns no hits here: {:#?}",+ sem.hits+ );+ let said = sem.warnings.join(" | ");+ assert!(!said.is_empty(), "an empty semantic answer explains itself");+ assert!(+ !said.contains("falls back to BM25"),+ "semantic does not fall back, and must not claim to: {said}"+ );+ assert!(+ said.contains("semantic_text") && said.contains("no hits"),+ "names why it is empty: {said}"+ );++ // `hybrid` over the same corpus answers from BM25 alone, and says that is+ // what happened.+ let hyb = h+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned&mode=hybrid")+ .await+ .answer();+ assert!(+ !hyb.hits.is_empty(),+ "hybrid degrades to BM25 and still answers"+ );+ let said = hyb.warnings.join(" | ");+ assert!(+ said.contains("BM25") && said.contains("semantic_text"),+ "names the arm it used and the mapping it lacked: {said}"+ );+ assert_ne!(+ sem.warnings, hyb.warnings,+ "the two modes behave differently, so they are described differently"+ );}#[tokio::test(flavor = "multi_thread", worker_threads = 4)]@@ -1216,10 +1477,28 @@ async fn unauthenticated_hub_traffic_cannot_write_the_audit_ring() {let h = hub(Opts::new(true, true)).await;let audit = &h.state.engine.audit;+ // The operator's evidence: whatever is in the ring before any stranger+ // arrives (the seeding above wrote real entries).+ let before = audit.next_seq();+ let evidence = audit.snapshot();+ assert!(+ !evidence.is_empty(),+ "fixture sanity: the ring holds the operator's own entries"+ );+ let unchanged = |what: &str| {+ assert_eq!(audit.next_seq(), before, "{what}: the sequence moved");+ let now = audit.snapshot();+ assert_eq!(now.len(), evidence.len(), "{what}: the ring changed size");+ assert_eq!(+ now.iter().map(|e| e.hash.as_str()).collect::<Vec<_>>(),+ evidence.iter().map(|e| e.hash.as_str()).collect::<Vec<_>>(),+ "{what}: the ring's entries changed"+ );+ };+// A flood of non-GET requests to the public paths and of 401s elsewhere// leaves nothing: the audit layer records unauthenticated write-shaped// requests, and it must not see these.- let before = audit.next_seq();// 4 rounds x 13 hub requests = 52, inside the 60-a-minute budget, so each// one reaches the layer that would have recorded it.for _ in 0..4 {@@ -1232,20 +1511,61 @@ async fn unauthenticated_hub_traffic_cannot_write_the_audit_ring() {h.get("/_cluster/health").await;h.get(FOUR[0]).await;}- assert_eq!(- audit.next_seq(),- before,- "no entry from unauthenticated hub traffic"- );+ unchanged("write-shaped and unauthenticated requests");++ // The reads themselves. Every valid `def`, `code` and `search` GET is served+ // by the node's own `_search` handler, which audits what it serves; the hub's+ // in-process call must not. Distinct client addresses, so the rate limit+ // does not stand in the way: this is what a crowd of strangers sends.+ let reads = [+ "/_hub/v1/def/mini-c/MINI_LIMIT",+ "/_hub/v1/code/mini-c?q=mini_hash+unsigned",+ "/_hub/v1/search?q=frobnicate",+ ];+ // More reads than the ring holds (4,096): had each one appended an entry,+ // not one of the operator's entries would be left.+ let mut sent = 0usize;+ for round in 0..1400 {+ for path in reads {+ let r = h.stranger(path).await;+ assert_eq!(r.status, StatusCode::OK, "round {round} {path}");+ sent += 1;+ }+ }+ assert!(sent > xerj_engine::audit::DEFAULT_AUDIT_CAPACITY, "{sent}");+ unchanged("4,200 anonymous def/code/search reads");- // A hosted search is the one thing that does append: the `_search` handler- // audits itself, unconditionally (documented in hub_api.rs).- h.get(FOUR[1]).await;+ // The same holds for a caller who does present the key: the hub's reads are+ // not a source of audit entries in either mode.+ for path in reads {+ assert_eq!(h.get_keyed(path).await.status, StatusCode::OK);+ }+ unchanged("keyed hub reads");++ // The suppression belongs to the hub's in-process call and nothing else: a+ // direct `_search` by the operator is still audited.+ let r = h+ .direct_search_with_key("xc-mini-c-b1-docs", ADMIN_KEY)+ .await;+ assert!(r.status.is_success(), "{:?}", r.json());let after = audit.snapshot();let new: Vec<_> = after.iter().filter(|e| e.seq >= before).collect();- assert!(- !new.is_empty() && new.iter().all(|e| e.op == "search"),- "{new:?}"+ assert_eq!(new.len(), 1, "exactly the direct search: {new:?}");+ assert_eq!(new[0].op, "search");+ assert_eq!(new[0].resource, "xc-mini-c-b1-docs");+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_keyed_node_does_not_audit_hub_reads_either() {+ let h = hub(Opts::new(false, true)).await;+ let before = h.state.engine.audit.next_seq();+ for path in FOUR {+ assert_eq!(h.get_keyed(path).await.status, StatusCode::OK, "{path}");+ }+ assert_eq!(+ h.state.engine.audit.next_seq(),+ before,+ "a hub read is not an audited data-path operation");}@@ -1684,6 +2004,905 @@ async fn a_full_house_is_a_503_with_retry_after_and_recovers() {assert_eq!(r.status(), StatusCode::NOT_FOUND);}+// ─────────────────────────────────────────────────────────────────────────────+// Responses too large for the in-process cap+// ─────────────────────────────────────────────────────────────────────────────++/// The hub's routes alone over `state`, with the in-process limits lowered.+fn lowered(+ state: &AppState,+ response_cap_bytes: usize,+ fields_failure_ttl: Duration,+) -> (axum::Router, std::sync::Arc<hub_api::HubShared>) {+ let shared = hub_api::HubShared::with_tuning(+ state.clone(),+ hub_api::Tuning {+ response_cap_bytes,+ fields_failure_ttl,+ ..Default::default()+ },+ );+ (hub_api::routes(shared.clone()), shared)+}++async fn get_on(app: &axum::Router, path: &str) -> Reply {+ let resp = app+ .clone()+ .oneshot(Request::get(path).body(Body::empty()).unwrap())+ .await+ .expect("response");+ let status = resp.status();+ let headers = resp.headers().clone();+ let body = axum::body::to_bytes(resp.into_body(), usize::MAX)+ .await+ .expect("body");+ Reply {+ status,+ headers,+ body,+ }+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_node_wide_mapping_over_the_cap_is_a_503_not_an_empty_search() {+ let h = hub(Opts::new(true, true)).await;+ // Premise: with the production cap the same node searches fine.+ let ok = h.stranger("/_hub/v1/search?q=frobnicate").await;+ assert_eq!(ok.status, StatusCode::OK);+ assert!(!ok.json()["corpora"].as_array().unwrap().is_empty());++ // What a pack with enormous mappings does at 16 MiB, in miniature: the+ // node-wide `xc-*` mapping is one byte too large for the cap on a single+ // in-process response (and everything else on the node still fits).+ let mapping = h.admin("GET", "/xc-*/_mapping", None).await;+ let cap = mapping.body.len() - 1;+ assert!(cap > 2048, "fixture sanity: a {cap}-byte mapping");+ let ttl = Duration::from_millis(600);+ let (app, shared) = lowered(&h.state, cap, ttl);++ let r = get_on(&app, "/_hub/v1/search?q=frobnicate").await;+ assert_eq!(+ r.status,+ StatusCode::SERVICE_UNAVAILABLE,+ "an unreadable mapping is an outage, not 'no results': {:?}",+ r.json()+ );+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));+ let text = String::from_utf8_lossy(&r.body).to_string();+ for leaked in [cap.to_string().as_str(), "bytes", "mapping", "xc-", "over"] {+ assert!(!text.contains(leaked), "leaked {leaked:?}: {text}");+ }+ assert_ne!(r.json(), json!({"corpora": []}));++ // The failure is remembered: a burst of requests does not repeat the+ // expensive read (each repeat costs seconds at the real size).+ for _ in 0..6 {+ let r = get_on(&app, "/_hub/v1/search?q=frobnicate").await;+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE);+ }+ assert_eq!(+ shared.mapping_reads_for_test(),+ 1,+ "seven requests, one read of the mappings"+ );++ // …but not for long: a transient fault clears itself, and the read is+ // tried again once the memory expires.+ tokio::time::sleep(ttl + Duration::from_millis(300)).await;+ let r = get_on(&app, "/_hub/v1/search?q=frobnicate").await;+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE);+ assert_eq!(+ shared.mapping_reads_for_test(),+ 2,+ "read again after the ttl"+ );++ // Routes about one corpus do not read the node-wide mapping and are not+ // taken down with `search`.+ let r = get_on(&app, "/_hub/v1/def/mini-c/MINI_LIMIT").await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn records_too_big_to_return_together_shrink_the_window_and_say_so() {+ let h = hub(Opts::new(false, false)).await;+ // Two records of ~48 KiB that both match; `code` asks the engine for whole+ // records, ten of them at once even for k=1.+ let filler = "int x; ".repeat(48 * 1024 / 7);+ h.create("xc-big-b1-docs", docs_mapping()).await;+ h.bulk(+ "xc-big-b1-docs",+ &[+ json!({"ax_path": "big/a.c", "title": "a.c",+ "body": format!("needleword alpha {filler}"), "defs": "function needleword"}),+ json!({"ax_path": "big/b.c", "title": "b.c",+ "body": format!("needleword beta {filler}"), "defs": "function needleword"}),+ ],+ )+ .await;+ h.admin("POST", "/xc-big-b1-docs/_refresh", None).await;++ // Premise: with the production cap both fit, and nothing is cut.+ let whole = h.get("/_hub/v1/code/big?q=needleword&k=1").await;+ assert_eq!(whole.status, StatusCode::OK, "{:?}", whole.json());+ let whole = whole.answer();+ assert_eq!(whole.hits.len(), 1);+ assert!(!whole.truncated, "{:?}", whole.warnings);+ assert!(whole.warnings.is_empty(), "{:?}", whole.warnings);++ // With the cap below two records (what two 9 MiB records do to 16 MiB), the+ // answer is built from fewer candidates and says so — not a 503.+ let (app, _shared) = lowered(&h.state, 64 * 1024, Duration::from_secs(15));+ for k in [1, 2, 5] {+ let r = get_on(&app, &format!("/_hub/v1/code/big?q=needleword&k={k}")).await;+ assert_eq!(+ r.status,+ StatusCode::OK,+ "k={k}: records that do not fit together must not be a 503: {:?}",+ r.json()+ );+ let a = r.answer();+ assert_eq!(a.corpus, "big");+ assert_eq!(a.hits.len(), 1, "k={k}: the one record that fits");+ assert!(+ a.hits[0].file == "big/a.c" || a.hits[0].file == "big/b.c",+ "{:?}",+ a.hits[0]+ );+ assert!(a.hits[0].score > 0.0);+ assert!(a.truncated, "k={k}: the answer is cut and says so");+ assert!(+ a.warnings.iter().any(|w| w.contains("too large")),+ "k={k}: {:?}",+ a.warnings+ );+ }++ // A single record over the cap cannot be returned at all, and that is a+ // fixed 503 rather than a made-up answer.+ let (tiny, _shared) = lowered(&h.state, 8 * 1024, Duration::from_secs(15));+ let r = get_on(&tiny, "/_hub/v1/code/big?q=needleword&k=1").await;+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE, "{:?}", r.json());+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));+}++// ─────────────────────────────────────────────────────────────────────────────+// What a pack's own ledger cannot talk the node into+// ─────────────────────────────────────────────────────────────────────────────++/// Text that lives only in somebody else's indices: the excluded corpus, an+/// index no catalogue entry lists, and the private index behind an alias.+const OTHER_PEOPLES_DATA: [&str; 6] = [+ "closed text",+ "orphan text",+ "CLOSED_LIMIT",+ "mini-closed/",+ "orphan/",+ "hunter2",+];++fn assert_no_leak(what: &str, r: &Reply) {+ let text = String::from_utf8_lossy(&r.body);+ for needle in OTHER_PEOPLES_DATA {+ assert!(+ !text.contains(needle),+ "{what} leaked {needle:?} (status {}): {text}",+ r.status+ );+ }+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_ledger_prefix_that_is_not_the_corpus_own_build_is_never_queried() {+ // A public node: the stranger is exactly who this guards against.+ let h = hub(Opts::new(true, true)).await;++ // Premises. A well-formed corpus answers; and the data a hostile prefix+ // would reach is really on this node (without these the test proves nothing).+ let ok = h.stranger("/_hub/v1/code/mini-c?q=frobnicate&k=20").await;+ assert_eq!(ok.status, StatusCode::OK, "{:?}", ok.json());+ assert!(!ok.answer().hits.is_empty());+ for (index, needle) in [+ ("xc-mini-closed-b1-docs", "closed text"),+ ("xc-orphan-b1-docs", "orphan text"),+ ("xc-mini-closed-b1-defs", "CLOSED_LIMIT"),+ ] {+ let r = h+ .admin(+ "POST",+ &format!("/{index}/_search"),+ Some(&json!({"query": {"match_all": {}}})),+ )+ .await;+ assert!(+ String::from_utf8_lossy(&r.body).contains(needle),+ "fixture sanity: {index} holds {needle:?}"+ );+ }++ // Every way a ledger can name a prefix that is not `xc-<its own slug>-b<n>`.+ // Each is refused before any query, with the same fixed 503, and none of+ // them reads another corpus.+ for slug in hostile_slugs() {+ for path in [+ format!("/_hub/v1/code/{slug}?q=frobnicate&k=20"),+ format!("/_hub/v1/code/{slug}?q=closed"),+ format!("/_hub/v1/code/{slug}?q=orphan"),+ format!("/_hub/v1/code/{slug}?q=frobnicate&mode=hybrid"),+ format!("/_hub/v1/def/{slug}/CLOSED_LIMIT"),+ format!("/_hub/v1/def/{slug}/MINI_LIMIT"),+ ] {+ let r = h.stranger(&path).await;+ assert_no_leak(&path, &r);+ assert_eq!(+ r.status,+ StatusCode::SERVICE_UNAVAILABLE,+ "{path}: {:?}",+ r.json()+ );+ assert_eq!(+ r.json()["error"]["type"],+ json!("hub_unavailable"),+ "{path}"+ );+ let text = String::from_utf8_lossy(&r.body).to_string();+ for leaked in ["xc-", "prefix", "ledger", "state", slug] {+ assert!(!text.contains(leaked), "{path} said {leaked:?}: {text}");+ }+ }+ }++ // `search` does not list them either, and still lists the honest corpora.+ let r = h.stranger("/_hub/v1/search?q=frobnicate").await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ assert_no_leak("search", &r);+ let slugs: Vec<String> = r.json()["corpora"]+ .as_array()+ .expect("corpora")+ .iter()+ .map(|c| c["slug"].as_str().unwrap().to_string())+ .collect();+ assert!(slugs.contains(&"mini-c".to_string()), "{slugs:?}");+ for slug in hostile_slugs() {+ assert!(!slugs.contains(&slug.to_string()), "{slug} in {slugs:?}");+ }+}++/// While armed, [`rewrite_flip_ledger`] does what a `xerj hub pull` landing at+/// the worst moment would.+static FLIP_ARMED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);++/// Replace the `flip` corpus's ledger with one naming the whole namespace. Run+/// by the hub right after the ledger passed its check and before the shared+/// query pipeline reads it a second time.+fn rewrite_flip_ledger(root: &std::path::Path, slug: &str) {+ if slug == "flip" && FLIP_ARMED.load(Ordering::SeqCst) {+ write_hubpack_state(+ root,+ slug,+ "http://x",+ "xc-",+ PIN_SHA,+ PIN_DATE,+ "MIT",+ "adapt-with-attribution",+ )+ .expect("rewrite the ledger");+ }+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_ledger_that_changes_after_it_was_checked_still_cannot_leave_its_build() {+ // The `def` and `code` pipelines read the ledger themselves, a second time,+ // to decide what to query. Checking the first read is not enough if the+ // second can differ (a `xerj hub pull` landing mid-request).+ let h = hub(Opts::new(true, true)).await;+ h.create("xc-flip-b1-docs", docs_mapping()).await;+ h.bulk(+ "xc-flip-b1-docs",+ &[json!({"ax_path": "flip/f.c", "body": "frobnicate flip text"})],+ )+ .await;+ h.admin("POST", "/xc-flip-b1-docs/_refresh", None).await;+ let shared = hub_api::HubShared::with_tuning(+ h.state.clone(),+ hub_api::Tuning {+ after_ledger_check: Some(rewrite_flip_ledger),+ ..Default::default()+ },+ );+ let app: axum::Router = hub_api::routes(shared);+ let ledger = std::path::PathBuf::from(std::env::var("XERJ_CODE_HOME").unwrap())+ .join("state")+ .join("flip.json");++ // Premise: with the ledger as written, the corpus answers from its own data.+ FLIP_ARMED.store(false, Ordering::SeqCst);+ let r = get_on(&app, "/_hub/v1/code/flip?q=frobnicate&k=20").await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ let hits = r.answer().hits;+ assert!(!hits.is_empty());+ assert!(+ hits.iter().all(|x| x.file.starts_with("flip/")),+ "{hits:#?}"+ );++ // Armed: the ledger names `xc-` by the time the pipeline reads it. Without+ // the pin that is every hub index; with it, a refusal.+ FLIP_ARMED.store(true, Ordering::SeqCst);+ for path in [+ "/_hub/v1/code/flip?q=frobnicate&k=20",+ "/_hub/v1/code/flip?q=closed",+ "/_hub/v1/def/flip/CLOSED_LIMIT",+ ] {+ let r = get_on(&app, path).await;+ assert_no_leak(path, &r);+ assert_eq!(+ r.status,+ StatusCode::SERVICE_UNAVAILABLE,+ "{path}: {:?}",+ r.json()+ );+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));+ }+ let on_disk = std::fs::read_to_string(&ledger).expect("ledger");+ assert!(+ on_disk.contains("\"index_prefix\":\"xc-\""),+ "premise: the ledger really was replaced mid-request: {on_disk}"+ );++ // Put it back for anything that runs after this.+ FLIP_ARMED.store(false, Ordering::SeqCst);+ write_hubpack_state(+ ledger.parent().and_then(|p| p.parent()).unwrap(),+ "flip",+ "http://x",+ "xc-flip-b1",+ PIN_SHA,+ PIN_DATE,+ "MIT",+ "adapt-with-attribution",+ )+ .expect("restore the ledger");+}++#[test]+fn only_the_exact_build_prefix_shape_is_accepted_for_a_ledger() {+ // `xc-<slug>-b<digits>` for the corpus's OWN slug, in every accepted shape:+ // digits of any length, slugs containing `-`, `.`, `_` and `+`.+ for (slug, prefix) in [+ ("mini-c", "xc-mini-c-b1"),+ ("mini-c", "xc-mini-c-b20260921"),+ ("mini-gpl", "xc-mini-gpl-b2"),+ ("rust-vulns", "xc-rust-vulns-b1700000000"),+ ("c++", "xc-c++-b3"),+ ("a.b_c", "xc-a.b_c-b0"),+ // A slug that itself ends in a build-like token is still that slug.+ ("lib-b2", "xc-lib-b2-b7"),+ ] {+ assert!(+ hub_api::valid_build_prefix(prefix, slug),+ "{prefix} for {slug}"+ );+ }+ for (slug, prefix) in [+ ("mini-c", "xc-"),+ ("mini-c", "xc-*"),+ ("mini-c", "xc-mini-c"),+ ("mini-c", "xc-mini-c-b"),+ ("mini-c", "xc-mini-c-b1x"),+ ("mini-c", "xc-mini-c-bx1"),+ ("mini-c", "xc-mini-c-b1-defs"),+ ("mini-c", "xc-mini-c-b1*"),+ ("mini-c", "xc-mini-c-b1,xc-mini-closed-b1"),+ ("mini-c", "xc-mini-c-b1 "),+ ("mini-c", " xc-mini-c-b1"),+ ("mini-c", "XC-mini-c-b1"),+ ("mini-c", "xc-MINI-C-b1"),+ ("mini-c", "xc-mini-gpl-b2"),+ ("mini-c", "xc-mini-closed-b1"),+ ("mini-c", "xc-mini-c-b-1"),+ ("mini-c", "xc-mini-c-b+1"),+ ("mini-c", "xc-mini-c-b١"),+ ("mini-c", ""),+ ("mini-c", "mini-c-b1"),+ ("mini", "xc-mini-c-b1"),+ ] {+ assert!(+ !hub_api::valid_build_prefix(prefix, slug),+ "{prefix:?} must not be accepted for {slug}"+ );+ }+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_scoped_key_sees_the_hub_through_its_own_grants_only() {+ // A keyed, non-public node: callers are principals with grants, and the hub+ // routes must not widen what a grant allows.+ let h = hub(Opts::new(false, true)).await;++ // A tenant key with nothing under `xc-*`.+ let tenant = h.mint_read_key("tenant-a", &["tenant-a-*"]).await;+ let direct = h.direct_search_with_key("xc-mini-c-b1-docs", &tenant).await;+ assert_eq!(+ direct.status,+ StatusCode::FORBIDDEN,+ "premise: the ordinary API refuses this key the corpus: {:?}",+ direct.json()+ );+ let r = h+ .stranger_with_key("/_hub/v1/code/mini-c?q=frobnicate", &tenant)+ .await;+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{:?}", r.json());+ assert_eq!(r.json()["error"]["type"], json!("corpus_not_loaded"));+ let r = h+ .stranger_with_key("/_hub/v1/def/mini-c/MINI_LIMIT", &tenant)+ .await;+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{:?}", r.json());+ assert_eq!(r.json()["error"]["type"], json!("corpus_not_loaded"));+ let r = h+ .stranger_with_key("/_hub/v1/search?q=frobnicate", &tenant)+ .await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ assert_eq!(+ r.json(),+ json!({"corpora": []}),+ "the key sees none of the hub, so `search` finds none of it"+ );+ for text in ["mini-c", "frobnicate the buffer", "MINI_LIMIT", "128"] {+ assert!(+ !String::from_utf8_lossy(&r.body).contains(text),+ "{text} reached a key that holds nothing under xc-*"+ );+ }++ // A key granted ONE corpus sees that corpus through the hub, and no other:+ // the hub narrows to its own namespace, it does not widen to it.+ let mini = h.mint_read_key("mini-reader", &["xc-mini-c-*"]).await;+ let ok = h.direct_search_with_key("xc-mini-c-b1-docs", &mini).await;+ assert!(ok.status.is_success(), "premise: granted: {:?}", ok.json());+ let no = h.direct_search_with_key("xc-mini-gpl-b2-docs", &mini).await;+ assert_eq!(no.status, StatusCode::FORBIDDEN, "premise: not granted");++ let r = h+ .stranger_with_key("/_hub/v1/code/mini-c?q=mini_hash+unsigned", &mini)+ .await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ assert!(!r.answer().hits.is_empty(), "its own corpus is answered");+ let r = h+ .stranger_with_key("/_hub/v1/def/mini-c/MINI_LIMIT", &mini)+ .await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ assert_eq!(r.answer().hits.len(), 1);+ for path in [+ "/_hub/v1/code/mini-gpl?q=frobnicate",+ "/_hub/v1/def/mini-gpl/GPL_LIMIT",+ ] {+ let r = h.stranger_with_key(path, &mini).await;+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{path}: {:?}", r.json());+ assert_eq!(r.json()["error"]["type"], json!("corpus_not_loaded"));+ }+ let r = h+ .stranger_with_key("/_hub/v1/search?q=frobnicate", &mini)+ .await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ let slugs: Vec<String> = r.json()["corpora"]+ .as_array()+ .unwrap()+ .iter()+ .map(|c| c["slug"].as_str().unwrap().to_string())+ .collect();+ assert_eq!(slugs, ["mini-c"], "only the granted corpus is searched");++ // The admin key is unrestricted, and it is the same hub: both corpora.+ let r = h.get_keyed("/_hub/v1/search?q=frobnicate").await;+ let all: Vec<String> = r.json()["corpora"]+ .as_array()+ .unwrap()+ .iter()+ .map(|c| c["slug"].as_str().unwrap().to_string())+ .collect();+ assert!(+ all.contains(&"mini-c".to_string()) && all.contains(&"mini-gpl".to_string()),+ "{all:?}"+ );+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_scoped_caller_neither_reads_nor_poisons_the_shared_field_list() {+ // `search` learns which of body/defs/symbol_text exist from the node's+ // mappings and reuses that for a minute. A caller who sees only part of the+ // node reads a different list; it must not be shared in either direction.+ async fn node() -> (Hub, String) {+ let h = hub(Opts::new(false, true)).await;+ h.create(+ "xc-bodyonly-b1-docs",+ json!({"properties": {"body": {"type": "text"}, "ax_path": {"type": "keyword"}}}),+ )+ .await;+ h.bulk(+ "xc-bodyonly-b1-docs",+ &[json!({"ax_path": "bodyonly/b.c", "body": "frobnicate quasar nebula"})],+ )+ .await;+ h.admin("POST", "/xc-bodyonly-b1-docs/_refresh", None).await;+ let key = h.mint_read_key("bodyonly", &["xc-bodyonly-*"]).await;+ (h, key)+ }+ let slugs = |r: &Reply| -> Vec<String> {+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ r.json()["corpora"]+ .as_array()+ .unwrap()+ .iter()+ .map(|c| c["slug"].as_str().unwrap().to_string())+ .collect()+ };++ // The operator searches first, so the shared list names all three fields; the+ // scoped key (whose only index maps `body`) must not borrow it.+ let (h, key) = node().await;+ assert!(slugs(&h.get_keyed("/_hub/v1/search?q=frobnicate").await).contains(&"mini-c".into()));+ let r = h+ .stranger_with_key("/_hub/v1/search?q=frobnicate+quasar", &key)+ .await;+ assert_eq!(+ slugs(&r),+ ["bodyonly"],+ "the scoped key finds its own corpus"+ );++ // The scoped key searches first; the operator must not inherit its view,+ // where only `body` exists: a definition name lives in `symbol_text`.+ let (h, key) = node().await;+ let r = h+ .stranger_with_key("/_hub/v1/search?q=frobnicate+quasar", &key)+ .await;+ assert_eq!(slugs(&r), ["bodyonly"]);+ let admin = slugs(&h.get_keyed("/_hub/v1/search?q=MINI_LIMIT").await);+ assert_eq!(+ admin.first().map(String::as_str),+ Some("mini-c"),+ "a definition is found through `symbol_text`: {admin:?}"+ );+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn the_stricter_of_the_catalogue_and_the_ledger_governs_every_route() {+ let h = hub(Opts::new(false, false)).await;+ let note = |spdx: &str| format!("approach-only licence ({spdx}): read, do not copy");++ // Premises: these corpora are really indexed, so an absence below is the+ // licence rule at work and not an empty fixture.+ for slug in ["ledger-closed", "cat-closed", "gplish", "ledger-gpl"] {+ let r = h+ .admin(+ "POST",+ &format!("/xc-{slug}-b1-docs/_search"),+ Some(&json!({"query": {"match": {"body": "frobnicate"}}})),+ )+ .await;+ assert!(+ String::from_utf8_lossy(&r.body).contains(&format!("{slug} text")),+ "fixture sanity: {slug} is indexed"+ );+ }++ // Either side can refuse: catalogue-servable but ledger-excluded, and the+ // other way round. Same 404 everywhere, and `search` never returns a hit.+ for slug in ["ledger-closed", "cat-closed"] {+ for path in [+ format!("/_hub/v1/def/{slug}/X"),+ format!("/_hub/v1/code/{slug}?q=frobnicate"),+ ] {+ let r = h.get(&path).await;+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{path}");+ assert_eq!(+ r.json()["error"]["type"],+ json!("resource_not_found_exception"),+ "{path}: refusals look alike"+ );+ }+ }+ for q in ["frobnicate", "ledger-closed", "cat-closed"] {+ let r = h.get(&format!("/_hub/v1/search?q={q}&k=10")).await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());+ let text = String::from_utf8_lossy(&r.body).to_string();+ assert!(+ !text.contains("ledger-closed") && !text.contains("cat-closed"),+ "search q={q} returned a refused corpus: {text}"+ );+ assert!(+ !text.contains("ledger-closed text") && !text.contains("cat-closed text"),+ "search q={q}: {text}"+ );+ }++ // Approach-only on EITHER side is approach-only: the header says so, the+ // licence it names is the one that decided, and every hit carries the note.+ for (slug, spdx, symbol) in [+ ("gplish", "GPL-3.0-only", "GPLISH_LIMIT"),+ ("ledger-gpl", "AGPL-3.0-only", "LEDGER_GPL_LIMIT"),+ ] {+ let d = h.get(&format!("/_hub/v1/def/{slug}/{symbol}")).await;+ assert_eq!(d.status, StatusCode::OK, "{:?}", d.json());+ let d = d.answer();+ assert_eq!(d.licence_class, LicenceClass::ApproachOnly, "{slug} def");+ assert_eq!(d.licence_spdx, spdx, "{slug} def");+ assert_eq!(d.hits.len(), 1, "{slug} def");+ assert_eq!(+ d.hits[0].licence_warning.as_deref(),+ Some(note(spdx).as_str())+ );++ let c = h.get(&format!("/_hub/v1/code/{slug}?q=frobnicate")).await;+ assert_eq!(c.status, StatusCode::OK, "{:?}", c.json());+ let c = c.answer();+ assert_eq!(c.licence_class, LicenceClass::ApproachOnly, "{slug} code");+ assert_eq!(c.licence_spdx, spdx, "{slug} code");+ assert!(!c.hits.is_empty(), "{slug} code");+ assert!(+ c.hits+ .iter()+ .all(|x| x.licence_warning.as_deref() == Some(note(spdx).as_str())),+ "{slug} code: {:#?}",+ c.hits+ );++ // `def` and `code` describe the corpus identically (the pin test's rule).+ assert_eq!(+ (&d.corpus, &d.licence_spdx, d.licence_class),+ (&c.corpus, &c.licence_spdx, c.licence_class)+ );++ let r = h.get("/_hub/v1/search?q=frobnicate&k=10").await;+ let rows = r.json()["corpora"].as_array().cloned().unwrap();+ let row = rows+ .iter()+ .find(|row| row["slug"] == slug)+ .unwrap_or_else(|| panic!("{slug} missing from search: {rows:?}"));+ assert_eq!(+ row["best_hit"]["licence_warning"],+ json!(note(spdx)),+ "{slug} search"+ );+ }++ // And the permissive corpora are untouched by all this.+ let c = h+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned")+ .await+ .answer();+ assert_eq!(c.licence_class, LicenceClass::AdaptWithAttribution);+ assert!(c.hits.iter().all(|x| x.licence_warning.is_none()));+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn one_bad_index_does_not_take_search_down_for_everyone() {+ let h = hub(Opts::new(true, true)).await;+ let slugs_of = |r: &Reply| -> Vec<String> {+ r.json()["corpora"]+ .as_array()+ .unwrap_or_else(|| panic!("no corpora in {:?}", r.json()))+ .iter()+ .map(|c| c["slug"].as_str().unwrap().to_string())+ .collect()+ };+ let baseline = h.stranger("/_hub/v1/search?q=frobnicate").await;+ assert_eq!(baseline.status, StatusCode::OK);+ let good = slugs_of(&baseline);+ assert!(good.contains(&"mini-c".to_string()), "{good:?}");++ // ── A corpus that maps `body` as a number ──────────────────────────────────+ // Its `defs` is ordinary text. A lenient clause skips the numeric field and+ // still searches `defs`, so this corpus is FOUND; a strict one fails the whole+ // index, and the corpus is missing.+ h.create(+ "xc-longbody-b1-docs",+ json!({"properties": {+ "body": {"type": "long"},+ "defs": {"type": "text"},+ "ax_path": {"type": "keyword"}}}),+ )+ .await;+ h.bulk(+ "xc-longbody-b1-docs",+ &[json!({"ax_path": "longbody/n.c", "body": 5, "defs": "function frobnicate numbers"})],+ )+ .await;+ h.admin("POST", "/xc-longbody-b1-docs/_refresh", None).await;+ // Premise: the engine itself refuses a text query against that field when the+ // clause is not lenient, and accepts it when it is — so the hub must ask.+ let strict = h+ .admin(+ "POST",+ "/xc-*/_search",+ Some(&json!({"query": {"multi_match": {+ "query": "frobnicate", "fields": ["body"], "lenient": false}}})),+ )+ .await;+ assert!(+ !strict.status.is_success(),+ "premise: a strict multi_match over a long field fails the fan-out: {:?}",+ strict.json()+ );+ for q in ["frobnicate", "frobnicate+buffer", "mini_clamp"] {+ let r = h.stranger(&format!("/_hub/v1/search?q={q}")).await;+ assert_eq!(+ r.status,+ StatusCode::OK,+ "q={q} with a long-mapped body somewhere on the node: {:?}",+ r.json()+ );+ }+ let with_long = slugs_of(&h.stranger("/_hub/v1/search?q=frobnicate").await);+ let as_set = |v: &[String]| v.iter().cloned().collect::<std::collections::BTreeSet<_>>();+ let mut want = as_set(&good);+ want.insert("longbody".to_string());+ assert_eq!(+ as_set(&with_long),+ want,+ "the honest corpora are answered exactly as before, and the numeric one is \+ searched through its text field (lenient), not dropped: {with_long:?}"+ );+ // A query that happens to be a number is no problem for the numeric field.+ let r = h.stranger("/_hub/v1/search?q=5").await;+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());++ // ── A corpus whose `body` cannot be searched at all ────────────────────────+ // Lenient does not help here (the engine refuses a NAMED unsearchable field),+ // so this is the failure the per-index isolation exists for.+ h.create(+ "xc-noindex-b1-docs",+ json!({"properties": {+ "body": {"type": "text", "index": false},+ "ax_path": {"type": "keyword"}}}),+ )+ .await;+ h.bulk(+ "xc-noindex-b1-docs",+ &[json!({"ax_path": "noindex/n.c", "body": "frobnicate hidden away"})],+ )+ .await;+ h.admin("POST", "/xc-noindex-b1-docs/_refresh", None).await;+ let direct = h+ .admin(+ "POST",+ "/xc-*/_search",+ Some(&json!({"query": {"multi_match": {+ "query": "frobnicate", "fields": ["body"], "lenient": true}}})),+ )+ .await;+ assert!(+ !direct.status.is_success(),+ "premise: even a lenient clause fails the whole fan-out on an unsearchable field: {:?}",+ direct.json()+ );+ let r = h.stranger("/_hub/v1/search?q=frobnicate").await;+ assert_eq!(+ r.status,+ StatusCode::OK,+ "one corpus it cannot search must not 503 the rest: {:?}",+ r.json()+ );+ let after = slugs_of(&r);+ assert_eq!(+ as_set(&after),+ want,+ "the bad corpus is skipped, the rest are kept: {after:?}"+ );+ assert!(!after.contains(&"noindex".to_string()));++ // The other routes are about one corpus each and are untouched.+ let r = h+ .stranger("/_hub/v1/code/mini-c?q=mini_hash+unsigned")+ .await;+ assert_eq!(r.status, StatusCode::OK);+ assert!(!r.answer().hits.is_empty());+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn code_skips_an_index_it_cannot_search_and_says_so() {+ let h = hub(Opts::new(true, true)).await;+ let unsearchable = json!({"properties": {+ "body": {"type": "text", "index": false},+ "ax_path": {"type": "keyword"}}});++ // `mixed`: one good index and one whose `body` cannot be searched.+ h.create("xc-mixed-b1-docs", docs_mapping()).await;+ h.bulk(+ "xc-mixed-b1-docs",+ &[json!({"ax_path": "mixed/good.c", "body": "frobnicate the good text"})],+ )+ .await;+ h.create("xc-mixed-b1-extra", unsearchable.clone()).await;+ h.bulk(+ "xc-mixed-b1-extra",+ &[json!({"ax_path": "mixed/bad.c", "body": "frobnicate the hidden text"})],+ )+ .await;+ h.admin("POST", "/xc-mixed-b1-docs,xc-mixed-b1-extra/_refresh", None)+ .await;+ let strict = h+ .admin(+ "POST",+ "/xc-mixed-b1*/_search",+ Some(&json!({"query": {"multi_match": {"query": "frobnicate", "fields": ["body"]}}})),+ )+ .await;+ assert!(+ !strict.status.is_success(),+ "premise: one unsearchable index fails the whole direct fan-out: {:?}",+ strict.json()+ );++ let r = h.stranger("/_hub/v1/code/mixed?q=frobnicate").await;+ assert_eq!(+ r.status,+ StatusCode::OK,+ "one bad index must not be a 503 for its corpus: {:?}",+ r.json()+ );+ let a = r.answer();+ assert!(!a.hits.is_empty());+ assert!(+ a.hits.iter().all(|x| x.file == "mixed/good.c"),+ "the answer is from the index that could be searched: {:#?}",+ a.hits+ );+ assert!(+ a.warnings+ .iter()+ .any(|w| w.starts_with("1 of this corpus's indices could not be searched")),+ "a partial answer says so: {:?}",+ a.warnings+ );+ // A corpus whose indices all search fine carries no such warning.+ let ok = h+ .stranger("/_hub/v1/code/mini-c?q=mini_hash+unsigned")+ .await+ .answer();+ assert!(ok.warnings.is_empty(), "{:?}", ok.warnings);++ // `allbad`: nothing could be searched, so there is no answer to give.+ h.create("xc-allbad-b1-docs", unsearchable).await;+ h.bulk(+ "xc-allbad-b1-docs",+ &[json!({"ax_path": "allbad/b.c", "body": "frobnicate hidden"})],+ )+ .await;+ h.admin("POST", "/xc-allbad-b1-docs/_refresh", None).await;+ let r = h.stranger("/_hub/v1/code/allbad?q=frobnicate").await;+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE, "{:?}", r.json());+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn when_every_searched_index_fails_search_says_so_instead_of_an_empty_answer() {+ // Nothing but an unsearchable corpus on the node: "no results" would be a lie.+ let h = bare(&Opts::new(true, true));+ h.create(+ "xc-noindex-b1-docs",+ json!({"properties": {+ "body": {"type": "text", "index": false},+ "ax_path": {"type": "keyword"}}}),+ )+ .await;+ h.bulk(+ "xc-noindex-b1-docs",+ &[json!({"ax_path": "noindex/n.c", "body": "frobnicate hidden away"})],+ )+ .await;+ h.admin("POST", "/xc-noindex-b1-docs/_refresh", None).await;+ let r = h.stranger("/_hub/v1/search?q=frobnicate").await;+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE, "{:?}", r.json());+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));+ let text = String::from_utf8_lossy(&r.body).to_string();+ for leaked in ["noindex", "body", "unsearchable", "xc-", "HTTP"] {+ assert!(!text.contains(leaked), "leaked {leaked:?}: {text}");+ }+}+// ─────────────────────────────────────────────────────────────────────────────// Pure helpers// ─────────────────────────────────────────────────────────────────────────────