Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(hub): remediate the PR #1403 adversarial-review findings in /_hub/v1

xerj-org6aeacecd97f8
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 8b83bb323..f4df0b160 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -10205,6 +10205,82 @@ fn search_actor_class(size: usize, index: &str) -> &'static str {
}
}
+/// How a search the node issues to ITSELF differs from one that arrived over
+/// HTTP. Only [`crate::hub_api`]'s in-process caller sets it, around the one
+/// internal request it makes; a request off the wire cannot carry it, because it
+/// is a task-local and not a header, a parameter or a body field.
+///
+/// Both effects exist because the hosted hub's routes are public and a pack's
+/// contents are not the operator's: neither may be a lever on the node.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub(crate) struct InProcessSearch {
+ /// Do not append this search to the audit ring. The ring is a bounded piece
+ /// of operator evidence shared by the whole node (4,096 entries); a public
+ /// route that appended one entry per request would let a crowd of strangers
+ /// age every earlier entry out of it. The HTTP access log still records the
+ /// request.
+ pub skip_audit: bool,
+ /// A fan-out over several indices skips an index whose search fails — it
+ /// is counted in `_shards.failed` and logged — instead of failing the whole
+ /// request. One pack with an unusable mapping must not take `search` down
+ /// for every other pack. When EVERY index fails the first failure is still
+ /// returned, so an outage never reads as "no results".
+ pub skip_failed_indices: bool,
+}
+
+tokio::task_local! {
+ static IN_PROCESS_SEARCH: InProcessSearch;
+}
+
+/// Run `fut` — the whole internal request, handler included — with `mode`
+/// installed. Same shape as `xerj_engine::index_guard::scoped`, and for the same
+/// reason: the handler reads it on the task that drives `fut`.
+pub(crate) async fn scoped_in_process_search<F>(mode: InProcessSearch, fut: F) -> F::Output
+where
+ F: std::future::Future,
+{
+ IN_PROCESS_SEARCH.scope(mode, fut).await
+}
+
+/// The mode the current request runs under; the default (a request off the
+/// wire, or engine-internal work) changes nothing.
+fn in_process_search() -> InProcessSearch {
+ IN_PROCESS_SEARCH.try_with(|m| *m).unwrap_or_default()
+}
+
+/// Bookkeeping for [`InProcessSearch::skip_failed_indices`]: how many indices a
+/// fan-out tried, how many failed, and the first failure to report if all did.
+#[derive(Default)]
+struct SkippedIndices {
+ searched: u32,
+ failed: u32,
+ first_failure: Option<Response>,
+}
+
+impl SkippedIndices {
+ /// One more index was attempted.
+ fn attempt(&mut self) {
+ self.searched += 1;
+ }
+
+ /// An attempted index failed with `why`; keep the first such response.
+ fn skip(&mut self, why: impl FnOnce() -> Response) {
+ self.failed += 1;
+ if self.first_failure.is_none() {
+ self.first_failure = Some(why());
+ }
+ }
+
+ /// The failure to answer with when nothing at all could be searched.
+ fn all_failed(self) -> Option<Response> {
+ if self.failed > 0 && self.failed == self.searched {
+ self.first_failure
+ } else {
+ None
+ }
+ }
+}
+
async fn search_impl(
state: AppState,
index: String,
@@ -10213,6 +10289,9 @@ async fn search_impl(
body: EsSearchJson,
) -> Response {
let started = Instant::now();
+ // Read once, before the first `.await`: what the in-process hub caller asked
+ // of this search. A request off the wire gets the default and is unaffected.
+ let in_process = in_process_search();
// Closed-index handling is deferred to after index resolution below so
// `ignore_unavailable` / `expand_wildcards` are honored per ES semantics.
@@ -12546,6 +12625,8 @@ async fn search_impl(
// and excess searches queue on that semaphore deadline-aware exactly as
// they did when this loop issued them one at a time.
let mut spawned_searches = Vec::with_capacity(index_names.len());
+ // Only used when an in-process caller asked to have failing indices skipped.
+ let mut skipped = SkippedIndices::default();
for idx_name in &index_names {
let idx = match state.engine.get_index(idx_name) {
Ok(i) => i,
@@ -12554,6 +12635,19 @@ async fn search_impl(
// Silently skip; behaves like an empty index.
continue;
}
+ if in_process.skip_failed_indices {
+ // An index that is listed but cannot be opened is one bad
+ // index, not a reason to refuse the rest.
+ tracing::warn!(
+ index = %idx_name,
+ error = ?e.to_string().chars().take(300).collect::<String>(),
+ "in-process search: index unavailable, skipping it"
+ );
+ skipped.attempt();
+ skipped.skip(|| ApiError::new(xerj_common::XerjError::from(e)).into_response());
+ shards_failed_count += 1;
+ continue;
+ }
let mut api_err = ApiError::new(xerj_common::XerjError::from(e));
// Byte-parity `reason` ("no such index [name]") is fixed by
// ES-compat tests and set unconditionally in `error.rs` — this
@@ -12651,6 +12745,7 @@ async fn search_impl(
}
idx.search(&req_clone).await
}));
+ skipped.attempt();
spawned_searches.push((idx_name, search_task));
}
@@ -12663,9 +12758,29 @@ async fn search_impl(
match search_result {
Err(join_err) => {
+ if in_process.skip_failed_indices {
+ tracing::warn!(index = %idx_name, "in-process search: index search panicked, skipping it");
+ skipped.skip(|| search_task_panic_response(join_err, idx_name));
+ shards_failed_count += 1;
+ continue;
+ }
return search_task_panic_response(join_err, idx_name);
}
- Ok(Err(e)) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
+ Ok(Err(e)) => {
+ if in_process.skip_failed_indices {
+ // The engine's own words can quote the query; bound and
+ // escape them before they reach the log.
+ tracing::warn!(
+ index = %idx_name,
+ error = ?e.to_string().chars().take(300).collect::<String>(),
+ "in-process search: index search failed, skipping it"
+ );
+ skipped.skip(|| ApiError::new(xerj_common::XerjError::from(e)).into_response());
+ shards_failed_count += 1;
+ continue;
+ }
+ return ApiError::new(xerj_common::XerjError::from(e)).into_response();
+ }
Ok(Ok(result)) => {
// A Painless resource limit tripped somewhere in scoring or
// aggregation. Those paths have no error channel, so they
@@ -12750,6 +12865,13 @@ async fn search_impl(
}
}
+ // Skipping a bad index is isolation, not a way to hide an outage: when every
+ // index that was tried failed there is nothing to answer from, and the first
+ // failure is the answer.
+ if let Some(failure) = skipped.all_failed() {
+ return failure;
+ }
+
// ── Post-merge `auto_date_histogram` re-coordination ────────────────────
// Each per-shard run chose its own interval against its local min/max,
// so a two-shard search with one doc per shard can end up with two
@@ -13315,29 +13437,35 @@ async fn search_impl(
// "anonymous" until issue #329 — the API-key operations had been passing a
// real `principal.label()` since #201, so the plumbing existed and this one
// site simply did not use it.
- state.engine.audit.append_as(
- // #1109: the machine/user split is decided HERE, server-side, from
- // what the request itself is — never from anything a client can set
- // (the spoofable `X-Xerj-Internal` header was considered and
- // rejected during #1107, and that decision stands). See
- // `search_actor_class` for exactly which requests are machine.
- search_actor_class(body.size, index.as_str()),
- "search",
- principal.label(),
- index.as_str(),
- "ok",
- // `size=` is the requested page size. A `size:0` search asks for no
- // hits — it is a counting probe (autoindex's verification reads back
- // exact group counts that way), not someone looking for documents,
- // and `xerj gain` needs exactly that distinction to report a hit
- // rate that is about people (#1105).
- &format!(
- "took={}ms hits={} size={}",
- took_ms,
- merged_hits.len(),
- body.size
- ),
- );
+ //
+ // Not for a search the hosted hub issues to itself (`InProcessSearch`): that
+ // is a public read, and one audit entry per request would let anyone age the
+ // operator's evidence out of the ring.
+ if !in_process.skip_audit {
+ state.engine.audit.append_as(
+ // #1109: the machine/user split is decided HERE, server-side, from
+ // what the request itself is — never from anything a client can set
+ // (the spoofable `X-Xerj-Internal` header was considered and
+ // rejected during #1107, and that decision stands). See
+ // `search_actor_class` for exactly which requests are machine.
+ search_actor_class(body.size, index.as_str()),
+ "search",
+ principal.label(),
+ index.as_str(),
+ "ok",
+ // `size=` is the requested page size. A `size:0` search asks for no
+ // hits — it is a counting probe (autoindex's verification reads back
+ // exact group counts that way), not someone looking for documents,
+ // and `xerj gain` needs exactly that distinction to report a hit
+ // rate that is about people (#1105).
+ &format!(
+ "took={}ms hits={} size={}",
+ took_ms,
+ merged_hits.len(),
+ body.size
+ ),
+ );
+ }
// ES semantics: when the sort has any non-score key, `max_score` is null
// (scores aren't tracked during field-sorted execution unless
diff --git a/engine/crates/xerj-api/src/hub_api.rs b/engine/crates/xerj-api/src/hub_api.rs
index b1cf4422b..a31b22a8b 100644
--- a/engine/crates/xerj-api/src/hub_api.rs
+++ b/engine/crates/xerj-api/src/hub_api.rs
@@ -65,30 +65,79 @@
//! * the private router offers three read routes and nothing else, which is a
//! smaller surface than "the admin key against the whole API".
//!
-//! Three further guards bound what a hosted query can touch, even if a bug let
-//! an unexpected name through: [`InProcessNode`] refuses any index expression
-//! that is not made of `xc-` names in a conservative character set, every call
-//! runs under an engine-level [`index_guard`] that hides every index outside
-//! `xc-*`, and the corpus name is checked against the catalogue before any
-//! filesystem path is built from it.
+//! Four further guards bound what a hosted query can touch, even if a bug let
+//! an unexpected name through:
+//!
+//! * [`InProcessNode`] refuses any index expression that is not made of `xc-`
+//! names in a conservative character set;
+//! * every call runs under an engine-level [`index_guard`] that hides every
+//! index outside `xc-*` AND every index the caller's own request could not
+//! see. `HubVisibility` is the INTERSECTION of the hub namespace and the rule
+//! `authz_middleware` installed for the caller, captured before the work leaves
+//! the request's task: a key scoped to `tenant-a-*` is refused a corpus here
+//! exactly as it is by `_search`, and a hosted route is never a way to widen a
+//! grant;
+//! * the ledger's `index_prefix` is pack-supplied text, so it is checked against
+//! the exact shape `xc-<slug>-b<digits>` for the corpus being asked about
+//! ([`valid_build_prefix`]) before any query is built from it, and a `def` or
+//! `code` request is then pinned to that build (`InProcessNode::pin_to_build`):
+//! the node refuses an index expression outside it, whatever the shared
+//! pipeline goes on to compute from its own second read of the ledger;
+//! * the corpus name is checked against the catalogue before any filesystem path
+//! is built from it.
+//!
+//! The licence is decided the same way on every route: the STRICTER of the
+//! catalogue's class and the corpus's own ledger's ([`effective_licence`]); either
+//! can refuse (`excluded`), and either can require the read-do-not-copy warning
+//! (`approach-only`).
//!
//! The `XcHttp` methods are synchronous and the handlers async, so each query
//! runs on the blocking pool (`run_query`) under a global concurrency cap
//! ([`MAX_CONCURRENT_QUERIES`], `503` + `Retry-After` when full) and a wall
//! clock timeout ([`QUERY_TIMEOUT`], `504`).
//!
+//! ## What the in-process call changes about the search it makes
+//!
+//! The hub's call sets `es_compat::InProcessSearch`, a task-local that a request
+//! off the wire cannot carry, and nothing else does:
+//!
+//! * **It is not audited.** The node's `_search` handler appends every search to
+//! the audit ring — 4,096 entries shared by the whole node — so one entry per
+//! public request would let a crowd of strangers age the operator's evidence out
+//! of it. The HTTP access log is the record of hub reads. (The audit LAYER in
+//! `router.rs` is skipped for the public routes for the same reason.)
+//! * **A failing index is skipped, not fatal.** `search` fans out over every pack
+//! on the node, and the engine used to fail the whole request on the first index
+//! that errored: one pack with an unusable mapping took `search` down for every
+//! other pack. The index is skipped (logged, counted in `_shards.failed`); when
+//! EVERY index fails the first failure is the answer, so an outage is a `503`
+//! and never an empty list. `search` also sends `lenient`, which removes the
+//! commonest cause (a pack that maps `body` as a number) at the source.
+//! * **An oversized response shrinks the window.** One in-process response is
+//! capped at 16 MiB (`MAX_INTERNAL_RESPONSE_BYTES`). `code` fetches whole records, ten
+//! times the `k` asked for, so two very large records overflow it even at
+//! `k=1`; the search is retried with half the window until it fits, and the
+//! answer says it was built from fewer candidates (`truncated: true`). A
+//! mapping read that overflows is a different matter — see `search` below.
+//!
//! ## Known limits, stated so they are not rediscovered
//!
-//! * Every hosted search appends an entry to the node's audit ring (the
-//! `_search` handler does so unconditionally). On a dedicated hub node that is
-//! harmless; on a node that also holds data worth auditing, a flood of
-//! rate-limited-but-distinct clients can age out older entries.
//! * `Answer.answered_from` is always `hosted` here: that is where the answer
//! came from as far as the client is concerned, even though the server read a
//! local pack.
//! * The shared query pipeline's JSON output does not report which arms ran for
//! `mode=semantic|hybrid` (its "BM25 only" note is prose-only), so those
-//! modes carry a warning saying so instead of a silent fallback.
+//! modes carry a warning saying what each one DOES instead: `semantic` over a
+//! corpus with no `semantic_text` mapping returns no hits (it does not fall
+//! back), `hybrid` answers from BM25 alone.
+//! * `search` reads the node-wide mappings to learn which of `body`, `defs` and
+//! `symbol_text` exist (this engine collapses a multi-token query to zero hits
+//! when a named field is mapped nowhere). If that read fails — typically
+//! because the mappings of every pack together are larger than the response
+//! cap — `search` is a `503`, not `{"corpora":[]}`, and the failure is
+//! remembered for a few seconds (`SEARCH_FIELDS_FAILURE_TTL`) so it is not recomputed by every
+//! request. A caller that runs under a visibility rule of its own reads (and
+//! shares) nothing from that cache.
//! * `search` queries `xc-*`, not the plan's `xc-*-docs,xc-*-defs`. This engine
//! resolves index-name wildcards only as `*`, `prefix*` or `*suffix`
//! (`glob_match_simple`, `es_compat.rs`), so a `*` in the middle of a name
@@ -103,7 +152,8 @@
use std::collections::{HashMap, VecDeque};
use std::net::IpAddr;
use std::path::{Path as FsPath, PathBuf};
-use std::sync::{Arc, Mutex};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
use axum::body::{Body, Bytes};
@@ -178,6 +228,19 @@ const SEARCH_FIELDS: [&str; 3] = ["body", "defs", "symbol_text"];
const SEARCH_POOL: usize = 100;
/// How long a resolved `search` field list is reused.
const SEARCH_FIELDS_TTL: Duration = Duration::from_secs(60);
+/// How long a FAILED read of the `search` field list is remembered. Short, so a
+/// transient fault clears itself quickly; not zero, because the usual cause is a
+/// node-wide mapping too large to read (see [`HubShared::search_fields_blocking`])
+/// and that read costs seconds every time it is repeated.
+const SEARCH_FIELDS_FAILURE_TTL: Duration = Duration::from_secs(15);
+/// The `size` an Elasticsearch-style search has when its body names none.
+const DEFAULT_SEARCH_SIZE: u64 = 10;
+/// How many times, in all, one hosted request may halve a search window because
+/// the response did not fit the cap. Each try is a whole search that is thrown
+/// away, so the budget is per request and not per search: a pipeline that
+/// searches three times (`hybrid`) gets no more tries than one that searches
+/// once. Seven halvings take the largest window (200) down to one.
+const MAX_WINDOW_SHRINKS: usize = 7;
/// Longest excerpt carried by a `search` best hit built from a docs record.
const EXCERPT_CHARS: usize = 240;
/// `_source` fields `search` asks the engine for — everything a best hit needs,
@@ -212,21 +275,76 @@ pub struct HubShared {
limiter: RateLimiter,
slots: Arc<tokio::sync::Semaphore>,
catalogue: Mutex<Option<CachedCatalogue>>,
- search_fields: Mutex<Option<(Instant, Vec<&'static str>)>>,
+ /// What the last read of the node-wide mappings found, and when. Shared by
+ /// callers who run under no visibility rule of their own; see
+ /// [`HubShared::search_fields_blocking`].
+ search_fields: Mutex<Option<(Instant, FieldsRead)>>,
+ tuning: Tuning,
+ /// How many times the node-wide mappings were read for `search`.
+ mapping_reads: AtomicU64,
+}
+
+/// The outcome of reading the node-wide mappings for `search`.
+enum FieldsRead {
+ Resolved(Vec<&'static str>),
+ /// The read failed. The detail is for the log; the client gets the fixed 503.
+ Failed(String),
+}
+
+/// Limits a test may lower so that a failure which needs 16 MiB in production
+/// needs a few KiB in a test. Production always uses [`Tuning::default`]; there
+/// is no way to set this over HTTP or from configuration.
+#[doc(hidden)]
+#[derive(Clone, Copy, Debug)]
+pub struct Tuning {
+ /// Cap on any single in-process response, in bytes.
+ pub response_cap_bytes: usize,
+ /// How long a failed read of the `search` field list is remembered.
+ pub fields_failure_ttl: Duration,
+ /// Called with the ledger root and the slug right after a `def`/`code`
+ /// request's ledger passed the build-prefix check and before the shared
+ /// pipeline reads it again: the instant at which a `xerj hub pull` landing
+ /// mid-request would change what that second read finds.
+ pub after_ledger_check: Option<fn(&FsPath, &str)>,
+}
+
+impl Default for Tuning {
+ fn default() -> Self {
+ Self {
+ response_cap_bytes: MAX_INTERNAL_RESPONSE_BYTES,
+ fields_failure_ttl: SEARCH_FIELDS_FAILURE_TTL,
+ after_ledger_check: None,
+ }
+ }
}
impl HubShared {
/// State for one router, over the node's [`AppState`].
pub fn new(app: AppState) -> Arc<Self> {
+ Self::with_tuning(app, Tuning::default())
+ }
+
+ /// Test seam: [`HubShared::new`] with lowered limits. Not reachable over
+ /// HTTP.
+ #[doc(hidden)]
+ pub fn with_tuning(app: AppState, tuning: Tuning) -> Arc<Self> {
Arc::new(Self {
app,
limiter: RateLimiter::new(RATE_LIMIT, RATE_WINDOW, MAX_TRACKED_CLIENTS),
slots: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_QUERIES)),
catalogue: Mutex::new(None),
search_fields: Mutex::new(None),
+ tuning,
+ mapping_reads: AtomicU64::new(0),
})
}
+ /// Test seam: how many times `search` has read the node-wide mappings.
+ #[doc(hidden)]
+ pub fn mapping_reads_for_test(&self) -> u64 {
+ self.mapping_reads.load(Ordering::Relaxed)
+ }
+
/// Test seam: take `n` query slots, as `n` slow queries would. Not
/// reachable over HTTP.
#[doc(hidden)]
@@ -289,13 +407,15 @@ impl HubShared {
}
}
- /// `Ok` when `slug` is a corpus the catalogue lists and permits serving,
- /// else the response to send instead.
+ /// The catalogue's rule for `slug` when it lists the corpus and permits
+ /// serving it, else the response to send instead.
///
/// Everything that is not a servable corpus — a malformed name, a name the
/// catalogue does not list, a listed name with the `excluded` licence class
- /// — is the same `404`: the caller learns nothing about which it was.
- async fn resolve_corpus(self: &Arc<Self>, slug: &str) -> Result<(), Response> {
+ /// — is the same `404`: the caller learns nothing about which it was. The
+ /// rule is the catalogue's half of the licence decision; the corpus's own
+ /// ledger is the other half, and [`corpus_facts`] takes the stricter.
+ async fn resolve_corpus(self: &Arc<Self>, slug: &str) -> Result<CorpusRule, Response> {
if !valid_slug(slug) {
return Err(unknown_corpus());
}
@@ -315,7 +435,7 @@ impl HubShared {
}
};
match index.corpora.get(slug) {
- Some(rule) if rule.licence_class != LicenceClass::Excluded => Ok(()),
+ Some(rule) if rule.licence_class != LicenceClass::Excluded => Ok(rule.clone()),
_ => Err(unknown_corpus()),
}
}
@@ -329,32 +449,64 @@ impl HubShared {
/// union — the same gate the CLI applies — and the answer is reused for
/// [`SEARCH_FIELDS_TTL`], since reading the mappings of every pack is far
/// more expensive than a search. Blocking.
- fn search_fields_blocking(&self, node: &InProcessNode) -> Vec<&'static str> {
- {
- let cache = self.search_fields.lock().unwrap_or_else(|e| e.into_inner());
- if let Some((at, fields)) = cache.as_ref() {
- if at.elapsed() < SEARCH_FIELDS_TTL {
- return fields.clone();
+ ///
+ /// `Err` is a read that FAILED, and `search` answers it with the fixed 503:
+ /// an unreadable mapping is an outage, not "no results". The usual cause is
+ /// a node-wide mapping bigger than the cap on one in-process response
+ /// (16 MiB: a few hundred wide indices will do it), and reading it costs
+ /// seconds each time, so a failure is remembered for
+ /// [`Tuning::fields_failure_ttl`] — short, so a transient fault clears
+ /// itself — and the read is single-flight: the lock is held while the
+ /// mappings are read, so a crowd of requests arriving together waits for one
+ /// read instead of making sixteen.
+ ///
+ /// A caller running under a visibility rule of its own sees only its slice
+ /// of the node, so what it reads is not shared and what others read is not
+ /// reused for it: it neither reads nor writes the cache.
+ fn search_fields_blocking(&self, node: &InProcessNode) -> Result<Vec<&'static str>, String> {
+ if node.is_guarded() {
+ return self.read_search_fields(node).map(Option::unwrap_or_default);
+ }
+ let mut cache = self.search_fields.lock().unwrap_or_else(|e| e.into_inner());
+ if let Some((at, read)) = cache.as_ref() {
+ match read {
+ FieldsRead::Resolved(fields) if at.elapsed() < SEARCH_FIELDS_TTL => {
+ return Ok(fields.clone());
+ }
+ FieldsRead::Failed(why) if at.elapsed() < self.tuning.fields_failure_ttl => {
+ return Err(why.clone());
}
+ _ => {}
}
}
- let path = format!("/{SEARCH_INDEX_EXPR}/_mapping");
- let fields = match node.get_mapping(&path) {
- Ok(mapping) => mapped_search_fields(&mapping),
+ match self.read_search_fields(node) {
+ Ok(Some(fields)) => {
+ *cache = Some((Instant::now(), FieldsRead::Resolved(fields.clone())));
+ Ok(fields)
+ }
// No `xc-*` index yet: nothing to search, and not an error. Not
// remembered, so the first pack restored is searchable on the next
// request rather than a minute later.
- Err(e) if e.contains("__404__") => return Vec::new(),
- // A failed read is not remembered either: one transient fault must
- // not blank `search` for the whole TTL.
- Err(e) => {
- tracing::warn!(error = ?truncate_chars(&e, 500), "hub search: cannot read mappings");
- return Vec::new();
+ Ok(None) => Ok(Vec::new()),
+ Err(why) => {
+ *cache = Some((Instant::now(), FieldsRead::Failed(why.clone())));
+ Err(why)
}
- };
- *self.search_fields.lock().unwrap_or_else(|e| e.into_inner()) =
- Some((Instant::now(), fields.clone()));
- fields
+ }
+ }
+
+ /// One read of the node-wide mappings, as the caller sees them. `Ok(None)`:
+ /// there is no `xc-*` index at all.
+ fn read_search_fields(
+ &self,
+ node: &InProcessNode,
+ ) -> Result<Option<Vec<&'static str>>, String> {
+ self.mapping_reads.fetch_add(1, Ordering::Relaxed);
+ match node.get_mapping(&format!("/{SEARCH_INDEX_EXPR}/_mapping")) {
+ Ok(mapping) => Ok(Some(mapped_search_fields(&mapping))),
+ Err(e) if e.contains("__404__") => Ok(None),
+ Err(e) => Err(format!("cannot read the node-wide mappings: {e}")),
+ }
}
}
@@ -614,17 +766,30 @@ async fn rate_limit(State(shared): State<Arc<HubShared>>, req: Request, next: Ne
// The in-process XcHttp
// ─────────────────────────────────────────────────────────────────────────────
-/// Engine-level visibility for hosted queries: `xc-*` indices only.
+/// Engine-level visibility for hosted queries: `xc-*` indices only, and only
+/// those the caller could already see.
///
/// Installed around every internal call. The engine consults it at the
/// funnel every name passes through (`Engine::get_index`, `list_indices`), so
/// an alias, a wildcard or a bug upstream of this module still cannot reach an
/// index outside the hub namespace.
-struct HubVisibility;
+///
+/// It is the INTERSECTION of two rules and a replacement for neither. The hub
+/// namespace is one. The other is `outer`: the rule the caller's own request was
+/// running under, which `authz_middleware` installs for every principal that is
+/// not the superuser (an API key's grants). A key scoped to `tenant-a-*` is
+/// refused `xc-mini-c-b1-docs` by the ordinary API, and must be refused it here
+/// too — a hosted route is a different way to ask the engine, not a different
+/// set of rights. With no `outer` the caller is unrestricted (the superuser, an
+/// open-mode node, or a `hub.public` request, which skips authorization) and
+/// only the namespace applies.
+struct HubVisibility {
+ outer: Option<Arc<dyn IndexVisibility>>,
+}
impl IndexVisibility for HubVisibility {
fn visible(&self, index: &str) -> bool {
- index.starts_with("xc-")
+ index.starts_with("xc-") && self.outer.as_ref().is_none_or(|rule| rule.visible(index))
}
}
@@ -662,18 +827,114 @@ fn hub_index_expr_ok(expr: &str) -> bool {
pub struct InProcessNode {
router: Router,
rt: tokio::runtime::Handle,
- /// The caller's `Authorization` header, forwarded only so the audit entry a
- /// search writes names the right principal. The private router has no
+ /// The caller's `Authorization` header, forwarded so the internal request
+ /// carries the principal it came from. The private router has no
/// authentication layer; nothing here grants anything.
auth: Option<HeaderValue>,
+ /// Cap on any single response read back from the engine, in bytes.
+ cap: usize,
+ /// The visibility rule the caller's own request was running under, captured
+ /// on the request's task before the work moved to the blocking pool (a
+ /// task-local does not follow a `spawn_blocking`). `None`: unrestricted.
+ outer: Option<Arc<dyn IndexVisibility>>,
+ /// When set, only index expressions that begin with this build prefix are
+ /// sent: see [`InProcessNode::pin_to_build`].
+ scope: OnceLock<String>,
+ /// How many candidate records the last search that had to be cut back to fit
+ /// the response cap was limited to; 0 when none was.
+ cut_to: AtomicUsize,
+ /// The most indices any one search of this request skipped because they
+ /// failed (`_shards.failed`); 0 when none did.
+ skipped: AtomicU64,
+ /// Halvings of a search window this request may still spend
+ /// ([`MAX_WINDOW_SHRINKS`] in all).
+ shrinks_left: AtomicUsize,
+}
+
+/// Why an internal request produced no response.
+enum SendError {
+ /// The response was larger than the cap on one in-process response.
+ TooLarge,
+ /// Anything else; the text is for the log.
+ Failed(String),
+}
+
+impl SendError {
+ fn describe(self, cap: usize) -> String {
+ match self {
+ SendError::TooLarge => format!("internal response over {cap} bytes"),
+ SendError::Failed(why) => why,
+ }
+ }
}
impl InProcessNode {
- fn new(app: &AppState, auth: Option<HeaderValue>) -> Self {
+ fn new(
+ app: &AppState,
+ auth: Option<HeaderValue>,
+ cap: usize,
+ outer: Option<Arc<dyn IndexVisibility>>,
+ ) -> Self {
Self {
router: internal_router(app),
rt: tokio::runtime::Handle::current(),
auth,
+ cap,
+ outer,
+ scope: OnceLock::new(),
+ cut_to: AtomicUsize::new(0),
+ skipped: AtomicU64::new(0),
+ shrinks_left: AtomicUsize::new(MAX_WINDOW_SHRINKS),
+ }
+ }
+
+ /// Spend one of this request's window halvings; `false` when none are left.
+ fn take_shrink(&self) -> bool {
+ self.shrinks_left
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1))
+ .is_ok()
+ }
+
+ /// How many of the indices a search covered could not be searched and were
+ /// skipped (see [`es_compat::InProcessSearch::skip_failed_indices`]).
+ fn skipped_indices(&self) -> u64 {
+ self.skipped.load(Ordering::Relaxed)
+ }
+
+ /// Is the caller running under a visibility rule of its own?
+ fn is_guarded(&self) -> bool {
+ self.outer.is_some()
+ }
+
+ /// Confine this node to ONE corpus build: from now on an index expression is
+ /// sent only if every comma-separated part begins with `prefix`.
+ ///
+ /// `prefix` is the build prefix [`valid_build_prefix`] accepted. The `def`
+ /// and `code` pipelines re-read the corpus's ledger themselves to decide
+ /// what to query, so validating the ledger once, up front, would not stop a
+ /// ledger that changed in between (a `xerj hub pull` landing mid-request)
+ /// from aiming a query somewhere else. Pinning the node makes the validated
+ /// prefix the only thing that request can address, whatever the pipeline
+ /// then computes. Set once; a second call changes nothing.
+ fn pin_to_build(&self, prefix: &str) {
+ let _ = self.scope.set(prefix.to_string());
+ }
+
+ /// [`hub_index_expr_ok`], and inside the build this node is pinned to.
+ fn expr_ok(&self, expr: &str) -> bool {
+ hub_index_expr_ok(expr)
+ && self.scope.get().is_none_or(|pinned| {
+ expr.split(',')
+ .all(|part| part.starts_with(pinned.as_str()))
+ })
+ }
+
+ /// The number of candidate records the answer was cut back to because the
+ /// full window did not fit the response cap, if it was.
+ fn window_cut_to(&self) -> Option<usize> {
+ match self.cut_to.load(Ordering::Relaxed) {
+ 0 => None,
+ n => Some(n),
}
}
@@ -682,7 +943,7 @@ impl InProcessNode {
method: Method,
path: &str,
body: Option<&Value>,
- ) -> Result<(u16, Bytes), String> {
+ ) -> Result<(u16, Bytes), SendError> {
let mut builder = Request::builder().method(method).uri(path);
if let Some(a) = &self.auth {
builder = builder.header(header::AUTHORIZATION, a.clone());
@@ -691,30 +952,40 @@ impl InProcessNode {
Some(b) => builder
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(
- serde_json::to_vec(b).map_err(|e| e.to_string())?,
+ serde_json::to_vec(b).map_err(|e| SendError::Failed(e.to_string()))?,
)),
None => builder.body(Body::empty()),
}
- .map_err(|e| format!("cannot build the internal request: {e}"))?;
+ .map_err(|e| SendError::Failed(format!("cannot build the internal request: {e}")))?;
let router = self.router.clone();
+ let cap = self.cap;
let call = async move {
let resp = match router.oneshot(req).await {
Ok(resp) => resp,
Err(never) => match never {},
};
let status = resp.status().as_u16();
- let bytes = axum::body::to_bytes(resp.into_body(), MAX_INTERNAL_RESPONSE_BYTES)
+ let bytes = axum::body::to_bytes(resp.into_body(), cap)
.await
- .map_err(|_| {
- format!("internal response over {MAX_INTERNAL_RESPONSE_BYTES} bytes")
- })?;
- Ok::<_, String>((status, bytes))
+ .map_err(|_| SendError::TooLarge)?;
+ Ok::<_, SendError>((status, bytes))
};
+ // This search is the node talking to itself on behalf of a public route:
+ // it is not audited, and one bad index must not fail it for the rest.
+ let call = es_compat::scoped_in_process_search(
+ es_compat::InProcessSearch {
+ skip_audit: true,
+ skip_failed_indices: true,
+ },
+ call,
+ );
+ let guard: Arc<dyn IndexVisibility> = Arc::new(HubVisibility {
+ outer: self.outer.clone(),
+ });
// `block_on` is legal here: this runs on a blocking-pool thread, never
// on an async worker. The visibility guard is a task-local, so it must
// be installed inside the future this thread drives.
- self.rt
- .block_on(index_guard::scoped(Arc::new(HubVisibility), call))
+ self.rt.block_on(index_guard::scoped(guard, call))
}
fn json(
@@ -722,7 +993,7 @@ impl InProcessNode {
method: Method,
path: &str,
body: Option<&Value>,
- ) -> Result<(u16, Value), String> {
+ ) -> Result<(u16, Value), SendError> {
let (status, bytes) = self.send(method, path, body)?;
let value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
Ok((status, value))
@@ -734,9 +1005,12 @@ impl XcHttp for InProcessNode {
let expr = path
.strip_prefix('/')
.and_then(|p| p.strip_suffix("/_mapping"))
- .filter(|e| hub_index_expr_ok(e))
+ .filter(|e| self.expr_ok(e))
.ok_or_else(|| "refused: not a hub mapping path".to_string())?;
- match self.json(Method::GET, &format!("/{expr}/_mapping"), None)? {
+ match self
+ .json(Method::GET, &format!("/{expr}/_mapping"), None)
+ .map_err(|e| e.describe(self.cap))?
+ {
(s, v) if (200..300).contains(&s) => Ok(v),
(404, _) => Err("__404__".to_string()),
(s, v) => Err(format!("HTTP {s}: {v}")),
@@ -744,11 +1018,14 @@ impl XcHttp for InProcessNode {
}
fn cat_indices_json(&self, pattern: &str) -> Result<Vec<String>, String> {
- if !hub_index_expr_ok(pattern) {
+ if !self.expr_ok(pattern) {
return Err("refused: not a hub index pattern".to_string());
}
let path = format!("/_cat/indices/{pattern}?format=json&h=index");
- match self.json(Method::GET, &path, None)? {
+ match self
+ .json(Method::GET, &path, None)
+ .map_err(|e| e.describe(self.cap))?
+ {
// A wildcard matching no index is "none", not an error.
(404, _) => Ok(Vec::new()),
(s, v) if (200..300).contains(&s) => v
@@ -764,14 +1041,71 @@ impl XcHttp for InProcessNode {
}
}
+ /// `POST /{index}/_search`, retried with a smaller `size` when the response
+ /// does not fit the cap on one in-process response.
+ ///
+ /// The `code` pipeline asks for whole records (the passage is cut from the
+ /// record's `body`) and for ten times the `k` it was asked for, up to 200.
+ /// Two 9 MiB records therefore overflow a 16 MiB cap even at `k=1`, and a
+ /// 503 for a question the corpus can answer is the wrong outcome. So an
+ /// overflowing search is repeated with half the window until it fits, within
+ /// a budget of [`MAX_WINDOW_SHRINKS`] halvings for the whole request (each
+ /// try is a full search that is thrown away);
+ /// [`InProcessNode::window_cut_to`] then reports that the answer was built
+ /// from fewer candidates. A window of one that still does not fit, or an
+ /// exhausted budget, is an error: there is no smaller answer to give.
fn search(&self, index: &str, body: &Value) -> Result<Value, String> {
- if !hub_index_expr_ok(index) {
+ if !self.expr_ok(index) {
return Err("refused: not a hub index expression".to_string());
}
- match self.json(Method::POST, &format!("/{index}/_search"), Some(body))? {
- (s, v) if (200..300).contains(&s) => Ok(v),
- (404, _) => Err("__404__".to_string()),
- (s, v) => Err(format!("HTTP {s}: {v}")),
+ let path = format!("/{index}/_search");
+ let mut body = std::borrow::Cow::Borrowed(body);
+ loop {
+ match self.json(Method::POST, &path, Some(&body)) {
+ Err(SendError::TooLarge) => {
+ let size = body
+ .get("size")
+ .and_then(Value::as_u64)
+ .unwrap_or(DEFAULT_SEARCH_SIZE);
+ if size <= 1 || !self.take_shrink() {
+ return Err(SendError::TooLarge.describe(self.cap));
+ }
+ let mut smaller = body.into_owned();
+ smaller["size"] = json!((size / 2).max(1));
+ body = std::borrow::Cow::Owned(smaller);
+ }
+ Err(e) => return Err(e.describe(self.cap)),
+ Ok((s, v)) if (200..300).contains(&s) => {
+ let failed = v
+ .pointer("/_shards/failed")
+ .and_then(Value::as_u64)
+ .unwrap_or(0);
+ if failed > 0 {
+ self.skipped.fetch_max(failed, Ordering::Relaxed);
+ }
+ if matches!(body, std::borrow::Cow::Owned(_)) {
+ let size = body
+ .get("size")
+ .and_then(Value::as_u64)
+ .unwrap_or(DEFAULT_SEARCH_SIZE);
+ let returned = v
+ .pointer("/hits/hits")
+ .and_then(Value::as_array)
+ .map_or(0, Vec::len) as u64;
+ // A window that came back FULL may have left matches
+ // behind; one that did not returned everything there was.
+ if returned >= size {
+ self.cut_to.store(
+ usize::try_from(size).unwrap_or(usize::MAX),
+ Ordering::Relaxed,
+ );
+ }
+ }
+ return Ok(v);
+ }
+ Ok((404, _)) => return Err("__404__".to_string()),
+ Ok((s, v)) => return Err(format!("HTTP {s}: {v}")),
+ }
}
}
}
@@ -801,7 +1135,16 @@ where
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
return Err(resp);
};
- let node = InProcessNode::new(&shared.app, headers.get(header::AUTHORIZATION).cloned());
+ // Captured HERE, on the request's own task, before the work moves to the
+ // blocking pool: the caller's visibility rule is a task-local and does not
+ // follow a `spawn_blocking`. Read inside the closure it would see nothing and
+ // silently widen the caller to the whole `xc-*` namespace.
+ let node = InProcessNode::new(
+ &shared.app,
+ headers.get(header::AUTHORIZATION).cloned(),
+ shared.tuning.response_cap_bytes,
+ index_guard::current(),
+ );
let job = tokio::task::spawn_blocking(move || {
let _permit = permit;
work(&node)
@@ -839,6 +1182,27 @@ pub fn valid_slug(slug: &str) -> bool {
&& bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'+' | b'-'))
}
+/// Is `prefix` exactly `xc-<slug>-b<digits>` — the build prefix of the corpus
+/// named `slug` and of nothing else?
+///
+/// The state ledger is written by `xerj hub pull` from the pack's own manifest,
+/// so its `index_prefix` is pack-supplied text. Every `code` and `def` query is
+/// aimed at `<index_prefix>*` or `<index_prefix>-defs`, so an unchecked prefix is
+/// a way to aim a hosted query at somebody else's indices: `xc-` is every hub
+/// index on the node, `xc-mini-closed-b1` is another corpus's real build (the
+/// excluded one, say), and a comma list is two expressions in one. The character
+/// set [`hub_index_expr_ok`] allows does not stop any of those, so the shape does:
+/// the corpus's OWN slug, then `-b`, then digits to the end — no wildcard, no
+/// comma, no suffix, and no other corpus's name.
+pub fn valid_build_prefix(prefix: &str, slug: &str) -> bool {
+ valid_slug(slug)
+ && prefix
+ .strip_prefix("xc-")
+ .and_then(|rest| rest.strip_prefix(slug))
+ .and_then(|rest| rest.strip_prefix("-b"))
+ .is_some_and(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()))
+}
+
/// Collect the query string into a map, refusing a parameter this route does
/// not take and a parameter given twice.
fn collect_params(
@@ -1090,18 +1454,27 @@ async fn def_route(
Ok(k) => k,
Err(why) => return bad_request(why),
};
- if let Err(resp) = shared.resolve_corpus(&slug).await {
- return resp;
- }
+ let rule = match shared.resolve_corpus(&slug).await {
+ Ok(rule) => rule,
+ Err(resp) => return resp,
+ };
+ let after_check = shared.tuning.after_ledger_check;
let outcome = run_query(&shared, &headers, move |node| {
let root = hub_root();
- let facts = match corpus_facts(&root, &slug) {
+ let facts = match corpus_facts(&root, &slug, &rule) {
Ok(f) => f,
- Err(_) => return Outcome::NotLoaded,
+ Err(LedgerError::NotLoaded) => return Outcome::NotLoaded,
+ Err(LedgerError::Invalid(why)) => return Outcome::Failed(why),
};
if facts.licence_class == LicenceClass::Excluded {
return Outcome::NotServed;
}
+ // Nothing this request sends may name an index outside the build the
+ // ledger was just checked to describe, whatever `lookup_in` then reads.
+ node.pin_to_build(&facts.index_prefix);
+ if let Some(hook) = after_check {
+ hook(&root, &slug);
+ }
// Mirror the CLI's refusals: a corpus the ledger knows but this node
// holds no index for is "not loaded", not "no such definition".
match node.cat_indices_json(&format!("{}*", facts.index_prefix)) {
@@ -1121,6 +1494,21 @@ async fn def_route(
return Outcome::Failed(w.clone());
}
answer.answered_from = AnsweredFrom::Hosted;
+ // `lookup_in` describes the corpus from its ledger alone. This node
+ // serves it under the stricter of the ledger and the catalogue, and the
+ // header and every hit must say so (a `code` answer for the same corpus
+ // does).
+ answer.licence_class = facts.licence_class;
+ answer.licence_spdx = facts.licence_spdx.clone();
+ if let Some(note) = approach_only_note(&facts) {
+ for hit in &mut answer.hits {
+ hit.licence_warning = Some(note.clone());
+ }
+ }
+ if let Some(window) = node.window_cut_to() {
+ answer.truncated = true;
+ answer.warnings.push(window_cut_warning(window));
+ }
Outcome::Done(Box::new(answer))
})
.await;
@@ -1154,19 +1542,29 @@ async fn code_route(
Ok(m) => m,
Err(why) => return bad_request(why),
};
- if let Err(resp) = shared.resolve_corpus(&slug).await {
- return resp;
- }
+ let rule = match shared.resolve_corpus(&slug).await {
+ Ok(rule) => rule,
+ Err(resp) => return resp,
+ };
let width = passage_width(max_tokens);
+ let after_check = shared.tuning.after_ledger_check;
let outcome = run_query(&shared, &headers, move |node| {
let root = hub_root();
- let facts = match corpus_facts(&root, &slug) {
+ let facts = match corpus_facts(&root, &slug, &rule) {
Ok(f) => f,
- Err(_) => return Outcome::NotLoaded,
+ Err(LedgerError::NotLoaded) => return Outcome::NotLoaded,
+ Err(LedgerError::Invalid(why)) => return Outcome::Failed(why),
};
if facts.licence_class == LicenceClass::Excluded {
return Outcome::NotServed;
}
+ // `run_code_query` reads the ledger again to decide what to query. Pin
+ // the node to the build just validated, so nothing it computes can leave
+ // that corpus.
+ node.pin_to_build(&facts.index_prefix);
+ if let Some(hook) = after_check {
+ hook(&root, &slug);
+ }
let mut params = CodeParams::new(&slug, &query);
params.k = k;
params.mode = mode;
@@ -1191,17 +1589,23 @@ async fn code_route(
hit.licence_warning = Some(note.clone());
}
}
- if mode != Mode::Bm25 {
- warnings.push(
- "mode semantic/hybrid falls back to BM25 when the corpus has no \
- semantic_text mapping, and this response does not say which arms ran"
- .to_string(),
- );
+ if let Some(note) = mode_warning(mode) {
+ warnings.push(note.to_string());
}
- let truncated = match max_tokens {
+ let mut truncated = match max_tokens {
Some(budget) => apply_token_budget(&mut hits, budget, &mut warnings),
None => false,
};
+ if let Some(window) = node.window_cut_to() {
+ truncated = true;
+ warnings.push(window_cut_warning(window));
+ }
+ // A miss must not read as "this code does not exist" when part
+ // of the corpus could not be searched.
+ let skipped = node.skipped_indices();
+ if skipped > 0 {
+ warnings.push(skipped_warning(skipped));
+ }
Outcome::Done(Box::new(facts.into_answer(
AnswerMode::How,
hits,
@@ -1249,25 +1653,35 @@ async fn search_route(
}
let blocking_shared = shared.clone();
let outcome = run_query(&shared, &headers, move |node| {
- let fields = blocking_shared.search_fields_blocking(node);
+ // A failed read of the mappings is an outage, not an empty node.
+ let fields = blocking_shared.search_fields_blocking(node)?;
if fields.is_empty() {
return Ok(Vec::new());
}
// Only the top hits are read, never a total: with the fan-out over every
// pack index, counting matches is work nobody asked for.
+ //
+ // `lenient`: the fan-out covers every pack on the node, and one pack that
+ // maps `body` as a number must not fail a text query for all the others
+ // (this engine answers a text query against a numeric field with a 400
+ // unless the clause is lenient). An index that still fails is skipped
+ // by the engine under [`es_compat::InProcessSearch`], not fatal.
let body = json!({
"size": SEARCH_POOL,
"track_total_hits": false,
"_source": SEARCH_SOURCE_FIELDS,
- "query": { "multi_match": { "query": query, "fields": fields } },
+ "query": { "multi_match": { "query": query, "fields": fields, "lenient": true } },
});
let resp = match node.search(SEARCH_INDEX_EXPR, &body) {
Ok(r) => r,
Err(e) if e.contains("__404__") => return Ok(Vec::new()),
Err(e) => return Err(format!("search failed: {e}")),
};
+ let root = hub_root();
let corpora = match &data.index {
- Ok(index) => group_by_corpus(&resp, &index.corpora, k),
+ Ok(index) => group_by_corpus(&resp, &index.corpora, k, |slug, rule| {
+ servable_licence(&root, slug, rule)
+ }),
Err(_) => Vec::new(),
};
Ok(corpora)
@@ -1300,6 +1714,11 @@ fn hub_root() -> PathBuf {
/// pull wrote. Mirrors `xerj_hub::def`'s private `DefMeta` so a `def` answer
/// and a `code` answer for the same corpus carry the same header; a test pins
/// the two together.
+///
+/// `index_prefix` is the build prefix [`valid_build_prefix`] accepted, never
+/// what the ledger merely said. `licence_class` and `licence_spdx` are the
+/// effective licence ([`effective_licence`]): the stricter of the ledger's and
+/// the catalogue's, not the ledger's alone.
struct CorpusFacts {
corpus: String,
index_prefix: String,
@@ -1310,20 +1729,143 @@ struct CorpusFacts {
licence_class: LicenceClass,
}
-fn corpus_facts(root: &FsPath, slug: &str) -> Result<CorpusFacts, String> {
- let st = ledger::load_state(root, slug)?;
+/// Why a corpus's ledger cannot be used.
+enum LedgerError {
+ /// There is no readable ledger: the corpus is not loaded on this node.
+ NotLoaded,
+ /// A ledger that does not describe a servable pack. The detail is for the
+ /// log; the client gets the fixed 503.
+ Invalid(String),
+}
+
+/// The corpus's ledger, checked, and its licence reconciled with the
+/// catalogue's `rule`.
+///
+/// The ledger is written by `xerj hub pull` from the pack's own manifest, so
+/// nothing in it is trusted to say where to query: its build prefix must be
+/// exactly `xc-<slug>-b<digits>` ([`valid_build_prefix`]) or the corpus is
+/// refused before any query is built from it.
+fn corpus_facts(root: &FsPath, slug: &str, rule: &CorpusRule) -> Result<CorpusFacts, LedgerError> {
+ let st = ledger::load_state(root, slug).map_err(|_| LedgerError::NotLoaded)?;
+ let index_prefix = ledger::query_prefix(&st);
+ if !valid_build_prefix(&index_prefix, slug) {
+ return Err(LedgerError::Invalid(format!(
+ "the ledger for {slug} names the build prefix [{}], not xc-{slug}-b<digits>",
+ truncate_chars(&index_prefix, 80)
+ )));
+ }
let pin_date = st.pin_date.clone().unwrap_or_default();
+ let (licence_class, licence_spdx) = effective_licence(
+ (rule.licence_class, &rule.licence_spdx),
+ (
+ parse_licence_class(st.licence_class.as_deref()),
+ st.licence_spdx.as_deref().unwrap_or_default(),
+ ),
+ );
Ok(CorpusFacts {
corpus: st.name(slug),
- index_prefix: ledger::query_prefix(&st),
+ index_prefix,
age_days: age_days_from(&pin_date, st.indexed_at.as_deref()),
pin_sha: st.pin_sha.clone().unwrap_or_default(),
pin_date,
- licence_spdx: st.licence_spdx.clone().unwrap_or_default(),
- licence_class: parse_licence_class(st.licence_class.as_deref()),
+ licence_spdx,
+ licence_class,
})
}
+/// How restrictive a class is: permissive < approach-only < excluded.
+fn licence_rank(class: LicenceClass) -> u8 {
+ match class {
+ LicenceClass::AdaptWithAttribution => 0,
+ LicenceClass::ApproachOnly => 1,
+ LicenceClass::Excluded => 2,
+ }
+}
+
+/// The licence a corpus is served under: the STRICTER of the catalogue's and the
+/// ledger's class, each given as `(class, spdx)`.
+///
+/// Both must permit serving and either can refuse — the catalogue is what the
+/// hub operator lists, the ledger is what this node's pack says about itself —
+/// so neither may loosen the other: a catalogue entry that is servable does not
+/// override a ledger that says `excluded`, and a ledger that says nothing (an
+/// absent class reads as permissive) does not override a catalogue that says
+/// `approach-only`. Every route takes its class from here, so `def`, `code` and
+/// `search` cannot disagree about the same corpus.
+///
+/// The SPDX id travels with the class that decided: the stricter side's, falling
+/// back to the other side's when it names none; on a tie the ledger's, because
+/// that is what the pack on this node says it is.
+pub fn effective_licence(
+ catalogue: (LicenceClass, &str),
+ ledger: (LicenceClass, &str),
+) -> (LicenceClass, String) {
+ let (decider, other) = if licence_rank(catalogue.0) > licence_rank(ledger.0) {
+ (catalogue, ledger)
+ } else {
+ (ledger, catalogue)
+ };
+ let spdx = if decider.1.is_empty() {
+ other.1
+ } else {
+ decider.1
+ };
+ (decider.0, spdx.to_string())
+}
+
+/// The effective licence for `search`, or `None` when the corpus must not be
+/// served from it: the stricter class is `excluded`, or the corpus's ledger
+/// cannot be used (not loaded here, or naming a build that is not its own) —
+/// the same corpora `def` and `code` answer `404`/`503` for.
+fn servable_licence(
+ root: &FsPath,
+ slug: &str,
+ rule: &CorpusRule,
+) -> Option<(LicenceClass, String)> {
+ let facts = corpus_facts(root, slug, rule).ok()?;
+ (facts.licence_class != LicenceClass::Excluded)
+ .then_some((facts.licence_class, facts.licence_spdx))
+}
+
+/// What `mode` does on a hosted corpus, for the one warning that says so. The
+/// shared query pipeline's JSON output does not report which arms ran, so the
+/// answer cannot say; it says what each mode DOES instead, and the two modes
+/// behave differently when the corpus has no `semantic_text` mapping for `body`.
+fn mode_warning(mode: Mode) -> Option<&'static str> {
+ match mode {
+ Mode::Bm25 => None,
+ Mode::Semantic => Some(
+ "mode=semantic searches only the indices that map `body` as semantic_text; a corpus \
+ with none returns no hits (it does not fall back to BM25), and this response does \
+ not say which indices were searched",
+ ),
+ Mode::Hybrid => Some(
+ "mode=hybrid combines BM25 with a vector search where the corpus maps `body` as \
+ semantic_text and answers from BM25 alone where it does not; this response does not \
+ say which arms ran",
+ ),
+ }
+}
+
+/// The warning for an answer from a corpus some of whose indices could not be
+/// searched (the engine skipped them rather than fail the whole request).
+fn skipped_warning(count: u64) -> String {
+ format!(
+ "{count} of this corpus's indices could not be searched and were skipped; the answer \
+ is from the rest, so a missing result may be in one of them"
+ )
+}
+
+/// The warning for an answer built from fewer candidate records than usual
+/// because the full window did not fit the response cap.
+fn window_cut_warning(window: usize) -> String {
+ format!(
+ "the matching records are too large to return together; the answer was built from the \
+ top {window} candidate record(s) instead of the usual window, so lower-ranked matches \
+ may be missing"
+ )
+}
+
impl CorpusFacts {
fn into_answer(
self,
@@ -1620,10 +2162,26 @@ struct BestHit {
}
/// Group a `multi_match` reply by corpus: the best hit of each, best first
-/// (ties by name), at most `want`. Hits from an index that belongs to no listed
-/// corpus, and from an `excluded` one, are skipped.
-fn group_by_corpus(resp: &Value, known: &HashMap<String, CorpusRule>, want: usize) -> Vec<BestHit> {
+/// (ties by name), at most `want`.
+///
+/// A hit's index is attributed to a corpus by `known`, the WHOLE catalogue
+/// (never a filtered subset: with `a` excluded and `a-b1` listed, the index
+/// `xc-a-b1-b2-docs` is `a`'s, and attributing it to `a-b1` would serve an
+/// excluded corpus's records under another name). What may then be served is
+/// decided by `licence`, asked once per corpus: it returns the licence class and
+/// SPDX id to serve the corpus under, or `None` to skip it. Hits from an index
+/// that belongs to no listed corpus are skipped too.
+fn group_by_corpus<F>(
+ resp: &Value,
+ known: &HashMap<String, CorpusRule>,
+ want: usize,
+ mut licence: F,
+) -> Vec<BestHit>
+where
+ F: FnMut(&str, &CorpusRule) -> Option<(LicenceClass, String)>,
+{
let mut best: HashMap<&str, Hit> = HashMap::new();
+ let mut licences: HashMap<&str, Option<(LicenceClass, String)>> = HashMap::new();
let rows = resp
.pointer("/hits/hits")
.and_then(Value::as_array)
@@ -1639,7 +2197,14 @@ fn group_by_corpus(resp: &Value, known: &HashMap<String, CorpusRule>, want: usiz
let Some(rule) = known.get(slug) else {
continue;
};
- if rule.licence_class == LicenceClass::Excluded {
+ let Some((class, spdx)) = licences
+ .entry(slug)
+ .or_insert_with(|| licence(slug, rule))
+ .as_ref()
+ else {
+ continue;
+ };
+ if *class == LicenceClass::Excluded {
continue;
}
let score = row.get("_score").and_then(Value::as_f64).unwrap_or(0.0) as f32;
@@ -1655,8 +2220,8 @@ fn group_by_corpus(resp: &Value, known: &HashMap<String, CorpusRule>, want: usiz
} else {
doc_hit(src, score)
};
- if rule.licence_class == LicenceClass::ApproachOnly {
- hit.licence_warning = Some(licence_note(&rule.licence_spdx));
+ if *class == LicenceClass::ApproachOnly {
+ hit.licence_warning = Some(licence_note(spdx));
}
best.insert(slug, hit);
}
@@ -1718,3 +2283,141 @@ fn doc_hit(src: &Value, score: f32) -> Hit {
licence_warning: None,
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ /// The pin is what makes the validated build the ONLY thing a `def`/`code`
+ /// request can address, whatever the shared pipeline computes from its own
+ /// second read of a ledger that may have changed since it was checked.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn a_node_pinned_to_a_build_refuses_every_expression_outside_it() {
+ let state = test_state();
+ let pinned = InProcessNode::new(&state, None, 1 << 20, None);
+ pinned.pin_to_build("xc-a-b1");
+ let unpinned = InProcessNode::new(&state, None, 1 << 20, None);
+
+ tokio::task::spawn_blocking(move || {
+ let refused =
+ |r: &Result<Value, String>| matches!(r, Err(e) if e.starts_with("refused"));
+ let outside = [
+ "xc-*",
+ "xc-",
+ "xc-b-b1*",
+ "xc-b-b1-defs",
+ "xc-a-b1*,xc-b-b1*",
+ "xc-b-b1-docs,xc-a-b1-docs",
+ "xc-a",
+ "xc-a-b",
+ ];
+ for expr in outside {
+ assert!(refused(&pinned.search(expr, &json!({}))), "search {expr}");
+ assert!(
+ pinned
+ .cat_indices_json(expr)
+ .is_err_and(|e| e.starts_with("refused")),
+ "cat {expr}"
+ );
+ assert!(
+ refused(&pinned.get_mapping(&format!("/{expr}/_mapping"))),
+ "mapping {expr}"
+ );
+ // Without the pin the same expressions are well-formed hub names:
+ // the pin, not the character set, is what refused them.
+ if expr != "xc-" {
+ assert!(
+ !refused(&unpinned.search(expr, &json!({}))),
+ "unpinned {expr} is a legal hub expression"
+ );
+ }
+ }
+ // Inside the build: well-formed, so the request reaches the engine,
+ // which has no such index here (a 404, not a refusal).
+ for expr in ["xc-a-b1*", "xc-a-b1-defs", "xc-a-b1-docs,xc-a-b1-defs"] {
+ let r = pinned.search(expr, &json!({}));
+ assert!(!refused(&r), "{expr}: {r:?}");
+ assert!(pinned.cat_indices_json(expr).is_ok(), "cat {expr}");
+ }
+ })
+ .await
+ .expect("blocking task");
+ }
+
+ /// Every overflow retry is a whole search that is thrown away, so a request
+ /// gets a fixed number of them however many searches its pipeline makes.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn a_request_has_a_fixed_budget_of_window_halvings() {
+ let state = test_state();
+ let node = InProcessNode::new(&state, None, 1 << 20, None);
+ for n in 0..MAX_WINDOW_SHRINKS {
+ assert!(node.take_shrink(), "halving {n} is within the budget");
+ }
+ assert!(!node.take_shrink(), "the budget is spent");
+ assert!(!node.take_shrink(), "and stays spent");
+ // Seven halvings are exactly what takes the widest window to one.
+ assert_eq!(
+ (0..MAX_WINDOW_SHRINKS).fold(200u64, |s, _| (s / 2).max(1)),
+ 1
+ );
+ // A fresh request starts with a full budget.
+ let next = InProcessNode::new(&state, None, 1 << 20, None);
+ assert!(next.take_shrink());
+ }
+
+ #[test]
+ fn effective_licence_is_the_stricter_class_with_the_deciding_spdx() {
+ use LicenceClass::{AdaptWithAttribution as Adapt, ApproachOnly as Approach, Excluded};
+ let el = effective_licence;
+ // Agreement.
+ assert_eq!(el((Adapt, "MIT"), (Adapt, "MIT")), (Adapt, "MIT".into()));
+ assert_eq!(
+ el((Approach, "GPL-3.0-only"), (Approach, "GPL-3.0-only")),
+ (Approach, "GPL-3.0-only".into())
+ );
+ // Either side can be the stricter one, in every pair.
+ assert_eq!(
+ el((Adapt, "MIT"), (Approach, "AGPL")),
+ (Approach, "AGPL".into())
+ );
+ assert_eq!(
+ el((Approach, "GPL"), (Adapt, "MIT")),
+ (Approach, "GPL".into())
+ );
+ assert_eq!(
+ el((Adapt, "MIT"), (Excluded, "Closed")),
+ (Excluded, "Closed".into())
+ );
+ assert_eq!(
+ el((Excluded, "Closed"), (Adapt, "MIT")),
+ (Excluded, "Closed".into())
+ );
+ assert_eq!(
+ el((Approach, "GPL"), (Excluded, "Closed")),
+ (Excluded, "Closed".into())
+ );
+ assert_eq!(
+ el((Excluded, "Closed"), (Approach, "GPL")),
+ (Excluded, "Closed".into())
+ );
+ // The stricter side names no licence: the other side's id is used.
+ assert_eq!(el((Approach, "GPL"), (Adapt, "")), (Approach, "GPL".into()));
+ assert_eq!(el((Adapt, "MIT"), (Approach, "")), (Approach, "MIT".into()));
+ // A tie takes the ledger's id, else the catalogue's.
+ assert_eq!(
+ el((Adapt, "MIT"), (Adapt, "Apache-2.0")),
+ (Adapt, "Apache-2.0".into())
+ );
+ assert_eq!(el((Adapt, "MIT"), (Adapt, "")), (Adapt, "MIT".into()));
+ }
+}
diff --git a/engine/crates/xerj-api/tests/hub_api.rs b/engine/crates/xerj-api/tests/hub_api.rs
index 6108e07e8..0e18cedc0 100644
--- a/engine/crates/xerj-api/tests/hub_api.rs
+++ b/engine/crates/xerj-api/tests/hub_api.rs
@@ -23,6 +23,7 @@
use std::net::SocketAddr;
use std::path::PathBuf;
+use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
@@ -44,6 +45,33 @@ const ADMIN_KEY: &str = "hub-test-admin-key";
const PIN_SHA: &str = "0774d05537f9762f838f7ab541b7765f1a729cb5";
const PIN_DATE: &str = "2026-10-02";
+/// `(slug, ledger index_prefix)`: pack-supplied build prefixes that are NOT
+/// `xc-<that slug>-b<digits>`. Every one uses only characters a URL path can
+/// carry, so the character-set guard alone does not stop them; each one is a way
+/// to aim `<prefix>*` at somebody else's indices.
+const HOSTILE_PREFIXES: [(&str, &str); 8] = [
+ // `xc-*`: every hub index on the node.
+ ("evil-bare", "xc-"),
+ ("evil-wild", "xc-*"),
+ // Another corpus's real build (the excluded one), and one no entry lists.
+ ("evil-other", "xc-mini-closed-b1"),
+ ("evil-orphan", "xc-orphan-b1"),
+ // A comma list is two index expressions in one.
+ ("evil-list", "xc-evil-list-b1,xc-mini-closed-b1"),
+ // Not a build prefix: a whole index name, and a name with no build at all.
+ ("evil-tail", "xc-evil-tail-b1-defs"),
+ ("evil-nobuild", "xc-evil-nobuild"),
+ ("evil-empty", ""),
+];
+
+/// Every slug whose ledger must be refused, `HOSTILE_PREFIXES` plus the one with
+/// no `index_prefix` field.
+fn hostile_slugs() -> Vec<&'static str> {
+ let mut v: Vec<&'static str> = HOSTILE_PREFIXES.iter().map(|(s, _)| *s).collect();
+ v.push("evil-legacy");
+ v
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// The shared fixture: ledgers + catalogue, written once
// ─────────────────────────────────────────────────────────────────────────────
@@ -56,6 +84,9 @@ struct Fixture {
static FIXTURE: OnceLock<Fixture> = OnceLock::new();
+/// Source of fresh client addresses for [`Hub::stranger`].
+static NEXT_PEER: AtomicU32 = AtomicU32::new(1);
+
fn entry(slug: &str, spdx: &str, class: LicenceClass) -> CatalogueEntry {
CatalogueEntry {
slug: slug.into(),
@@ -132,7 +163,67 @@ fn fixture() -> &'static Fixture {
"MIT",
"adapt-with-attribution",
);
- let bytes = catalogue_bytes(vec![
+ // Ledgers whose pack-supplied `index_prefix` is not `xc-<own slug>-b<digits>`.
+ // Each is built from characters the URL guard accepts, and each points at
+ // indices that belong to somebody else.
+ for (slug, prefix) in HOSTILE_PREFIXES {
+ led(slug, prefix, "MIT", "adapt-with-attribution");
+ }
+ // No `index_prefix` at all: `query_prefix` falls back to the ledger's own
+ // `prefix`, which here is the bare namespace.
+ {
+ let state_dir = home.join("state");
+ std::fs::create_dir_all(&state_dir).expect("state dir");
+ std::fs::write(
+ state_dir.join("evil-legacy.json"),
+ format!(
+ "{{\"corpus\":\"evil-legacy\",\"indexed_at\":\"2026-10-01T00:00:00Z\",\
+ \"prefix\":\"xc-\",\"url\":\"http://x\",\"autoindex_exit\":0,\
+ \"salvaged\":false,\"source\":\"hubpack\",\"pin_sha\":\"{PIN_SHA}\",\
+ \"pin_date\":\"{PIN_DATE}\",\"licence_spdx\":\"MIT\",\
+ \"licence_class\":\"adapt-with-attribution\"}}\n"
+ ),
+ )
+ .expect("legacy ledger");
+ }
+ // The two sides of "either can refuse" the other way round, plus both
+ // directions of "the stricter class wins".
+ led(
+ "cat-closed",
+ "xc-cat-closed-b1",
+ "MIT",
+ "adapt-with-attribution",
+ );
+ led("gplish", "xc-gplish-b1", "MIT", "adapt-with-attribution");
+ led(
+ "ledger-gpl",
+ "xc-ledger-gpl-b1",
+ "AGPL-3.0-only",
+ "approach-only",
+ );
+ // Corpora used by the failure-isolation tests: indices exist only on the
+ // nodes of the tests that create them.
+ led(
+ "longbody",
+ "xc-longbody-b1",
+ "MIT",
+ "adapt-with-attribution",
+ );
+ led("noindex", "xc-noindex-b1", "MIT", "adapt-with-attribution");
+ led("big", "xc-big-b1", "MIT", "adapt-with-attribution");
+ led(
+ "bodyonly",
+ "xc-bodyonly-b1",
+ "MIT",
+ "adapt-with-attribution",
+ );
+ // Well-formed until `rewrite_flip_ledger` replaces it mid-request.
+ led("flip", "xc-flip-b1", "MIT", "adapt-with-attribution");
+ // One corpus with an index that cannot be searched next to a good one,
+ // and one whose every index cannot.
+ led("mixed", "xc-mixed-b1", "MIT", "adapt-with-attribution");
+ led("allbad", "xc-allbad-b1", "MIT", "adapt-with-attribution");
+ let mut entries = vec![
entry("mini-c", "MIT", LicenceClass::AdaptWithAttribution),
entry("mini-gpl", "GPL-3.0-only", LicenceClass::ApproachOnly),
entry("mini-closed", "LicenseRef-Closed", LicenceClass::Excluded),
@@ -141,7 +232,25 @@ fn fixture() -> &'static Fixture {
entry("badprefix", "MIT", LicenceClass::AdaptWithAttribution),
// Listed, but no ledger and no indices on any test node.
entry("ghost", "MIT", LicenceClass::AdaptWithAttribution),
- ]);
+ // The catalogue refuses it; this node's ledger would serve it.
+ entry("cat-closed", "LicenseRef-Closed", LicenceClass::Excluded),
+ // The catalogue says approach-only; the ledger says permissive.
+ entry("gplish", "GPL-3.0-only", LicenceClass::ApproachOnly),
+ // The catalogue says permissive; the ledger says approach-only.
+ entry("ledger-gpl", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("longbody", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("noindex", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("big", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("bodyonly", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("flip", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("mixed", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("allbad", "MIT", LicenceClass::AdaptWithAttribution),
+ entry("evil-legacy", "MIT", LicenceClass::AdaptWithAttribution),
+ ];
+ for (slug, _) in HOSTILE_PREFIXES {
+ entries.push(entry(slug, "MIT", LicenceClass::AdaptWithAttribution));
+ }
+ let bytes = catalogue_bytes(entries);
let catalogue = dir.join("catalogue.json");
std::fs::write(&catalogue, &bytes).expect("catalogue file");
std::env::set_var("XERJ_CODE_HOME", &home);
@@ -324,6 +433,62 @@ impl Hub {
self.send(req).await
}
+ /// A GET from a fresh client address each time, so the per-address rate
+ /// limit never fires: for tests that send many requests and are about
+ /// something else.
+ async fn stranger(&self, path: &str) -> Reply {
+ let n = NEXT_PEER.fetch_add(1, Ordering::Relaxed);
+ let peer = format!("10.{}.{}.{}", (n >> 16) & 255, (n >> 8) & 255, n & 255);
+ self.get_from(path, &peer, None).await
+ }
+
+ /// `stranger`, presenting `ApiKey <encoded>`.
+ async fn stranger_with_key(&self, path: &str, encoded: &str) -> Reply {
+ let n = NEXT_PEER.fetch_add(1, Ordering::Relaxed);
+ let peer = format!("10.{}.{}.{}", (n >> 16) & 255, (n >> 8) & 255, n & 255);
+ let mut req = Self::req("GET", path)
+ .header(header::AUTHORIZATION, format!("ApiKey {encoded}"))
+ .body(Body::empty())
+ .unwrap();
+ let addr: SocketAddr = format!("{peer}:40000").parse().expect("peer");
+ req.extensions_mut().insert(ConnectInfo(addr));
+ self.send(req).await
+ }
+
+ /// Mint an API key (as the admin) that may read only `patterns`; returns the
+ /// `encoded` value to present as `ApiKey <encoded>`.
+ async fn mint_read_key(&self, name: &str, patterns: &[&str]) -> String {
+ let r = self
+ .admin(
+ "POST",
+ "/_security/api_key",
+ Some(&json!({
+ "name": name,
+ "role_descriptors": { "r": { "indices": [
+ { "names": patterns, "privileges": ["read"] }
+ ]}}
+ })),
+ )
+ .await;
+ assert!(r.status.is_success(), "mint {name}: {:?}", r.json());
+ r.json()["encoded"]
+ .as_str()
+ .unwrap_or_else(|| panic!("no encoded key in {:?}", r.json()))
+ .to_string()
+ }
+
+ /// A direct (non-hub) `POST /{index}/_search` presenting `ApiKey <encoded>`.
+ async fn direct_search_with_key(&self, index: &str, encoded: &str) -> Reply {
+ self.send(
+ Self::req("POST", &format!("/{index}/_search"))
+ .header(header::AUTHORIZATION, format!("ApiKey {encoded}"))
+ .header(header::CONTENT_TYPE, "application/json")
+ .body(Body::from(json!({"query": {"match_all": {}}}).to_string()))
+ .unwrap(),
+ )
+ .await
+ }
+
/// A request of any method as if it arrived from `peer`.
async fn method_from(&self, method: &str, path: &str, peer: &str) -> Reply {
let mut req = Self::req(method, path).body(Body::empty()).unwrap();
@@ -560,13 +725,53 @@ async fn seed(h: &Hub) {
.await;
assert!(alias.status.is_success(), "alias: {:?}", alias.json());
+ // ── corpora whose catalogue entry and node ledger disagree about the licence ──
+ //
+ // Each gets a docs index (`frobnicate …`) and a defs index, so every route
+ // has something of theirs to find.
+ for (slug, limit) in [
+ ("ledger-closed", "LEDGER_CLOSED_LIMIT"),
+ ("cat-closed", "CAT_CLOSED_LIMIT"),
+ ("gplish", "GPLISH_LIMIT"),
+ ("ledger-gpl", "LEDGER_GPL_LIMIT"),
+ ] {
+ h.create(&format!("xc-{slug}-b1-defs"), defs_mapping())
+ .await;
+ h.bulk(
+ &format!("xc-{slug}-b1-defs"),
+ &[def(
+ limit,
+ "define",
+ "limits.h",
+ 3,
+ &format!("#define {limit} 9"),
+ Some("9"),
+ )],
+ )
+ .await;
+ h.create(&format!("xc-{slug}-b1-docs"), docs_mapping())
+ .await;
+ h.bulk(
+ &format!("xc-{slug}-b1-docs"),
+ &[json!({"ax_path": format!("{slug}/s.c"),
+ "body": format!("frobnicate {slug} text frobnicate"),
+ "defs": format!("function {slug}_fn"),
+ "symbols": [{"name": format!("{slug}_fn"), "kind": "function", "line": 1}]})],
+ )
+ .await;
+ }
+
// Searchable now, not after the next refresh tick.
let r = h
.admin(
"POST",
"/xc-mini-c-b1-defs,xc-mini-c-b1-docs,xc-mini-gpl-b2-defs,xc-mini-gpl-b2-docs,\
xc-mini-closed-b1-defs,xc-mini-closed-b1-docs,xc-orphan-b1-docs,\
- xc-leaky-b1-docs,secret-notes/_refresh",
+ xc-leaky-b1-docs,secret-notes,\
+ xc-ledger-closed-b1-defs,xc-ledger-closed-b1-docs,\
+ xc-cat-closed-b1-defs,xc-cat-closed-b1-docs,\
+ xc-gplish-b1-defs,xc-gplish-b1-docs,\
+ xc-ledger-gpl-b1-defs,xc-ledger-gpl-b1-docs/_refresh",
None,
)
.await;
@@ -828,14 +1033,70 @@ async fn code_max_tokens_holds_the_answer_and_says_so() {
}
let r = h.get("/_hub/v1/code/mini-c?q=x&mode=nope").await;
assert_eq!(r.status, StatusCode::BAD_REQUEST);
- // The other arms are accepted; the answer warns that it cannot say which ran.
- let r = h.get("/_hub/v1/code/mini-c?q=mini_hash&mode=hybrid").await;
- assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
- assert!(r
- .answer()
- .warnings
- .iter()
- .any(|w| w.contains("semantic/hybrid")));
+ // The other arms are accepted (what they do is pinned in
+ // `semantic_and_hybrid_say_what_they_actually_do`).
+ for mode in ["semantic", "hybrid"] {
+ let r = h
+ .get(&format!("/_hub/v1/code/mini-c?q=mini_hash&mode={mode}"))
+ .await;
+ assert_eq!(r.status, StatusCode::OK, "{mode}: {:?}", r.json());
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn semantic_and_hybrid_say_what_they_actually_do() {
+ // The fixture corpus maps `body` as plain text: it has no `semantic_text`
+ // index for a vector arm to search.
+ let h = hub(Opts::new(false, false)).await;
+ let bm25 = h
+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned")
+ .await
+ .answer();
+ assert!(!bm25.hits.is_empty());
+ assert!(bm25.warnings.is_empty(), "{:?}", bm25.warnings);
+
+ // `semantic` over a corpus with no semantic index finds NOTHING; it does not
+ // fall back to BM25 (that is what `hybrid` does). The answer must say so.
+ let sem = h
+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned&mode=semantic")
+ .await;
+ assert_eq!(sem.status, StatusCode::OK, "{:?}", sem.json());
+ let sem = sem.answer();
+ assert!(
+ sem.hits.is_empty(),
+ "the premise: semantic returns no hits here: {:#?}",
+ sem.hits
+ );
+ let said = sem.warnings.join(" | ");
+ assert!(!said.is_empty(), "an empty semantic answer explains itself");
+ assert!(
+ !said.contains("falls back to BM25"),
+ "semantic does not fall back, and must not claim to: {said}"
+ );
+ assert!(
+ said.contains("semantic_text") && said.contains("no hits"),
+ "names why it is empty: {said}"
+ );
+
+ // `hybrid` over the same corpus answers from BM25 alone, and says that is
+ // what happened.
+ let hyb = h
+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned&mode=hybrid")
+ .await
+ .answer();
+ assert!(
+ !hyb.hits.is_empty(),
+ "hybrid degrades to BM25 and still answers"
+ );
+ let said = hyb.warnings.join(" | ");
+ assert!(
+ said.contains("BM25") && said.contains("semantic_text"),
+ "names the arm it used and the mapping it lacked: {said}"
+ );
+ assert_ne!(
+ sem.warnings, hyb.warnings,
+ "the two modes behave differently, so they are described differently"
+ );
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
@@ -1216,10 +1477,28 @@ async fn unauthenticated_hub_traffic_cannot_write_the_audit_ring() {
let h = hub(Opts::new(true, true)).await;
let audit = &h.state.engine.audit;
+ // The operator's evidence: whatever is in the ring before any stranger
+ // arrives (the seeding above wrote real entries).
+ let before = audit.next_seq();
+ let evidence = audit.snapshot();
+ assert!(
+ !evidence.is_empty(),
+ "fixture sanity: the ring holds the operator's own entries"
+ );
+ let unchanged = |what: &str| {
+ assert_eq!(audit.next_seq(), before, "{what}: the sequence moved");
+ let now = audit.snapshot();
+ assert_eq!(now.len(), evidence.len(), "{what}: the ring changed size");
+ assert_eq!(
+ now.iter().map(|e| e.hash.as_str()).collect::<Vec<_>>(),
+ evidence.iter().map(|e| e.hash.as_str()).collect::<Vec<_>>(),
+ "{what}: the ring's entries changed"
+ );
+ };
+
// A flood of non-GET requests to the public paths and of 401s elsewhere
// leaves nothing: the audit layer records unauthenticated write-shaped
// requests, and it must not see these.
- let before = audit.next_seq();
// 4 rounds x 13 hub requests = 52, inside the 60-a-minute budget, so each
// one reaches the layer that would have recorded it.
for _ in 0..4 {
@@ -1232,20 +1511,61 @@ async fn unauthenticated_hub_traffic_cannot_write_the_audit_ring() {
h.get("/_cluster/health").await;
h.get(FOUR[0]).await;
}
- assert_eq!(
- audit.next_seq(),
- before,
- "no entry from unauthenticated hub traffic"
- );
+ unchanged("write-shaped and unauthenticated requests");
+
+ // The reads themselves. Every valid `def`, `code` and `search` GET is served
+ // by the node's own `_search` handler, which audits what it serves; the hub's
+ // in-process call must not. Distinct client addresses, so the rate limit
+ // does not stand in the way: this is what a crowd of strangers sends.
+ let reads = [
+ "/_hub/v1/def/mini-c/MINI_LIMIT",
+ "/_hub/v1/code/mini-c?q=mini_hash+unsigned",
+ "/_hub/v1/search?q=frobnicate",
+ ];
+ // More reads than the ring holds (4,096): had each one appended an entry,
+ // not one of the operator's entries would be left.
+ let mut sent = 0usize;
+ for round in 0..1400 {
+ for path in reads {
+ let r = h.stranger(path).await;
+ assert_eq!(r.status, StatusCode::OK, "round {round} {path}");
+ sent += 1;
+ }
+ }
+ assert!(sent > xerj_engine::audit::DEFAULT_AUDIT_CAPACITY, "{sent}");
+ unchanged("4,200 anonymous def/code/search reads");
- // A hosted search is the one thing that does append: the `_search` handler
- // audits itself, unconditionally (documented in hub_api.rs).
- h.get(FOUR[1]).await;
+ // The same holds for a caller who does present the key: the hub's reads are
+ // not a source of audit entries in either mode.
+ for path in reads {
+ assert_eq!(h.get_keyed(path).await.status, StatusCode::OK);
+ }
+ unchanged("keyed hub reads");
+
+ // The suppression belongs to the hub's in-process call and nothing else: a
+ // direct `_search` by the operator is still audited.
+ let r = h
+ .direct_search_with_key("xc-mini-c-b1-docs", ADMIN_KEY)
+ .await;
+ assert!(r.status.is_success(), "{:?}", r.json());
let after = audit.snapshot();
let new: Vec<_> = after.iter().filter(|e| e.seq >= before).collect();
- assert!(
- !new.is_empty() && new.iter().all(|e| e.op == "search"),
- "{new:?}"
+ assert_eq!(new.len(), 1, "exactly the direct search: {new:?}");
+ assert_eq!(new[0].op, "search");
+ assert_eq!(new[0].resource, "xc-mini-c-b1-docs");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_keyed_node_does_not_audit_hub_reads_either() {
+ let h = hub(Opts::new(false, true)).await;
+ let before = h.state.engine.audit.next_seq();
+ for path in FOUR {
+ assert_eq!(h.get_keyed(path).await.status, StatusCode::OK, "{path}");
+ }
+ assert_eq!(
+ h.state.engine.audit.next_seq(),
+ before,
+ "a hub read is not an audited data-path operation"
);
}
@@ -1684,6 +2004,905 @@ async fn a_full_house_is_a_503_with_retry_after_and_recovers() {
assert_eq!(r.status(), StatusCode::NOT_FOUND);
}
+// ─────────────────────────────────────────────────────────────────────────────
+// Responses too large for the in-process cap
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// The hub's routes alone over `state`, with the in-process limits lowered.
+fn lowered(
+ state: &AppState,
+ response_cap_bytes: usize,
+ fields_failure_ttl: Duration,
+) -> (axum::Router, std::sync::Arc<hub_api::HubShared>) {
+ let shared = hub_api::HubShared::with_tuning(
+ state.clone(),
+ hub_api::Tuning {
+ response_cap_bytes,
+ fields_failure_ttl,
+ ..Default::default()
+ },
+ );
+ (hub_api::routes(shared.clone()), shared)
+}
+
+async fn get_on(app: &axum::Router, path: &str) -> Reply {
+ let resp = app
+ .clone()
+ .oneshot(Request::get(path).body(Body::empty()).unwrap())
+ .await
+ .expect("response");
+ let status = resp.status();
+ let headers = resp.headers().clone();
+ let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ Reply {
+ status,
+ headers,
+ body,
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_node_wide_mapping_over_the_cap_is_a_503_not_an_empty_search() {
+ let h = hub(Opts::new(true, true)).await;
+ // Premise: with the production cap the same node searches fine.
+ let ok = h.stranger("/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(ok.status, StatusCode::OK);
+ assert!(!ok.json()["corpora"].as_array().unwrap().is_empty());
+
+ // What a pack with enormous mappings does at 16 MiB, in miniature: the
+ // node-wide `xc-*` mapping is one byte too large for the cap on a single
+ // in-process response (and everything else on the node still fits).
+ let mapping = h.admin("GET", "/xc-*/_mapping", None).await;
+ let cap = mapping.body.len() - 1;
+ assert!(cap > 2048, "fixture sanity: a {cap}-byte mapping");
+ let ttl = Duration::from_millis(600);
+ let (app, shared) = lowered(&h.state, cap, ttl);
+
+ let r = get_on(&app, "/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(
+ r.status,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "an unreadable mapping is an outage, not 'no results': {:?}",
+ r.json()
+ );
+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));
+ let text = String::from_utf8_lossy(&r.body).to_string();
+ for leaked in [cap.to_string().as_str(), "bytes", "mapping", "xc-", "over"] {
+ assert!(!text.contains(leaked), "leaked {leaked:?}: {text}");
+ }
+ assert_ne!(r.json(), json!({"corpora": []}));
+
+ // The failure is remembered: a burst of requests does not repeat the
+ // expensive read (each repeat costs seconds at the real size).
+ for _ in 0..6 {
+ let r = get_on(&app, "/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE);
+ }
+ assert_eq!(
+ shared.mapping_reads_for_test(),
+ 1,
+ "seven requests, one read of the mappings"
+ );
+
+ // …but not for long: a transient fault clears itself, and the read is
+ // tried again once the memory expires.
+ tokio::time::sleep(ttl + Duration::from_millis(300)).await;
+ let r = get_on(&app, "/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE);
+ assert_eq!(
+ shared.mapping_reads_for_test(),
+ 2,
+ "read again after the ttl"
+ );
+
+ // Routes about one corpus do not read the node-wide mapping and are not
+ // taken down with `search`.
+ let r = get_on(&app, "/_hub/v1/def/mini-c/MINI_LIMIT").await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn records_too_big_to_return_together_shrink_the_window_and_say_so() {
+ let h = hub(Opts::new(false, false)).await;
+ // Two records of ~48 KiB that both match; `code` asks the engine for whole
+ // records, ten of them at once even for k=1.
+ let filler = "int x; ".repeat(48 * 1024 / 7);
+ h.create("xc-big-b1-docs", docs_mapping()).await;
+ h.bulk(
+ "xc-big-b1-docs",
+ &[
+ json!({"ax_path": "big/a.c", "title": "a.c",
+ "body": format!("needleword alpha {filler}"), "defs": "function needleword"}),
+ json!({"ax_path": "big/b.c", "title": "b.c",
+ "body": format!("needleword beta {filler}"), "defs": "function needleword"}),
+ ],
+ )
+ .await;
+ h.admin("POST", "/xc-big-b1-docs/_refresh", None).await;
+
+ // Premise: with the production cap both fit, and nothing is cut.
+ let whole = h.get("/_hub/v1/code/big?q=needleword&k=1").await;
+ assert_eq!(whole.status, StatusCode::OK, "{:?}", whole.json());
+ let whole = whole.answer();
+ assert_eq!(whole.hits.len(), 1);
+ assert!(!whole.truncated, "{:?}", whole.warnings);
+ assert!(whole.warnings.is_empty(), "{:?}", whole.warnings);
+
+ // With the cap below two records (what two 9 MiB records do to 16 MiB), the
+ // answer is built from fewer candidates and says so — not a 503.
+ let (app, _shared) = lowered(&h.state, 64 * 1024, Duration::from_secs(15));
+ for k in [1, 2, 5] {
+ let r = get_on(&app, &format!("/_hub/v1/code/big?q=needleword&k={k}")).await;
+ assert_eq!(
+ r.status,
+ StatusCode::OK,
+ "k={k}: records that do not fit together must not be a 503: {:?}",
+ r.json()
+ );
+ let a = r.answer();
+ assert_eq!(a.corpus, "big");
+ assert_eq!(a.hits.len(), 1, "k={k}: the one record that fits");
+ assert!(
+ a.hits[0].file == "big/a.c" || a.hits[0].file == "big/b.c",
+ "{:?}",
+ a.hits[0]
+ );
+ assert!(a.hits[0].score > 0.0);
+ assert!(a.truncated, "k={k}: the answer is cut and says so");
+ assert!(
+ a.warnings.iter().any(|w| w.contains("too large")),
+ "k={k}: {:?}",
+ a.warnings
+ );
+ }
+
+ // A single record over the cap cannot be returned at all, and that is a
+ // fixed 503 rather than a made-up answer.
+ let (tiny, _shared) = lowered(&h.state, 8 * 1024, Duration::from_secs(15));
+ let r = get_on(&tiny, "/_hub/v1/code/big?q=needleword&k=1").await;
+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE, "{:?}", r.json());
+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// What a pack's own ledger cannot talk the node into
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Text that lives only in somebody else's indices: the excluded corpus, an
+/// index no catalogue entry lists, and the private index behind an alias.
+const OTHER_PEOPLES_DATA: [&str; 6] = [
+ "closed text",
+ "orphan text",
+ "CLOSED_LIMIT",
+ "mini-closed/",
+ "orphan/",
+ "hunter2",
+];
+
+fn assert_no_leak(what: &str, r: &Reply) {
+ let text = String::from_utf8_lossy(&r.body);
+ for needle in OTHER_PEOPLES_DATA {
+ assert!(
+ !text.contains(needle),
+ "{what} leaked {needle:?} (status {}): {text}",
+ r.status
+ );
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_ledger_prefix_that_is_not_the_corpus_own_build_is_never_queried() {
+ // A public node: the stranger is exactly who this guards against.
+ let h = hub(Opts::new(true, true)).await;
+
+ // Premises. A well-formed corpus answers; and the data a hostile prefix
+ // would reach is really on this node (without these the test proves nothing).
+ let ok = h.stranger("/_hub/v1/code/mini-c?q=frobnicate&k=20").await;
+ assert_eq!(ok.status, StatusCode::OK, "{:?}", ok.json());
+ assert!(!ok.answer().hits.is_empty());
+ for (index, needle) in [
+ ("xc-mini-closed-b1-docs", "closed text"),
+ ("xc-orphan-b1-docs", "orphan text"),
+ ("xc-mini-closed-b1-defs", "CLOSED_LIMIT"),
+ ] {
+ let r = h
+ .admin(
+ "POST",
+ &format!("/{index}/_search"),
+ Some(&json!({"query": {"match_all": {}}})),
+ )
+ .await;
+ assert!(
+ String::from_utf8_lossy(&r.body).contains(needle),
+ "fixture sanity: {index} holds {needle:?}"
+ );
+ }
+
+ // Every way a ledger can name a prefix that is not `xc-<its own slug>-b<n>`.
+ // Each is refused before any query, with the same fixed 503, and none of
+ // them reads another corpus.
+ for slug in hostile_slugs() {
+ for path in [
+ format!("/_hub/v1/code/{slug}?q=frobnicate&k=20"),
+ format!("/_hub/v1/code/{slug}?q=closed"),
+ format!("/_hub/v1/code/{slug}?q=orphan"),
+ format!("/_hub/v1/code/{slug}?q=frobnicate&mode=hybrid"),
+ format!("/_hub/v1/def/{slug}/CLOSED_LIMIT"),
+ format!("/_hub/v1/def/{slug}/MINI_LIMIT"),
+ ] {
+ let r = h.stranger(&path).await;
+ assert_no_leak(&path, &r);
+ assert_eq!(
+ r.status,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "{path}: {:?}",
+ r.json()
+ );
+ assert_eq!(
+ r.json()["error"]["type"],
+ json!("hub_unavailable"),
+ "{path}"
+ );
+ let text = String::from_utf8_lossy(&r.body).to_string();
+ for leaked in ["xc-", "prefix", "ledger", "state", slug] {
+ assert!(!text.contains(leaked), "{path} said {leaked:?}: {text}");
+ }
+ }
+ }
+
+ // `search` does not list them either, and still lists the honest corpora.
+ let r = h.stranger("/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ assert_no_leak("search", &r);
+ let slugs: Vec<String> = r.json()["corpora"]
+ .as_array()
+ .expect("corpora")
+ .iter()
+ .map(|c| c["slug"].as_str().unwrap().to_string())
+ .collect();
+ assert!(slugs.contains(&"mini-c".to_string()), "{slugs:?}");
+ for slug in hostile_slugs() {
+ assert!(!slugs.contains(&slug.to_string()), "{slug} in {slugs:?}");
+ }
+}
+
+/// While armed, [`rewrite_flip_ledger`] does what a `xerj hub pull` landing at
+/// the worst moment would.
+static FLIP_ARMED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
+/// Replace the `flip` corpus's ledger with one naming the whole namespace. Run
+/// by the hub right after the ledger passed its check and before the shared
+/// query pipeline reads it a second time.
+fn rewrite_flip_ledger(root: &std::path::Path, slug: &str) {
+ if slug == "flip" && FLIP_ARMED.load(Ordering::SeqCst) {
+ write_hubpack_state(
+ root,
+ slug,
+ "http://x",
+ "xc-",
+ PIN_SHA,
+ PIN_DATE,
+ "MIT",
+ "adapt-with-attribution",
+ )
+ .expect("rewrite the ledger");
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_ledger_that_changes_after_it_was_checked_still_cannot_leave_its_build() {
+ // The `def` and `code` pipelines read the ledger themselves, a second time,
+ // to decide what to query. Checking the first read is not enough if the
+ // second can differ (a `xerj hub pull` landing mid-request).
+ let h = hub(Opts::new(true, true)).await;
+ h.create("xc-flip-b1-docs", docs_mapping()).await;
+ h.bulk(
+ "xc-flip-b1-docs",
+ &[json!({"ax_path": "flip/f.c", "body": "frobnicate flip text"})],
+ )
+ .await;
+ h.admin("POST", "/xc-flip-b1-docs/_refresh", None).await;
+ let shared = hub_api::HubShared::with_tuning(
+ h.state.clone(),
+ hub_api::Tuning {
+ after_ledger_check: Some(rewrite_flip_ledger),
+ ..Default::default()
+ },
+ );
+ let app: axum::Router = hub_api::routes(shared);
+ let ledger = std::path::PathBuf::from(std::env::var("XERJ_CODE_HOME").unwrap())
+ .join("state")
+ .join("flip.json");
+
+ // Premise: with the ledger as written, the corpus answers from its own data.
+ FLIP_ARMED.store(false, Ordering::SeqCst);
+ let r = get_on(&app, "/_hub/v1/code/flip?q=frobnicate&k=20").await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ let hits = r.answer().hits;
+ assert!(!hits.is_empty());
+ assert!(
+ hits.iter().all(|x| x.file.starts_with("flip/")),
+ "{hits:#?}"
+ );
+
+ // Armed: the ledger names `xc-` by the time the pipeline reads it. Without
+ // the pin that is every hub index; with it, a refusal.
+ FLIP_ARMED.store(true, Ordering::SeqCst);
+ for path in [
+ "/_hub/v1/code/flip?q=frobnicate&k=20",
+ "/_hub/v1/code/flip?q=closed",
+ "/_hub/v1/def/flip/CLOSED_LIMIT",
+ ] {
+ let r = get_on(&app, path).await;
+ assert_no_leak(path, &r);
+ assert_eq!(
+ r.status,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "{path}: {:?}",
+ r.json()
+ );
+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));
+ }
+ let on_disk = std::fs::read_to_string(&ledger).expect("ledger");
+ assert!(
+ on_disk.contains("\"index_prefix\":\"xc-\""),
+ "premise: the ledger really was replaced mid-request: {on_disk}"
+ );
+
+ // Put it back for anything that runs after this.
+ FLIP_ARMED.store(false, Ordering::SeqCst);
+ write_hubpack_state(
+ ledger.parent().and_then(|p| p.parent()).unwrap(),
+ "flip",
+ "http://x",
+ "xc-flip-b1",
+ PIN_SHA,
+ PIN_DATE,
+ "MIT",
+ "adapt-with-attribution",
+ )
+ .expect("restore the ledger");
+}
+
+#[test]
+fn only_the_exact_build_prefix_shape_is_accepted_for_a_ledger() {
+ // `xc-<slug>-b<digits>` for the corpus's OWN slug, in every accepted shape:
+ // digits of any length, slugs containing `-`, `.`, `_` and `+`.
+ for (slug, prefix) in [
+ ("mini-c", "xc-mini-c-b1"),
+ ("mini-c", "xc-mini-c-b20260921"),
+ ("mini-gpl", "xc-mini-gpl-b2"),
+ ("rust-vulns", "xc-rust-vulns-b1700000000"),
+ ("c++", "xc-c++-b3"),
+ ("a.b_c", "xc-a.b_c-b0"),
+ // A slug that itself ends in a build-like token is still that slug.
+ ("lib-b2", "xc-lib-b2-b7"),
+ ] {
+ assert!(
+ hub_api::valid_build_prefix(prefix, slug),
+ "{prefix} for {slug}"
+ );
+ }
+ for (slug, prefix) in [
+ ("mini-c", "xc-"),
+ ("mini-c", "xc-*"),
+ ("mini-c", "xc-mini-c"),
+ ("mini-c", "xc-mini-c-b"),
+ ("mini-c", "xc-mini-c-b1x"),
+ ("mini-c", "xc-mini-c-bx1"),
+ ("mini-c", "xc-mini-c-b1-defs"),
+ ("mini-c", "xc-mini-c-b1*"),
+ ("mini-c", "xc-mini-c-b1,xc-mini-closed-b1"),
+ ("mini-c", "xc-mini-c-b1 "),
+ ("mini-c", " xc-mini-c-b1"),
+ ("mini-c", "XC-mini-c-b1"),
+ ("mini-c", "xc-MINI-C-b1"),
+ ("mini-c", "xc-mini-gpl-b2"),
+ ("mini-c", "xc-mini-closed-b1"),
+ ("mini-c", "xc-mini-c-b-1"),
+ ("mini-c", "xc-mini-c-b+1"),
+ ("mini-c", "xc-mini-c-b١"),
+ ("mini-c", ""),
+ ("mini-c", "mini-c-b1"),
+ ("mini", "xc-mini-c-b1"),
+ ] {
+ assert!(
+ !hub_api::valid_build_prefix(prefix, slug),
+ "{prefix:?} must not be accepted for {slug}"
+ );
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_scoped_key_sees_the_hub_through_its_own_grants_only() {
+ // A keyed, non-public node: callers are principals with grants, and the hub
+ // routes must not widen what a grant allows.
+ let h = hub(Opts::new(false, true)).await;
+
+ // A tenant key with nothing under `xc-*`.
+ let tenant = h.mint_read_key("tenant-a", &["tenant-a-*"]).await;
+ let direct = h.direct_search_with_key("xc-mini-c-b1-docs", &tenant).await;
+ assert_eq!(
+ direct.status,
+ StatusCode::FORBIDDEN,
+ "premise: the ordinary API refuses this key the corpus: {:?}",
+ direct.json()
+ );
+ let r = h
+ .stranger_with_key("/_hub/v1/code/mini-c?q=frobnicate", &tenant)
+ .await;
+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{:?}", r.json());
+ assert_eq!(r.json()["error"]["type"], json!("corpus_not_loaded"));
+ let r = h
+ .stranger_with_key("/_hub/v1/def/mini-c/MINI_LIMIT", &tenant)
+ .await;
+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{:?}", r.json());
+ assert_eq!(r.json()["error"]["type"], json!("corpus_not_loaded"));
+ let r = h
+ .stranger_with_key("/_hub/v1/search?q=frobnicate", &tenant)
+ .await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ assert_eq!(
+ r.json(),
+ json!({"corpora": []}),
+ "the key sees none of the hub, so `search` finds none of it"
+ );
+ for text in ["mini-c", "frobnicate the buffer", "MINI_LIMIT", "128"] {
+ assert!(
+ !String::from_utf8_lossy(&r.body).contains(text),
+ "{text} reached a key that holds nothing under xc-*"
+ );
+ }
+
+ // A key granted ONE corpus sees that corpus through the hub, and no other:
+ // the hub narrows to its own namespace, it does not widen to it.
+ let mini = h.mint_read_key("mini-reader", &["xc-mini-c-*"]).await;
+ let ok = h.direct_search_with_key("xc-mini-c-b1-docs", &mini).await;
+ assert!(ok.status.is_success(), "premise: granted: {:?}", ok.json());
+ let no = h.direct_search_with_key("xc-mini-gpl-b2-docs", &mini).await;
+ assert_eq!(no.status, StatusCode::FORBIDDEN, "premise: not granted");
+
+ let r = h
+ .stranger_with_key("/_hub/v1/code/mini-c?q=mini_hash+unsigned", &mini)
+ .await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ assert!(!r.answer().hits.is_empty(), "its own corpus is answered");
+ let r = h
+ .stranger_with_key("/_hub/v1/def/mini-c/MINI_LIMIT", &mini)
+ .await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ assert_eq!(r.answer().hits.len(), 1);
+ for path in [
+ "/_hub/v1/code/mini-gpl?q=frobnicate",
+ "/_hub/v1/def/mini-gpl/GPL_LIMIT",
+ ] {
+ let r = h.stranger_with_key(path, &mini).await;
+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{path}: {:?}", r.json());
+ assert_eq!(r.json()["error"]["type"], json!("corpus_not_loaded"));
+ }
+ let r = h
+ .stranger_with_key("/_hub/v1/search?q=frobnicate", &mini)
+ .await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ let slugs: Vec<String> = r.json()["corpora"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|c| c["slug"].as_str().unwrap().to_string())
+ .collect();
+ assert_eq!(slugs, ["mini-c"], "only the granted corpus is searched");
+
+ // The admin key is unrestricted, and it is the same hub: both corpora.
+ let r = h.get_keyed("/_hub/v1/search?q=frobnicate").await;
+ let all: Vec<String> = r.json()["corpora"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|c| c["slug"].as_str().unwrap().to_string())
+ .collect();
+ assert!(
+ all.contains(&"mini-c".to_string()) && all.contains(&"mini-gpl".to_string()),
+ "{all:?}"
+ );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_scoped_caller_neither_reads_nor_poisons_the_shared_field_list() {
+ // `search` learns which of body/defs/symbol_text exist from the node's
+ // mappings and reuses that for a minute. A caller who sees only part of the
+ // node reads a different list; it must not be shared in either direction.
+ async fn node() -> (Hub, String) {
+ let h = hub(Opts::new(false, true)).await;
+ h.create(
+ "xc-bodyonly-b1-docs",
+ json!({"properties": {"body": {"type": "text"}, "ax_path": {"type": "keyword"}}}),
+ )
+ .await;
+ h.bulk(
+ "xc-bodyonly-b1-docs",
+ &[json!({"ax_path": "bodyonly/b.c", "body": "frobnicate quasar nebula"})],
+ )
+ .await;
+ h.admin("POST", "/xc-bodyonly-b1-docs/_refresh", None).await;
+ let key = h.mint_read_key("bodyonly", &["xc-bodyonly-*"]).await;
+ (h, key)
+ }
+ let slugs = |r: &Reply| -> Vec<String> {
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ r.json()["corpora"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|c| c["slug"].as_str().unwrap().to_string())
+ .collect()
+ };
+
+ // The operator searches first, so the shared list names all three fields; the
+ // scoped key (whose only index maps `body`) must not borrow it.
+ let (h, key) = node().await;
+ assert!(slugs(&h.get_keyed("/_hub/v1/search?q=frobnicate").await).contains(&"mini-c".into()));
+ let r = h
+ .stranger_with_key("/_hub/v1/search?q=frobnicate+quasar", &key)
+ .await;
+ assert_eq!(
+ slugs(&r),
+ ["bodyonly"],
+ "the scoped key finds its own corpus"
+ );
+
+ // The scoped key searches first; the operator must not inherit its view,
+ // where only `body` exists: a definition name lives in `symbol_text`.
+ let (h, key) = node().await;
+ let r = h
+ .stranger_with_key("/_hub/v1/search?q=frobnicate+quasar", &key)
+ .await;
+ assert_eq!(slugs(&r), ["bodyonly"]);
+ let admin = slugs(&h.get_keyed("/_hub/v1/search?q=MINI_LIMIT").await);
+ assert_eq!(
+ admin.first().map(String::as_str),
+ Some("mini-c"),
+ "a definition is found through `symbol_text`: {admin:?}"
+ );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn the_stricter_of_the_catalogue_and_the_ledger_governs_every_route() {
+ let h = hub(Opts::new(false, false)).await;
+ let note = |spdx: &str| format!("approach-only licence ({spdx}): read, do not copy");
+
+ // Premises: these corpora are really indexed, so an absence below is the
+ // licence rule at work and not an empty fixture.
+ for slug in ["ledger-closed", "cat-closed", "gplish", "ledger-gpl"] {
+ let r = h
+ .admin(
+ "POST",
+ &format!("/xc-{slug}-b1-docs/_search"),
+ Some(&json!({"query": {"match": {"body": "frobnicate"}}})),
+ )
+ .await;
+ assert!(
+ String::from_utf8_lossy(&r.body).contains(&format!("{slug} text")),
+ "fixture sanity: {slug} is indexed"
+ );
+ }
+
+ // Either side can refuse: catalogue-servable but ledger-excluded, and the
+ // other way round. Same 404 everywhere, and `search` never returns a hit.
+ for slug in ["ledger-closed", "cat-closed"] {
+ for path in [
+ format!("/_hub/v1/def/{slug}/X"),
+ format!("/_hub/v1/code/{slug}?q=frobnicate"),
+ ] {
+ let r = h.get(&path).await;
+ assert_eq!(r.status, StatusCode::NOT_FOUND, "{path}");
+ assert_eq!(
+ r.json()["error"]["type"],
+ json!("resource_not_found_exception"),
+ "{path}: refusals look alike"
+ );
+ }
+ }
+ for q in ["frobnicate", "ledger-closed", "cat-closed"] {
+ let r = h.get(&format!("/_hub/v1/search?q={q}&k=10")).await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+ let text = String::from_utf8_lossy(&r.body).to_string();
+ assert!(
+ !text.contains("ledger-closed") && !text.contains("cat-closed"),
+ "search q={q} returned a refused corpus: {text}"
+ );
+ assert!(
+ !text.contains("ledger-closed text") && !text.contains("cat-closed text"),
+ "search q={q}: {text}"
+ );
+ }
+
+ // Approach-only on EITHER side is approach-only: the header says so, the
+ // licence it names is the one that decided, and every hit carries the note.
+ for (slug, spdx, symbol) in [
+ ("gplish", "GPL-3.0-only", "GPLISH_LIMIT"),
+ ("ledger-gpl", "AGPL-3.0-only", "LEDGER_GPL_LIMIT"),
+ ] {
+ let d = h.get(&format!("/_hub/v1/def/{slug}/{symbol}")).await;
+ assert_eq!(d.status, StatusCode::OK, "{:?}", d.json());
+ let d = d.answer();
+ assert_eq!(d.licence_class, LicenceClass::ApproachOnly, "{slug} def");
+ assert_eq!(d.licence_spdx, spdx, "{slug} def");
+ assert_eq!(d.hits.len(), 1, "{slug} def");
+ assert_eq!(
+ d.hits[0].licence_warning.as_deref(),
+ Some(note(spdx).as_str())
+ );
+
+ let c = h.get(&format!("/_hub/v1/code/{slug}?q=frobnicate")).await;
+ assert_eq!(c.status, StatusCode::OK, "{:?}", c.json());
+ let c = c.answer();
+ assert_eq!(c.licence_class, LicenceClass::ApproachOnly, "{slug} code");
+ assert_eq!(c.licence_spdx, spdx, "{slug} code");
+ assert!(!c.hits.is_empty(), "{slug} code");
+ assert!(
+ c.hits
+ .iter()
+ .all(|x| x.licence_warning.as_deref() == Some(note(spdx).as_str())),
+ "{slug} code: {:#?}",
+ c.hits
+ );
+
+ // `def` and `code` describe the corpus identically (the pin test's rule).
+ assert_eq!(
+ (&d.corpus, &d.licence_spdx, d.licence_class),
+ (&c.corpus, &c.licence_spdx, c.licence_class)
+ );
+
+ let r = h.get("/_hub/v1/search?q=frobnicate&k=10").await;
+ let rows = r.json()["corpora"].as_array().cloned().unwrap();
+ let row = rows
+ .iter()
+ .find(|row| row["slug"] == slug)
+ .unwrap_or_else(|| panic!("{slug} missing from search: {rows:?}"));
+ assert_eq!(
+ row["best_hit"]["licence_warning"],
+ json!(note(spdx)),
+ "{slug} search"
+ );
+ }
+
+ // And the permissive corpora are untouched by all this.
+ let c = h
+ .get("/_hub/v1/code/mini-c?q=mini_hash+unsigned")
+ .await
+ .answer();
+ assert_eq!(c.licence_class, LicenceClass::AdaptWithAttribution);
+ assert!(c.hits.iter().all(|x| x.licence_warning.is_none()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn one_bad_index_does_not_take_search_down_for_everyone() {
+ let h = hub(Opts::new(true, true)).await;
+ let slugs_of = |r: &Reply| -> Vec<String> {
+ r.json()["corpora"]
+ .as_array()
+ .unwrap_or_else(|| panic!("no corpora in {:?}", r.json()))
+ .iter()
+ .map(|c| c["slug"].as_str().unwrap().to_string())
+ .collect()
+ };
+ let baseline = h.stranger("/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(baseline.status, StatusCode::OK);
+ let good = slugs_of(&baseline);
+ assert!(good.contains(&"mini-c".to_string()), "{good:?}");
+
+ // ── A corpus that maps `body` as a number ──────────────────────────────────
+ // Its `defs` is ordinary text. A lenient clause skips the numeric field and
+ // still searches `defs`, so this corpus is FOUND; a strict one fails the whole
+ // index, and the corpus is missing.
+ h.create(
+ "xc-longbody-b1-docs",
+ json!({"properties": {
+ "body": {"type": "long"},
+ "defs": {"type": "text"},
+ "ax_path": {"type": "keyword"}}}),
+ )
+ .await;
+ h.bulk(
+ "xc-longbody-b1-docs",
+ &[json!({"ax_path": "longbody/n.c", "body": 5, "defs": "function frobnicate numbers"})],
+ )
+ .await;
+ h.admin("POST", "/xc-longbody-b1-docs/_refresh", None).await;
+ // Premise: the engine itself refuses a text query against that field when the
+ // clause is not lenient, and accepts it when it is — so the hub must ask.
+ let strict = h
+ .admin(
+ "POST",
+ "/xc-*/_search",
+ Some(&json!({"query": {"multi_match": {
+ "query": "frobnicate", "fields": ["body"], "lenient": false}}})),
+ )
+ .await;
+ assert!(
+ !strict.status.is_success(),
+ "premise: a strict multi_match over a long field fails the fan-out: {:?}",
+ strict.json()
+ );
+ for q in ["frobnicate", "frobnicate+buffer", "mini_clamp"] {
+ let r = h.stranger(&format!("/_hub/v1/search?q={q}")).await;
+ assert_eq!(
+ r.status,
+ StatusCode::OK,
+ "q={q} with a long-mapped body somewhere on the node: {:?}",
+ r.json()
+ );
+ }
+ let with_long = slugs_of(&h.stranger("/_hub/v1/search?q=frobnicate").await);
+ let as_set = |v: &[String]| v.iter().cloned().collect::<std::collections::BTreeSet<_>>();
+ let mut want = as_set(&good);
+ want.insert("longbody".to_string());
+ assert_eq!(
+ as_set(&with_long),
+ want,
+ "the honest corpora are answered exactly as before, and the numeric one is \
+ searched through its text field (lenient), not dropped: {with_long:?}"
+ );
+ // A query that happens to be a number is no problem for the numeric field.
+ let r = h.stranger("/_hub/v1/search?q=5").await;
+ assert_eq!(r.status, StatusCode::OK, "{:?}", r.json());
+
+ // ── A corpus whose `body` cannot be searched at all ────────────────────────
+ // Lenient does not help here (the engine refuses a NAMED unsearchable field),
+ // so this is the failure the per-index isolation exists for.
+ h.create(
+ "xc-noindex-b1-docs",
+ json!({"properties": {
+ "body": {"type": "text", "index": false},
+ "ax_path": {"type": "keyword"}}}),
+ )
+ .await;
+ h.bulk(
+ "xc-noindex-b1-docs",
+ &[json!({"ax_path": "noindex/n.c", "body": "frobnicate hidden away"})],
+ )
+ .await;
+ h.admin("POST", "/xc-noindex-b1-docs/_refresh", None).await;
+ let direct = h
+ .admin(
+ "POST",
+ "/xc-*/_search",
+ Some(&json!({"query": {"multi_match": {
+ "query": "frobnicate", "fields": ["body"], "lenient": true}}})),
+ )
+ .await;
+ assert!(
+ !direct.status.is_success(),
+ "premise: even a lenient clause fails the whole fan-out on an unsearchable field: {:?}",
+ direct.json()
+ );
+ let r = h.stranger("/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(
+ r.status,
+ StatusCode::OK,
+ "one corpus it cannot search must not 503 the rest: {:?}",
+ r.json()
+ );
+ let after = slugs_of(&r);
+ assert_eq!(
+ as_set(&after),
+ want,
+ "the bad corpus is skipped, the rest are kept: {after:?}"
+ );
+ assert!(!after.contains(&"noindex".to_string()));
+
+ // The other routes are about one corpus each and are untouched.
+ let r = h
+ .stranger("/_hub/v1/code/mini-c?q=mini_hash+unsigned")
+ .await;
+ assert_eq!(r.status, StatusCode::OK);
+ assert!(!r.answer().hits.is_empty());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn code_skips_an_index_it_cannot_search_and_says_so() {
+ let h = hub(Opts::new(true, true)).await;
+ let unsearchable = json!({"properties": {
+ "body": {"type": "text", "index": false},
+ "ax_path": {"type": "keyword"}}});
+
+ // `mixed`: one good index and one whose `body` cannot be searched.
+ h.create("xc-mixed-b1-docs", docs_mapping()).await;
+ h.bulk(
+ "xc-mixed-b1-docs",
+ &[json!({"ax_path": "mixed/good.c", "body": "frobnicate the good text"})],
+ )
+ .await;
+ h.create("xc-mixed-b1-extra", unsearchable.clone()).await;
+ h.bulk(
+ "xc-mixed-b1-extra",
+ &[json!({"ax_path": "mixed/bad.c", "body": "frobnicate the hidden text"})],
+ )
+ .await;
+ h.admin("POST", "/xc-mixed-b1-docs,xc-mixed-b1-extra/_refresh", None)
+ .await;
+ let strict = h
+ .admin(
+ "POST",
+ "/xc-mixed-b1*/_search",
+ Some(&json!({"query": {"multi_match": {"query": "frobnicate", "fields": ["body"]}}})),
+ )
+ .await;
+ assert!(
+ !strict.status.is_success(),
+ "premise: one unsearchable index fails the whole direct fan-out: {:?}",
+ strict.json()
+ );
+
+ let r = h.stranger("/_hub/v1/code/mixed?q=frobnicate").await;
+ assert_eq!(
+ r.status,
+ StatusCode::OK,
+ "one bad index must not be a 503 for its corpus: {:?}",
+ r.json()
+ );
+ let a = r.answer();
+ assert!(!a.hits.is_empty());
+ assert!(
+ a.hits.iter().all(|x| x.file == "mixed/good.c"),
+ "the answer is from the index that could be searched: {:#?}",
+ a.hits
+ );
+ assert!(
+ a.warnings
+ .iter()
+ .any(|w| w.starts_with("1 of this corpus's indices could not be searched")),
+ "a partial answer says so: {:?}",
+ a.warnings
+ );
+ // A corpus whose indices all search fine carries no such warning.
+ let ok = h
+ .stranger("/_hub/v1/code/mini-c?q=mini_hash+unsigned")
+ .await
+ .answer();
+ assert!(ok.warnings.is_empty(), "{:?}", ok.warnings);
+
+ // `allbad`: nothing could be searched, so there is no answer to give.
+ h.create("xc-allbad-b1-docs", unsearchable).await;
+ h.bulk(
+ "xc-allbad-b1-docs",
+ &[json!({"ax_path": "allbad/b.c", "body": "frobnicate hidden"})],
+ )
+ .await;
+ h.admin("POST", "/xc-allbad-b1-docs/_refresh", None).await;
+ let r = h.stranger("/_hub/v1/code/allbad?q=frobnicate").await;
+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE, "{:?}", r.json());
+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn when_every_searched_index_fails_search_says_so_instead_of_an_empty_answer() {
+ // Nothing but an unsearchable corpus on the node: "no results" would be a lie.
+ let h = bare(&Opts::new(true, true));
+ h.create(
+ "xc-noindex-b1-docs",
+ json!({"properties": {
+ "body": {"type": "text", "index": false},
+ "ax_path": {"type": "keyword"}}}),
+ )
+ .await;
+ h.bulk(
+ "xc-noindex-b1-docs",
+ &[json!({"ax_path": "noindex/n.c", "body": "frobnicate hidden away"})],
+ )
+ .await;
+ h.admin("POST", "/xc-noindex-b1-docs/_refresh", None).await;
+ let r = h.stranger("/_hub/v1/search?q=frobnicate").await;
+ assert_eq!(r.status, StatusCode::SERVICE_UNAVAILABLE, "{:?}", r.json());
+ assert_eq!(r.json()["error"]["type"], json!("hub_unavailable"));
+ let text = String::from_utf8_lossy(&r.body).to_string();
+ for leaked in ["noindex", "body", "unsearchable", "xc-", "HTTP"] {
+ assert!(!text.contains(leaked), "leaked {leaked:?}: {text}");
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Pure helpers
// ─────────────────────────────────────────────────────────────────────────────