AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(governor): parent circuit breaker, global search pool, disk flood-stage, ANN coverage, embed error class (RC4 W3 Stream A: items 1,2,3,8,9)
xerj-org11ee72fc595c
diff --git a/engine/crates/xerj-ai/src/embed.rs b/engine/crates/xerj-ai/src/embed.rsindex 7fc23bb66..55e340b66 100644--- a/engine/crates/xerj-ai/src/embed.rs+++ b/engine/crates/xerj-ai/src/embed.rs@@ -90,6 +90,37 @@ struct EmbedDatum {index: usize,}+/// A single embedding attempt's failure, tagged with whether a retry could+/// plausibly succeed (item 9). Only `retryable` failures are re-attempted by+/// [`EmbeddingProxy::send_with_retry`]; a permanent failure (4xx client error,+/// contract violation) returns immediately instead of stalling ~2 min/doc.+struct EmbedFailure {+ err: XerjError,+ retryable: bool,+}++impl EmbedFailure {+ fn transient(err: XerjError) -> Self {+ Self {+ err,+ retryable: true,+ }+ }+ fn permanent(err: XerjError) -> Self {+ Self {+ err,+ retryable: false,+ }+ }+}++/// Whether an upstream HTTP status warrants a retry (item 9). Server errors+/// (5xx), rate-limit (429), and request-timeout (408) are transient; all+/// other 4xx client errors are permanent (fail fast).+fn status_is_retryable(code: u16) -> bool {+ code >= 500 || code == 429 || code == 408+}+// ─────────────────────────────────────────────────────────────────────────────// EmbeddingProxy// ─────────────────────────────────────────────────────────────────────────────@@ -163,9 +194,23 @@ impl EmbeddingProxy {match self.send_once(texts, model).await {Ok(result) => return Ok(result),- Err(e) => {- debug!("embed attempt {attempt} failed: {e}");- last_err = e;+ Err(EmbedFailure {+ err,+ retryable: false,+ }) => {+ // Non-transient: a 4xx client error (bad model / bad key /+ // malformed payload) or a contract violation from the+ // upstream. Retrying just stalls ~max_retries × backoff+ // (≈2 min/doc) to arrive at the same failure. Fail fast.+ warn!("embed failed (non-transient, not retrying): {err}");+ return Err(err);+ }+ Err(EmbedFailure {+ err,+ retryable: true,+ }) => {+ debug!("embed attempt {attempt} failed (transient): {err}");+ last_err = err;}}}@@ -173,7 +218,11 @@ impl EmbeddingProxy {Err(last_err)}- async fn send_once(&self, texts: &[String], model: &str) -> Result<Vec<Vec<f32>>> {+ async fn send_once(+ &self,+ texts: &[String],+ model: &str,+ ) -> std::result::Result<Vec<Vec<f32>>, EmbedFailure> {let body = EmbedRequest {input: texts,model,@@ -188,35 +237,44 @@ impl EmbeddingProxy {req = req.header("Authorization", format!("Bearer {key}"));}- let resp = req- .json(&body)- .send()- .await- .map_err(|e| XerjError::embedding(format!("HTTP request: {e}")))?;+ // Transport-level failures (connection refused, DNS, timeout) are+ // transient: the proxy may be restarting or briefly unreachable.+ let resp = req.json(&body).send().await.map_err(|e| {+ EmbedFailure::transient(XerjError::embedding(format!("HTTP request: {e}")))+ })?;if !resp.status().is_success() {let status = resp.status();+ // Classify by status (item 9): 5xx are server-side and transient;+ // 429 (rate limit) and 408 (request timeout) are transient; every+ // other 4xx is a permanent client error — a wrong model, a bad API+ // key, or a malformed request — that no amount of retrying fixes.+ let retryable = status_is_retryable(status.as_u16());let body = resp.text().await.unwrap_or_default();- return Err(XerjError::embedding(format!(- "embedding API returned {status}: {body}"- )));+ let err = XerjError::embedding(format!("embedding API returned {status}: {body}"));+ return Err(if retryable {+ EmbedFailure::transient(err)+ } else {+ EmbedFailure::permanent(err)+ });}- let response: EmbedResponse = resp- .json()- .await- .map_err(|e| XerjError::embedding(format!("response parse: {e}")))?;+ // A 200 with an unparseable / short body is a contract violation, not+ // a transient blip — don't spin on it.+ let response: EmbedResponse = resp.json().await.map_err(|e| {+ EmbedFailure::permanent(XerjError::embedding(format!("response parse: {e}")))+ })?;// Sort by index to restore original orderlet mut data = response.data;data.sort_by_key(|d| d.index);if data.len() != texts.len() {- return Err(XerjError::embedding(format!(+ return Err(EmbedFailure::permanent(XerjError::embedding(format!("expected {} embeddings, got {}",texts.len(),data.len()- )));+ ))));}Ok(data.into_iter().map(|d| d.embedding).collect())@@ -265,4 +323,64 @@ mod tests {// Note: actual HTTP tests require a live embedding endpoint.// Integration tests should use a mock server (e.g., wiremock).++ #[test]+ fn status_classification_item9() {+ // Permanent client errors → fail fast.+ assert!(!status_is_retryable(400)); // bad request+ assert!(!status_is_retryable(401)); // bad api key+ assert!(!status_is_retryable(403)); // forbidden+ assert!(!status_is_retryable(404)); // wrong model / endpoint+ assert!(!status_is_retryable(422)); // unprocessable payload+ // Transient → retry.+ assert!(status_is_retryable(408)); // request timeout+ assert!(status_is_retryable(429)); // rate limited+ assert!(status_is_retryable(500)); // internal+ assert!(status_is_retryable(502)); // bad gateway+ assert!(status_is_retryable(503)); // unavailable (proxy restarting)+ }++ #[tokio::test]+ async fn permanent_4xx_fails_fast_no_retry() {+ // A proxy that always 404s a bad model: with max_retries=5 and a+ // 200ms base backoff, the OLD code would sleep 200+400+800+1600+3200ms+ // ≈ 6.2s before giving up. Classified as permanent, we return on the+ // first attempt — well under the first backoff.+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ let handle = std::thread::spawn(move || {+ use std::io::{Read, Write};+ // Serve a few 404s so a retrying client would keep hitting us.+ for _ in 0..8 {+ if let Ok((mut s, _)) = listener.accept() {+ let mut buf = [0u8; 1024];+ let _ = s.read(&mut buf);+ let body = "{\"error\":\"model not found\"}";+ let resp = format!(+ "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",+ body.len(),+ body+ );+ let _ = s.write_all(resp.as_bytes());+ }+ }+ });++ let mut cfg =+ EmbeddingProxyConfig::new(format!("http://{addr}/v1/embeddings"), "bad-model");+ cfg.max_retries = 5;+ let proxy = EmbeddingProxy::new(cfg).unwrap();++ let start = std::time::Instant::now();+ let res = proxy.embed(String::from("hello")).await;+ let elapsed = start.elapsed();++ assert!(res.is_err(), "a 404 must surface as an error");+ // Fail-fast: no backoff sleeps at all (first backoff is 200ms).+ assert!(+ elapsed < Duration::from_millis(200),+ "permanent 4xx should not retry; took {elapsed:?}"+ );+ drop(handle); // listener already closed by loop exit+ }}diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rsindex 59b4558dd..44ef1ba27 100644--- a/engine/crates/xerj-api/src/error.rs+++ b/engine/crates/xerj-api/src/error.rs@@ -93,6 +93,17 @@ impl IntoResponse for ApiError {fn into_response(self) -> Response {let mut status_code = self.inner.http_status();+ // ES parity: an explicit write/metadata block is 403+ // (`FORBIDDEN/…/cluster_block_exception`), but the disk *flood-stage*+ // block (`read_only_allow_delete`) is rejected with HTTP 429+ // (`TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark`).+ // Both keep the `cluster_block_exception` type; only the status differs.+ if let XerjError::IndexBlocked { block_type, .. } = &self.inner {+ if block_type.contains("read_only_allow_delete") {+ status_code = 429;+ }+ }+let mut error_type = xerj_error_type(&self.inner);// ES wraps mapping/query errors with a specific phrasing in `reason`;// our Display impl adds a "invalid mapping:" / "invalid query:" prefix@@ -198,6 +209,7 @@ impl IntoResponse for ApiError {/// | `TlsError` | `connect_exception` | 500 |/// | `EmbeddingError` | `circuit_breaking_exception` | 500 |/// | `ResourceExhausted` | `es_rejected_execution_exception` | 429 |+/// | `CircuitBreaking` | `circuit_breaking_exception` | 429 |/// | `VersionConflict` | `version_conflict_engine_exception` | 409 |/// | `ResultWindowTooLarge` | `illegal_argument_exception` | 400 |/// | `IndexBlocked` | `cluster_block_exception` | 403 |@@ -227,6 +239,8 @@ fn xerj_error_type(e: &XerjError) -> String {XerjError::EmbeddingError { .. } => "circuit_breaking_exception",// ── Resource limits ───────────────────────────────────────────────XerjError::ResourceExhausted { .. } => "es_rejected_execution_exception",+ // ── Circuit breaker (parent memory breaker) ───────────────────────+ XerjError::CircuitBreaking { .. } => "circuit_breaking_exception",// ── Optimistic concurrency ────────────────────────────────────────XerjError::VersionConflict { .. } => "version_conflict_engine_exception",// ── Result window ─────────────────────────────────────────────────diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex 107ce0490..afcf78cf3 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -706,7 +706,7 @@ impl Default for EmbeddingConfig {/// Resource limits.///-/// **7 settings.**+/// **10 settings.**#[derive(Debug, Clone, Serialize, Deserialize)]#[serde(default, deny_unknown_fields)]pub struct LimitsConfig {@@ -741,6 +741,34 @@ pub struct LimitsConfig {/// accumulator boundary, not after sort, so memory never grows/// past the cap.pub max_buckets: usize,+ /// Process-wide ceiling on the total bytes buffered across ALL indices'+ /// memtables (default: `0` = auto-derive to 25% of system RAM, floored+ /// at 2048 MiB). This is the parent circuit breaker for the ingest path:+ /// per-index back-pressure only bounds one index at `~3×flush_size_mb`,+ /// so `N` indices could buffer `N × 1.5 GiB` with no global ceiling —+ /// the structural cause of the 112 GiB OOM. When the summed memtable+ /// footprint crosses this budget, writes are rejected with HTTP 429+ /// `circuit_breaking_exception` instead of growing until the kernel+ /// OOM-kills the process.+ pub max_total_memtable_mb: u64,+ /// RSS admission watermark, as a percentage of the effective process+ /// memory limit (default: `95`). The effective limit is the cgroup+ /// memory limit when one is set (e.g. under `systemd-run -p MemoryMax=`+ /// or a container), else total system RAM. When resident set size+ /// crosses this fraction of the limit, writes are rejected with HTTP 429+ /// `circuit_breaking_exception` so a 429 beats the OOM-killer. Mirrors+ /// Elasticsearch's real-memory parent circuit breaker (default 95%).+ /// Set to `0` to disable the RSS admission check.+ pub memory_watermark_percent: u8,+ /// Disk flood-stage watermark, as a percentage of the data-dir+ /// filesystem that is *used* (default: `95`). A background `statvfs`+ /// poll auto-engages a write block when used space crosses this+ /// threshold, mirroring Elasticsearch's+ /// `cluster.routing.allocation.disk.watermark.flood_stage`. This+ /// prevents the engine from writing until `ENOSPC`, which poisons the+ /// WAL. The block clears automatically once usage drops back below the+ /// threshold. Set to `0` to disable the disk watermark.+ pub disk_flood_stage_percent: u8,}impl Default for LimitsConfig {@@ -753,6 +781,9 @@ impl Default for LimitsConfig {max_result_window: 10_000,max_mget_docs: 10_000,max_buckets: 65_536,+ max_total_memtable_mb: 0, // 0 = auto-derive (25% RAM, floor 2 GiB)+ memory_watermark_percent: 95,+ disk_flood_stage_percent: 95,}}}diff --git a/engine/crates/xerj-common/src/error.rs b/engine/crates/xerj-common/src/error.rsindex 2e7fe4363..b12554055 100644--- a/engine/crates/xerj-common/src/error.rs+++ b/engine/crates/xerj-common/src/error.rs@@ -80,6 +80,17 @@ pub enum XerjError {#[error("resource exhausted: {reason}")]ResourceExhausted { reason: String },+ // ── Circuit breaker ────────────────────────────────────────────────────+ /// A memory circuit breaker tripped: admitting the request would push the+ /// process past its memtable / RSS budget. Surfaced as HTTP 429+ /// `circuit_breaking_exception`, mirroring Elasticsearch's parent breaker.+ /// Distinct from [`ResourceExhausted`] (thread-pool queue rejection →+ /// `es_rejected_execution_exception`). Display is the bare reason (no+ /// type prefix): the ES mapper already emits the `circuit_breaking_exception`+ /// `type`, so the reason should read like ES's (e.g. "[parent] …").+ #[error("{reason}")]+ CircuitBreaking { reason: String },+// ── Optimistic concurrency ─────────────────────────────────────────────/// Optimistic concurrency control check failed (if_seq_no / if_primary_term).#[error(@@ -197,6 +208,12 @@ impl XerjError {}}+ pub fn circuit_breaking(reason: impl Into<String>) -> Self {+ Self::CircuitBreaking {+ reason: reason.into(),+ }+ }+pub fn internal(reason: impl Into<String>) -> Self {Self::Internal {reason: reason.into(),@@ -239,7 +256,7 @@ impl XerjError {| Self::ResultWindowTooLarge { .. } => 400,Self::IndexBlocked { .. } => 403,Self::AuthError { .. } => 401,- Self::ResourceExhausted { .. } => 429,+ Self::ResourceExhausted { .. } | Self::CircuitBreaking { .. } => 429,Self::StorageError { .. }| Self::WalError { .. }| Self::SerializationError { .. }@@ -254,6 +271,7 @@ impl XerjError {matches!(self,Self::ResourceExhausted { .. }+ | Self::CircuitBreaking { .. }| Self::StorageError { .. }| Self::WalError { .. }| Self::EmbeddingError { .. }diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 76623052d..644fcc3a2 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -1012,7 +1012,12 @@ pub async fn process_bulk_with_opts(// Whole-batch failure (e.g. ResourceExhausted → 429):// mark every item in the batch with the same status.let status = match &e {- EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,+ EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. })+ | EngineError::Common(xerj_common::XerjError::CircuitBreaking { .. }) => 429,+ EngineError::Common(xerj_common::XerjError::IndexBlocked {+ block_type,+ ..+ }) if block_type.contains("read_only_allow_delete") => 429,_ => 500,};for item_idx in item_indices {@@ -1105,7 +1110,14 @@ pub async fn process_bulk_with_opts(let status = match &e {EngineError::Common(xerj_common::XerjError::ResourceExhausted {..- }) => 429,+ })+ | EngineError::Common(xerj_common::XerjError::CircuitBreaking { .. }) => {+ 429+ }+ EngineError::Common(xerj_common::XerjError::IndexBlocked {+ block_type,+ ..+ }) if block_type.contains("read_only_allow_delete") => 429,_ => 500,};items[item_idx] = Some(BulkItemResult {@@ -1550,7 +1562,12 @@ pub async fn process_bulk_with_opts(let status = match &e {EngineError::Common(xerj_common::XerjError::VersionConflict { .. }) => 409,EngineError::Common(xerj_common::XerjError::DocumentNotFound { .. }) => 404,- EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,+ EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. })+ | EngineError::Common(xerj_common::XerjError::CircuitBreaking { .. }) => 429,+ EngineError::Common(xerj_common::XerjError::IndexBlocked {+ block_type,+ ..+ }) if block_type.contains("read_only_allow_delete") => 429,_ => 500,};items[item_idx] = Some(BulkItemResult {diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 8da25acbb..20c47e74c 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -304,6 +304,15 @@ impl Engine {// can read it with no plumbing through every agg signature.crate::aggs::set_max_buckets(config.limits.max_buckets);+ // Initialise the process-wide resource governor (parent circuit+ // breaker) from config: the memtable byte budget, RSS watermark,+ // per-query memory guard, global search pool, and disk flood-stage+ // block. Idempotent — a second in-process Engine (tests) re-uses the+ // first governor. The background sampler that drives its trip flags+ // is started by `spawn_resource_sampler`, called once the engine is+ // Arc-wrapped.+ crate::governor::init(&config);+let engine = Self {config: Arc::new(config),indices: Arc::new(DashMap::new()),@@ -1111,6 +1120,89 @@ impl Engine {&self.config}+ /// Sum of every open index's live memtable footprint, in bytes. This is+ /// the quantity the process-wide memtable budget (item 1) is checked+ /// against — per-index back-pressure only ever sees one index's slice.+ pub fn total_memtable_bytes(&self) -> u64 {+ self.indices+ .iter()+ .map(|e| e.value().memtable_bytes() as u64)+ .sum()+ }++ /// Spawn the background resource sampler (item 1/3): every+ /// [`crate::governor::SAMPLE_INTERVAL_MS`] it refreshes the governor's+ /// summed-memtable / RSS / disk-usage atomics, which drive the hot-path+ /// admission checks. Uses a `Weak` self-pointer so it exits when the+ /// engine is dropped. Idempotent-safe to call once after Arc-wrapping.+ ///+ /// LOAD-BEARING: runs on a dedicated OS thread, NOT a `tokio::spawn` task.+ /// The whole point of the parent breaker is to fire under a runaway ingest+ /// load — exactly the condition that saturates the tokio worker pool. A+ /// tokio-task sampler starves under that load: its `interval.tick()` wakes+ /// but never gets scheduled, so the trip flags never flip and the process+ /// OOMs anyway (observed: a MemoryMax=2G ingest reached the 2G cap and was+ /// OOM-killed while `memory_tripped` stayed false). A plain thread with+ /// `std::thread::sleep` is immune to runtime starvation. All work here is+ /// sync (DashMap scan + two syscalls), so no runtime is needed.+ pub fn spawn_resource_sampler(self: &Arc<Self>) {+ // Nothing to drive the flags if the governor was never installed.+ if crate::governor::global().is_none() {+ return;+ }+ let period = std::time::Duration::from_millis(crate::governor::SAMPLE_INTERVAL_MS);++ // ── Thread A: memory + disk ──────────────────────────────────────+ // Touches NO engine lock — just `/proc/self/statm` + `statvfs`. Kept+ // on its own thread so a memtable-sum stall (below) can NEVER delay+ // the memory breaker: the observed OOM was exactly RSS climbing past+ // the watermark while the sampler was blocked summing memtables under+ // a turbo batch's shard write-locks. Uses a `Weak` liveness check to+ // exit when the engine is dropped.+ let weak_a = Arc::downgrade(self);+ let data_dir = self.data_dir.to_string_lossy().to_string();+ let _ = std::thread::Builder::new()+ .name("xerj-mem-sampler".to_string())+ .spawn(move || {+ tracing::info!(+ period_ms = crate::governor::SAMPLE_INTERVAL_MS,+ "memory/disk sampler thread started (parent circuit breaker)"+ );+ loop {+ std::thread::sleep(period);+ if weak_a.strong_count() == 0 {+ return; // engine dropped+ }+ let governor = match crate::governor::global() {+ Some(g) => g,+ None => return,+ };+ let rss = crate::governor::current_rss_bytes();+ let disk_pct = crate::governor::disk_used_pct(&data_dir);+ governor.refresh_memory_disk(rss, disk_pct);+ }+ });++ // ── Thread B: summed memtable budget ─────────────────────────────+ // Reads a lock on every memtable shard, so it may briefly block under+ // a turbo batch — that is fine here, isolated from the memory breaker.+ let weak_b = Arc::downgrade(self);+ let _ = std::thread::Builder::new()+ .name("xerj-memtable-sampler".to_string())+ .spawn(move || loop {+ std::thread::sleep(period);+ let engine = match weak_b.upgrade() {+ Some(e) => e,+ None => return, // engine dropped+ };+ let governor = match crate::governor::global() {+ Some(g) => g,+ None => return,+ };+ governor.refresh_memtable(engine.total_memtable_bytes());+ });+ }+// ── Transform pipeline methods ────────────────────────────────────────────/// Compile and register a typed transform pipeline from a JSON config.diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsnew file mode 100644index 000000000..129fd672e--- /dev/null+++ b/engine/crates/xerj-engine/src/governor.rs@@ -0,0 +1,717 @@+//! Process-wide resource governor — the parent circuit breaker for the+//! ingest and search paths.+//!+//! Per-index back-pressure (the memtable soft/hard block in+//! [`crate::index`]) only ever bounds ONE index at `~3×flush_size_mb`. With+//! `N` indices there was no global ceiling: `N × ~1.5 GiB` of memtable could+//! accumulate until the kernel OOM-killed the process — the structural cause+//! of the 112 GiB incident. This module is the missing ceiling. It adds:+//!+//! * **item 1** — a process-wide memtable byte budget, plus an RSS+//! admission watermark measured against the cgroup/system memory limit.+//! Crossing either rejects writes with HTTP 429+//! `circuit_breaking_exception` (so a 429 beats the OOM-killer), and+//! wires the hitherto-inert `max_query_memory_mb` into a per-query+//! allocation guard.+//! * **item 2** — a global search-concurrency pool sized from+//! `max_concurrent_searches` (previously a hardcoded per-index+//! `Semaphore::new(64)`, i.e. no global cap).+//! * **item 3** — a disk flood-stage write block driven by a background+//! `statvfs` poll, mirroring Elasticsearch's+//! `disk.watermark.flood_stage`.+//!+//! A single process-wide [`OnceLock`] holds the governor. [`init`]+//! initialises it from config; [`Engine::spawn_resource_sampler`] refreshes+//! the RSS / memtable / disk atomics every ~250 ms, so the hot-path+//! admission checks are relaxed atomic loads — never syscalls.++use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{Arc, OnceLock};++use tokio::sync::{OwnedSemaphorePermit, Semaphore};+use xerj_common::config::Config;+use xerj_common::XerjError;++/// The process-wide governor singleton.+static GOVERNOR: OnceLock<Arc<ResourceGovernor>> = OnceLock::new();++/// Interval at which the background sampler refreshes the RSS / memtable /+/// disk atomics. Kept short so the RSS admission has a tight overshoot margin+/// under a runaway ingest — the sampler runs on a dedicated OS thread (see+/// `Engine::spawn_resource_sampler`), so this cadence is honoured even when the+/// tokio pool is saturated.+pub const SAMPLE_INTERVAL_MS: u64 = 100;++/// Process-wide resource governor. See the module docs.+pub struct ResourceGovernor {+ // ── item 1: process-wide memtable budget ────────────────────────────+ /// Ceiling on summed memtable bytes across ALL indices. `0` = disabled.+ memtable_budget_bytes: u64,+ /// Last sampled sum of every index's memtable footprint.+ memtable_used_bytes: AtomicU64,+ /// Latched: last sample crossed the memtable budget.+ memtable_tripped: AtomicBool,++ // ── item 1: RSS admission watermark ─────────────────────────────────+ /// Effective process memory limit (cgroup limit, else system RAM).+ memory_limit_bytes: u64,+ /// RSS admission threshold = `memory_limit_bytes * pct/100`. `0` = off.+ memory_watermark_bytes: u64,+ /// Last sampled resident set size of this process.+ rss_bytes: AtomicU64,+ /// Latched: last sample crossed the RSS watermark.+ memory_tripped: AtomicBool,++ // ── item 1: per-query memory guard (`max_query_memory_mb`) ──────────+ /// Maximum bytes a single query may be estimated to allocate. `0` = off.+ max_query_memory_bytes: u64,++ // ── item 2: global search pool (`max_concurrent_searches`) ──────────+ /// Global search-concurrency permits.+ search_pool: Arc<Semaphore>,+ /// Configured permit count (for observability / stats).+ max_concurrent_searches: usize,+ /// Live count of in-flight search permits (proof of the cap). `Arc` so a+ /// [`SearchPermit`] guard can decrement it safely on drop.+ search_inflight: Arc<AtomicU64>,+ /// High-water mark of concurrent searches observed (proof of the cap).+ search_inflight_peak: Arc<AtomicU64>,++ // ── item 3: disk flood-stage write block ────────────────────────────+ /// Used-percentage watermark that engages the write block. `0` = off.+ disk_flood_pct: u8,+ /// Latched: the data-dir filesystem is at/over the flood-stage watermark.+ disk_blocked: AtomicBool,+ /// Last sampled used-percentage of the data-dir filesystem.+ disk_used_pct: AtomicU64,+}++impl ResourceGovernor {+ // ── Admission checks (hot path — relaxed atomic loads only) ─────────++ /// Ingest admission. Returns a 429 `circuit_breaking_exception` when the+ /// process is at/over the memtable budget or the RSS watermark. This is+ /// the parent breaker that turns the OOM into a survivable 429.+ pub fn check_ingest_admission(&self) -> Result<(), XerjError> {+ if self.memtable_tripped.load(Ordering::Relaxed) {+ let used = self.memtable_used_bytes.load(Ordering::Relaxed);+ return Err(XerjError::circuit_breaking(format!(+ "[parent] memtable byte budget exceeded: used={}MB, limit={}MB across all \+ indices; writes rejected to prevent an out-of-memory kill (raise \+ limits.max_total_memtable_mb or slow ingest)",+ used / (1024 * 1024),+ self.memtable_budget_bytes / (1024 * 1024),+ )));+ }+ if self.memory_tripped.load(Ordering::Relaxed) {+ let rss = self.rss_bytes.load(Ordering::Relaxed);+ return Err(XerjError::circuit_breaking(format!(+ "[parent] real memory circuit breaker tripped: rss={}MB >= watermark={}MB \+ ({}% of limit={}MB); writes rejected to prevent an out-of-memory kill",+ rss / (1024 * 1024),+ self.memory_watermark_bytes / (1024 * 1024),+ pct_of(self.memory_watermark_bytes, self.memory_limit_bytes),+ self.memory_limit_bytes / (1024 * 1024),+ )));+ }+ Ok(())+ }++ /// Disk flood-stage admission. Returns an ES-shaped+ /// `read_only_allow_delete` cluster block (HTTP 429) when the data-dir+ /// filesystem is over the flood-stage watermark. `index` names the+ /// blocked index for the ES `root_cause`.+ pub fn check_disk_block(&self, index: &str) -> Result<(), XerjError> {+ if self.disk_blocked.load(Ordering::Relaxed) {+ // The `read_only_allow_delete` substring drives the 429 status in+ // the ES error mapper (flood-stage rejections are 429, unlike an+ // explicit 403 write block). Mirrors ES's flood-stage message.+ return Err(XerjError::index_blocked(+ index,+ format!(+ "read_only_allow_delete (disk usage {}% exceeded flood-stage watermark [{}%])",+ self.disk_used_pct.load(Ordering::Relaxed),+ self.disk_flood_pct,+ ),+ ));+ }+ Ok(())+ }++ /// Per-query memory guard for `max_query_memory_mb` (item 1). Rejects a+ /// query whose *estimated* peak allocation (`bytes`) exceeds the budget,+ /// before the allocation is made, with a 429 `circuit_breaking_exception`.+ /// `label` names the allocation site (e.g. "hydrate", "terms-agg").+ pub fn check_query_alloc(&self, bytes: u64, label: &str) -> Result<(), XerjError> {+ if self.max_query_memory_bytes != 0 && bytes > self.max_query_memory_bytes {+ return Err(XerjError::circuit_breaking(format!(+ "[request] query allocation ({}) would exceed limits.max_query_memory_mb={}MB \+ at [{label}]; reduce size/aggregation cardinality",+ human_bytes(bytes),+ self.max_query_memory_bytes / (1024 * 1024),+ )));+ }+ Ok(())+ }++ /// Whether the per-query memory guard is active (non-zero budget).+ pub fn query_memory_enabled(&self) -> bool {+ self.max_query_memory_bytes != 0+ }++ // ── item 2: global search pool ──────────────────────────────────────++ /// Acquire one global search permit. Bounds process-wide search+ /// concurrency to `max_concurrent_searches`; the returned guard releases+ /// the permit (and decrements the in-flight gauge) on drop. Excess+ /// searches queue on the semaphore, exactly like ES's search thread+ /// pool bounds active workers.+ pub async fn acquire_search(&self) -> Result<SearchPermit, XerjError> {+ let permit = Arc::clone(&self.search_pool)+ .acquire_owned()+ .await+ .map_err(|_| XerjError::internal("global search pool closed — shutting down"))?;+ let now = self.search_inflight.fetch_add(1, Ordering::Relaxed) + 1;+ self.search_inflight_peak.fetch_max(now, Ordering::Relaxed);+ Ok(SearchPermit {+ _permit: permit,+ inflight: Arc::clone(&self.search_inflight),+ })+ }++ /// Current in-flight search count (for stats / proof).+ pub fn search_inflight(&self) -> u64 {+ self.search_inflight.load(Ordering::Relaxed)+ }++ /// Peak concurrent search count observed since boot (for proof).+ pub fn search_inflight_peak(&self) -> u64 {+ self.search_inflight_peak.load(Ordering::Relaxed)+ }++ /// Configured global search-concurrency cap.+ pub fn max_concurrent_searches(&self) -> usize {+ self.max_concurrent_searches+ }++ // ── Sampler surface (called by the background task) ─────────────────++ /// Refresh every sampled atomic in one call (memtable, RSS, disk).+ /// Convenience wrapper used by tests; the live sampler calls+ /// [`Self::refresh_memory_disk`] and [`Self::refresh_memtable`]+ /// SEPARATELY so a contended memtable read can never delay the memory+ /// admission update (see `Engine::spawn_resource_sampler`).+ pub fn refresh(&self, memtable_used: u64, rss: u64, disk_used_pct: u64) {+ self.refresh_memory_disk(rss, disk_used_pct);+ self.refresh_memtable(memtable_used);+ }++ /// Update the RSS + disk atomics and their latched trip flags. Depends on+ /// NOTHING but two syscalls — never blocks on an engine lock — so the+ /// parent memory breaker stays responsive even while a turbo batch holds+ /// every memtable shard's write lock.+ pub fn refresh_memory_disk(&self, rss: u64, disk_used_pct: u64) {+ // ── RSS watermark ──+ self.rss_bytes.store(rss, Ordering::Relaxed);+ let mem_next = self.memory_watermark_bytes != 0 && rss >= self.memory_watermark_bytes;+ if mem_next != self.memory_tripped.swap(mem_next, Ordering::Relaxed) {+ if mem_next {+ tracing::warn!(+ rss_mb = rss / (1024 * 1024),+ watermark_mb = self.memory_watermark_bytes / (1024 * 1024),+ "RSS crossed the memory watermark — engaging the parent memory circuit breaker (writes → 429)"+ );+ } else {+ tracing::info!(+ rss_mb = rss / (1024 * 1024),+ "RSS back below the memory watermark — releasing the memory circuit breaker"+ );+ }+ }++ // ── disk flood stage (1% release hysteresis to avoid flapping) ──+ if self.disk_flood_pct != 0 {+ self.disk_used_pct.store(disk_used_pct, Ordering::Relaxed);+ let cur = self.disk_blocked.load(Ordering::Relaxed);+ let release_pct = (self.disk_flood_pct as u64).saturating_sub(1);+ let next = if cur {+ disk_used_pct >= release_pct+ } else {+ disk_used_pct >= self.disk_flood_pct as u64+ };+ if next != cur {+ if next {+ tracing::warn!(+ used_pct = disk_used_pct,+ flood_pct = self.disk_flood_pct,+ "disk flood-stage watermark crossed — engaging read_only_allow_delete write block"+ );+ } else {+ tracing::info!(+ used_pct = disk_used_pct,+ "disk usage back below flood-stage watermark — releasing write block"+ );+ }+ }+ self.disk_blocked.store(next, Ordering::Relaxed);+ }+ }++ /// Update the summed-memtable atomic + its trip flag. Called AFTER+ /// [`Self::refresh_memory_disk`] in the sampler, because computing the sum+ /// reads a lock on every memtable shard and a turbo batch can hold those+ /// write-locked for the whole batch — blocking here must never stall the+ /// memory/disk update above.+ pub fn refresh_memtable(&self, memtable_used: u64) {+ self.memtable_used_bytes+ .store(memtable_used, Ordering::Relaxed);+ let next = self.memtable_budget_bytes != 0 && memtable_used >= self.memtable_budget_bytes;+ if next != self.memtable_tripped.swap(next, Ordering::Relaxed) {+ if next {+ tracing::warn!(+ used_mb = memtable_used / (1024 * 1024),+ budget_mb = self.memtable_budget_bytes / (1024 * 1024),+ "summed memtable crossed the process budget — engaging the parent circuit breaker (writes → 429)"+ );+ } else {+ tracing::info!(+ used_mb = memtable_used / (1024 * 1024),+ "summed memtable back below the process budget — releasing the parent circuit breaker"+ );+ }+ }+ }++ // ── Observability ───────────────────────────────────────────────────++ /// Snapshot of the current governor state for `_nodes/stats`-style+ /// surfaces.+ pub fn snapshot(&self) -> GovernorSnapshot {+ GovernorSnapshot {+ memtable_used_bytes: self.memtable_used_bytes.load(Ordering::Relaxed),+ memtable_budget_bytes: self.memtable_budget_bytes,+ memtable_tripped: self.memtable_tripped.load(Ordering::Relaxed),+ rss_bytes: self.rss_bytes.load(Ordering::Relaxed),+ memory_limit_bytes: self.memory_limit_bytes,+ memory_watermark_bytes: self.memory_watermark_bytes,+ memory_tripped: self.memory_tripped.load(Ordering::Relaxed),+ disk_used_pct: self.disk_used_pct.load(Ordering::Relaxed),+ disk_flood_pct: self.disk_flood_pct,+ disk_blocked: self.disk_blocked.load(Ordering::Relaxed),+ max_concurrent_searches: self.max_concurrent_searches,+ search_inflight: self.search_inflight.load(Ordering::Relaxed),+ search_inflight_peak: self.search_inflight_peak.load(Ordering::Relaxed),+ }+ }+}++/// RAII guard for a held global search permit. Decrements the in-flight+/// gauge on drop; the underlying semaphore permit is released with it.+pub struct SearchPermit {+ _permit: OwnedSemaphorePermit,+ inflight: Arc<AtomicU64>,+}++impl Drop for SearchPermit {+ fn drop(&mut self) {+ self.inflight.fetch_sub(1, Ordering::Relaxed);+ }+}++/// A cheap, `Copy`-able snapshot of governor state.+#[derive(Debug, Clone, Copy)]+pub struct GovernorSnapshot {+ pub memtable_used_bytes: u64,+ pub memtable_budget_bytes: u64,+ pub memtable_tripped: bool,+ pub rss_bytes: u64,+ pub memory_limit_bytes: u64,+ pub memory_watermark_bytes: u64,+ pub memory_tripped: bool,+ pub disk_used_pct: u64,+ pub disk_flood_pct: u8,+ pub disk_blocked: bool,+ pub max_concurrent_searches: usize,+ pub search_inflight: u64,+ pub search_inflight_peak: u64,+}++// ─────────────────────────────────────────────────────────────────────────+// Initialisation / access+// ─────────────────────────────────────────────────────────────────────────++/// Initialise the process-wide governor from config. Idempotent: the first+/// call wins (subsequent calls return the already-installed governor, so a+/// second `Engine` in-process — e.g. a test — does not re-key the budgets).+pub fn init(config: &Config) -> Arc<ResourceGovernor> {+ Arc::clone(GOVERNOR.get_or_init(|| Arc::new(build(config))))+}++/// The installed governor, if [`init`] has run. Engine-only unit tests that+/// construct an [`crate::index::Index`] directly (never calling+/// `Engine::new`) get `None` and skip all admission checks — behaviour is+/// unchanged for them.+pub fn global() -> Option<Arc<ResourceGovernor>> {+ GOVERNOR.get().map(Arc::clone)+}++fn build(config: &Config) -> ResourceGovernor {+ let limits = &config.limits;++ // ── memtable budget: 0 = auto-derive to 25% RAM, floored at 2 GiB ──+ let memtable_budget_bytes = if limits.max_total_memtable_mb != 0 {+ limits.max_total_memtable_mb.saturating_mul(1024 * 1024)+ } else {+ let sys = system_total_bytes();+ (sys / 4).max(2 * 1024 * 1024 * 1024)+ };++ // ── RSS watermark against the effective memory limit ──+ let memory_limit_bytes = effective_memory_limit_bytes();+ let memory_watermark_bytes = if limits.memory_watermark_percent == 0 {+ 0+ } else {+ let pct = limits.memory_watermark_percent.min(100) as u64;+ ((memory_limit_bytes as u128 * pct as u128) / 100) as u64+ };++ let max_query_memory_bytes = limits.max_query_memory_mb.saturating_mul(1024 * 1024);++ let max_concurrent_searches = (limits.max_concurrent_searches.max(1)) as usize;++ tracing::info!(+ memtable_budget_mb = memtable_budget_bytes / (1024 * 1024),+ memory_limit_mb = memory_limit_bytes / (1024 * 1024),+ memory_watermark_mb = memory_watermark_bytes / (1024 * 1024),+ memory_watermark_pct = limits.memory_watermark_percent,+ max_query_memory_mb = limits.max_query_memory_mb,+ max_concurrent_searches,+ disk_flood_pct = limits.disk_flood_stage_percent,+ "resource governor initialised (parent circuit breaker)"+ );++ ResourceGovernor {+ memtable_budget_bytes,+ memtable_used_bytes: AtomicU64::new(0),+ memtable_tripped: AtomicBool::new(false),+ memory_limit_bytes,+ memory_watermark_bytes,+ rss_bytes: AtomicU64::new(0),+ memory_tripped: AtomicBool::new(false),+ max_query_memory_bytes,+ search_pool: Arc::new(Semaphore::new(max_concurrent_searches)),+ max_concurrent_searches,+ search_inflight: Arc::new(AtomicU64::new(0)),+ search_inflight_peak: Arc::new(AtomicU64::new(0)),+ disk_flood_pct: limits.disk_flood_stage_percent.min(100),+ disk_blocked: AtomicBool::new(false),+ disk_used_pct: AtomicU64::new(0),+ }+}++// ─────────────────────────────────────────────────────────────────────────+// System probes (Linux; best-effort with safe fallbacks elsewhere)+// ─────────────────────────────────────────────────────────────────────────++/// Current resident set size of this process, in bytes. Reads+/// `/proc/self/statm` (field 2 = resident pages). Returns 0 if unreadable+/// (the RSS watermark then never trips — safe).+pub fn current_rss_bytes() -> u64 {+ #[cfg(target_os = "linux")]+ {+ if let Ok(s) = std::fs::read_to_string("/proc/self/statm") {+ if let Some(res) = s.split_whitespace().nth(1) {+ if let Ok(pages) = res.parse::<u64>() {+ return pages.saturating_mul(page_size_bytes());+ }+ }+ }+ 0+ }+ #[cfg(not(target_os = "linux"))]+ {+ 0+ }+}++#[cfg(target_os = "linux")]+fn page_size_bytes() -> u64 {+ // SAFETY: sysconf is a pure read of a system constant.+ let p = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };+ if p > 0 {+ p as u64+ } else {+ 4096+ }+}++/// Total system RAM in bytes (from `/proc/meminfo`). Falls back to 8 GiB if+/// unreadable so budgets stay sane on exotic platforms.+fn system_total_bytes() -> u64 {+ #[cfg(target_os = "linux")]+ {+ if let Ok(s) = std::fs::read_to_string("/proc/meminfo") {+ for line in s.lines() {+ if let Some(rest) = line.strip_prefix("MemTotal:") {+ if let Some(kb) = rest.split_whitespace().next() {+ if let Ok(kb) = kb.parse::<u64>() {+ return kb.saturating_mul(1024);+ }+ }+ }+ }+ }+ }+ 8 * 1024 * 1024 * 1024+}++/// Effective process memory limit: the cgroup memory limit when one is set+/// (container / `systemd-run -p MemoryMax=`), otherwise total system RAM.+/// Takes the min of the two so a generous cgroup value never exceeds RAM.+pub fn effective_memory_limit_bytes() -> u64 {+ let sys = system_total_bytes().max(1);+ match cgroup_memory_limit_bytes() {+ Some(c) if c > 0 && c < sys => c,+ _ => sys,+ }+}++/// Read the cgroup memory limit for THIS process. Handles cgroup v2+/// (`memory.max`, walking up to a parent when a leaf reads `max`) and falls+/// back to cgroup v1 (`memory.limit_in_bytes`). Returns `None` when no+/// finite limit applies.+#[cfg(target_os = "linux")]+fn cgroup_memory_limit_bytes() -> Option<u64> {+ // Sentinels used by the kernel/systemd to mean "unlimited".+ const UNLIMITED_V1: u64 = 9_223_372_036_854_771_712; // PAGE_COUNTER_MAX * page+ let finite = |v: u64| -> Option<u64> {+ if v == 0 || v >= UNLIMITED_V1 {+ None+ } else {+ Some(v)+ }+ };++ // cgroup v2: /proc/self/cgroup has a single "0::<path>" line.+ if let Ok(cg) = std::fs::read_to_string("/proc/self/cgroup") {+ for line in cg.lines() {+ if let Some(path) = line.strip_prefix("0::") {+ // Walk from the leaf cgroup up to the root, honouring the+ // nearest finite memory.max (systemd sets MemoryMax on the+ // scope, which may be a parent of the leaf).+ let mut rel = path.trim().to_string();+ loop {+ let full = format!("/sys/fs/cgroup{rel}/memory.max");+ if let Ok(s) = std::fs::read_to_string(&full) {+ let s = s.trim();+ if s != "max" {+ if let Ok(v) = s.parse::<u64>() {+ if let Some(v) = finite(v) {+ return Some(v);+ }+ }+ }+ }+ if rel.is_empty() || rel == "/" {+ break;+ }+ match rel.rfind('/') {+ Some(0) => rel = String::new(), // step to root next iter+ Some(i) => rel.truncate(i),+ None => break,+ }+ }+ }+ }+ }++ // cgroup v1 fallback.+ if let Ok(s) = std::fs::read_to_string("/sys/fs/cgroup/memory/memory.limit_in_bytes") {+ if let Ok(v) = s.trim().parse::<u64>() {+ return finite(v);+ }+ }+ None+}++#[cfg(not(target_os = "linux"))]+fn cgroup_memory_limit_bytes() -> Option<u64> {+ None+}++/// `(total_bytes, avail_bytes)` for the filesystem backing `path`, via+/// `statvfs(2)`. Returns `None` on syscall failure.+#[cfg(unix)]+#[allow(clippy::unnecessary_cast)]+pub fn disk_stats(path: &str) -> Option<(u64, u64)> {+ let c = std::ffi::CString::new(path).ok()?;+ // SAFETY: statvfs fully initialises the struct; we only read scalars.+ let mut st: libc::statvfs = unsafe { std::mem::zeroed() };+ if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 {+ return None;+ }+ let bsize = if st.f_frsize > 0 {+ st.f_frsize as u64+ } else {+ st.f_bsize as u64+ };+ let total = (st.f_blocks as u64).saturating_mul(bsize);+ let avail = (st.f_bavail as u64).saturating_mul(bsize);+ if total == 0 {+ return None;+ }+ Some((total, avail))+}++#[cfg(not(unix))]+pub fn disk_stats(_path: &str) -> Option<(u64, u64)> {+ None+}++/// Used-percentage of the filesystem backing `path` (0..=100). Uses+/// `total - avail` over `total`, matching ES's disk-watermark accounting.+/// Returns 0 when `statvfs` is unavailable (the disk block never trips).+pub fn disk_used_pct(path: &str) -> u64 {+ match disk_stats(path) {+ Some((total, avail)) if total > 0 => {+ let used = total.saturating_sub(avail);+ ((used as u128 * 100) / total as u128) as u64+ }+ _ => 0,+ }+}++// ─────────────────────────────────────────────────────────────────────────+// Small formatting helpers+// ─────────────────────────────────────────────────────────────────────────++fn pct_of(part: u64, whole: u64) -> u64 {+ if whole == 0 {+ 0+ } else {+ ((part as u128 * 100) / whole as u128) as u64+ }+}++fn human_bytes(b: u64) -> String {+ const KB: u64 = 1024;+ const MB: u64 = 1024 * KB;+ const GB: u64 = 1024 * MB;+ if b >= GB {+ format!("{:.1}gb", b as f64 / GB as f64)+ } else if b >= MB {+ format!("{:.1}mb", b as f64 / MB as f64)+ } else if b >= KB {+ format!("{:.1}kb", b as f64 / KB as f64)+ } else {+ format!("{b}b")+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn effective_limit_is_positive() {+ assert!(effective_memory_limit_bytes() > 0);+ }++ #[test]+ fn disk_used_pct_bounded() {+ let p = disk_used_pct("/");+ assert!(p <= 100);+ }++ #[test]+ fn human_bytes_scales() {+ assert_eq!(human_bytes(512), "512b");+ assert_eq!(human_bytes(2 * 1024 * 1024), "2.0mb");+ }++ #[test]+ fn build_from_default_config_trips_nothing() {+ let cfg = Config::default();+ let g = build(&cfg);+ // Fresh: nothing sampled yet, so no trips and ingest is admitted.+ assert!(g.check_ingest_admission().is_ok());+ assert!(g.check_disk_block("i").is_ok());+ assert_eq!(g.max_concurrent_searches(), 64);+ }++ #[test]+ fn memtable_budget_trips_on_refresh() {+ let mut cfg = Config::default();+ cfg.limits.max_total_memtable_mb = 100; // 100 MiB ceiling+ cfg.limits.memory_watermark_percent = 0; // isolate the memtable path+ let g = build(&cfg);+ assert!(g.check_ingest_admission().is_ok());+ g.refresh(200 * 1024 * 1024, 0, 0); // 200 MiB used > 100 MiB budget+ let err = g.check_ingest_admission().unwrap_err();+ // 429 + the CircuitBreaking variant (the ES mapper stamps the+ // `circuit_breaking_exception` type; the Display is the bare reason).+ assert_eq!(err.http_status(), 429);+ assert!(matches!(err, XerjError::CircuitBreaking { .. }));+ assert!(format!("{err}").contains("memtable byte budget exceeded"));+ // Recovery once usage drops back under the budget.+ g.refresh(10 * 1024 * 1024, 0, 0);+ assert!(g.check_ingest_admission().is_ok());+ }++ #[test]+ fn disk_flood_blocks_with_hysteresis() {+ let mut cfg = Config::default();+ cfg.limits.disk_flood_stage_percent = 95;+ let g = build(&cfg);+ assert!(g.check_disk_block("i").is_ok());+ g.refresh(0, 0, 96); // over flood stage+ assert!(g.check_disk_block("i").is_err());+ g.refresh(0, 0, 95); // still within release hysteresis (>= 94)+ assert!(g.check_disk_block("i").is_err());+ g.refresh(0, 0, 90); // clearly recovered+ assert!(g.check_disk_block("i").is_ok());+ }++ #[test]+ fn query_alloc_guard_fires() {+ let mut cfg = Config::default();+ cfg.limits.max_query_memory_mb = 1; // 1 MiB+ let g = build(&cfg);+ assert!(g.check_query_alloc(512 * 1024, "t").is_ok());+ let err = g.check_query_alloc(4 * 1024 * 1024, "t").unwrap_err();+ assert_eq!(err.http_status(), 429);+ }++ #[tokio::test]+ async fn global_search_pool_caps_concurrency() {+ // Item 2: a pool of 2 admits exactly 2 concurrent searches; the 3rd+ // acquire blocks until one is released.+ let mut cfg = Config::default();+ cfg.limits.max_concurrent_searches = 2;+ let g = Arc::new(build(&cfg));+ assert_eq!(g.max_concurrent_searches(), 2);++ let p1 = g.acquire_search().await.unwrap();+ let p2 = g.acquire_search().await.unwrap();+ assert_eq!(g.search_inflight(), 2);++ // A 3rd acquire must NOT complete while 2 are held.+ let g3 = Arc::clone(&g);+ let third = tokio::spawn(async move { g3.acquire_search().await.map(|_p| ()) });+ assert!(+ tokio::time::timeout(std::time::Duration::from_millis(150), third)+ .await+ .is_err(),+ "3rd search must block while the pool of 2 is full"+ );++ // Release one → a subsequent acquire succeeds and peak stays at 2.+ drop(p1);+ let _p3 = tokio::time::timeout(std::time::Duration::from_millis(500), g.acquire_search())+ .await+ .expect("acquire must proceed once a permit frees")+ .unwrap();+ assert_eq!(g.search_inflight_peak(), 2, "concurrency never exceeded 2");+ drop(p2);+ }+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0254d8f50..b2fb86a97 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -432,6 +432,17 @@ pub struct Index {hnsw_id_rev: Arc<RwLock<HashMap<u64, String>>>,/// Monotonic counter for HNSW node IDs.hnsw_next_id: Arc<AtomicU64>,+ /// Number of distinct live docs that CARRY the pinned vector field (item+ /// 8). This is the coverage-gate denominator: the ANN query path may only+ /// serve when `hnsw_id_map.len() == vector_doc_count`, i.e. every+ /// vector-bearing doc made it into the graph. The old gate compared+ /// against the index-global `doc_count`, so a SINGLE doc lacking the+ /// vector field (a mixed log/vector corpus) left `id_map.len() < doc_count`+ /// forever and silently pinned the whole index on O(N) brute force.+ /// Incremented once per doc-id newly added to the graph, decremented on+ /// delete; never above the true count (over-count on updates biases the+ /// gate to brute, which is safe).+ vector_doc_count: Arc<AtomicU64>,/// FIELD IDENTITY: the single field this index's HNSW graph indexes.////// Pinned when the graph is first created (preferring the schema's first@@ -878,6 +889,7 @@ impl Index {hnsw_id_map: Arc::new(RwLock::new(HashMap::new())),hnsw_id_rev: Arc::new(RwLock::new(HashMap::new())),hnsw_next_id: Arc::new(AtomicU64::new(1)),+ vector_doc_count: Arc::new(AtomicU64::new(0)),hnsw_field: Arc::new(std::sync::RwLock::new(None)),hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(false)),hnsw_rebuilding: Arc::new(std::sync::atomic::AtomicBool::new(false)),@@ -1071,6 +1083,11 @@ impl Index {}None => (None, HashMap::new(), HashMap::new(), 1, None, false),};+ // Item 8: seed the coverage-gate denominator from the persisted graph.+ // At flush time the graph covered every vector-bearing doc, so the+ // reloaded `id_map.len()` IS the vector-doc count. If the snapshot is+ // stale, `hnsw_stale` already forces brute regardless of this value.+ let vector_doc_count_init = id_map_init.len() as u64;if initial_hnsw {info!(name = name.as_str(),@@ -1109,6 +1126,7 @@ impl Index {hnsw_id_map: Arc::new(RwLock::new(id_map_init)),hnsw_id_rev: Arc::new(RwLock::new(id_rev_init)),hnsw_next_id: Arc::new(AtomicU64::new(next_id_init)),+ vector_doc_count: Arc::new(AtomicU64::new(vector_doc_count_init)),hnsw_field: Arc::new(std::sync::RwLock::new(hnsw_field_init)),hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(hnsw_stale_init)),hnsw_rebuilding: Arc::new(std::sync::atomic::AtomicBool::new(false)),@@ -1223,6 +1241,9 @@ impl Index {"write",)));}+ // Process-wide admission: disk flood-stage block (item 3) + parent+ // memory circuit breaker (item 1). Fires before any per-index work.+ self.governor_write_gate()?;// Invalidate the response cache: every doc write bumps the// dataset version, which is part of the cache key, so old// entries become invisible to lookups. No clear() needed —@@ -1432,6 +1453,9 @@ impl Index {"write",)));}+ // Process-wide admission: disk flood-stage block (item 3) + parent+ // memory circuit breaker (item 1). Fires before any per-index work.+ self.governor_write_gate()?;// ── Back-pressure ─────────────────────────────────────────────────// Reject the batch with 429 ResourceExhausted if the memtable is@@ -1746,6 +1770,9 @@ impl Index {"write",)));}+ // Process-wide admission: disk flood-stage block (item 3) + parent+ // memory circuit breaker (item 1). Fires before any per-index work.+ self.governor_write_gate()?;let hard_block = self.flush_byte_threshold.saturating_mul(3);let soft_block = self.flush_byte_threshold.saturating_mul(2);@@ -2259,6 +2286,9 @@ impl Index {"write",)));}+ // Process-wide admission: disk flood-stage block (item 3) + parent+ // memory circuit breaker (item 1). Fires before any per-index work.+ self.governor_write_gate()?;let batch_len = docs.len();let index_start = std::time::Instant::now();@@ -2353,6 +2383,9 @@ impl Index {"write",)));}+ // Process-wide admission: disk flood-stage block (item 3) + parent+ // memory circuit breaker (item 1). Fires before any per-index work.+ self.governor_write_gate()?;// Fail with 409 if a live document already exists with this id.let already_exists = {@@ -3855,6 +3888,15 @@ impl Index {// that node tombstoned before the fresh vector goes in (fresh// node_id; the old node is never unmarked).let old_node: Option<u64> = self.hnsw_id_map.read().await.get(doc_id).copied();+ if old_node.is_none() {+ // Item 8: a new vector-bearing doc-id. Counted on ATTEMPT (here),+ // not on success below: if the insert then fails (dim mismatch /+ // graph error), `id_map.len()` stays behind `vector_doc_count`,+ // which holds the coverage gate on brute force — the safe+ // direction (a doc that isn't in the graph must not be answered+ // from it). An update (old_node.is_some()) is already counted.+ self.vector_doc_count.fetch_add(1, Ordering::Relaxed);+ }let node_id = self.hnsw_next_id.fetch_add(1, Ordering::Relaxed);{let mut hnsw_guard = self.hnsw.write().await;@@ -4140,6 +4182,12 @@ impl Index {if let Some(h) = self.hnsw.read().await.as_ref() {h.mark_deleted(node_id);}+ // Item 8: pruned doc leaves the coverage-gate denominator.+ self.vector_doc_count+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| {+ Some(v.saturating_sub(1))+ })+ .ok();pruned += 1;}@@ -4194,12 +4242,26 @@ impl Index {None => (false, 0, 0),}};+ let graphed = self.hnsw_id_map.read().await.len() as u64;+ // Item 8: coverage is graphed-docs / vector-bearing-docs. `covered`+ // means the ANN path is eligible (every vector doc is in the graph);+ // it stays `true` even when many docs lack the vector field (they are+ // not vector docs and belong in neither the graph nor a brute scan).+ let vector_docs = self.vector_doc_count.load(Ordering::Relaxed);+ let doc_count = self.doc_count.load(Ordering::Relaxed);+ let covered = present && graphed == vector_docs && !self.hnsw_stale.load(Ordering::Acquire);serde_json::json!({"present": present,"field": self.hnsw_field.read().unwrap().clone(),"nodes": nodes,"tombstones": tombstones,- "doc_coverage": self.hnsw_id_map.read().await.len() as u64,+ "doc_coverage": graphed,+ // How many live docs carry the pinned vector field. The ANN gate+ // compares this against `doc_coverage`, NOT the index-wide+ // `total_docs` — a mixed corpus can be fully covered.+ "vector_doc_count": vector_docs,+ "total_docs": doc_count,+ "covered": covered,"stale": self.hnsw_stale.load(Ordering::Acquire),"rebuilding": self.hnsw_rebuilding@@ -4226,12 +4288,14 @@ impl Index {/// flush-time stale (see `hnsw_field` / `hnsw_stale`);/// * the field does not opt into SQ8 (`scalar8` keeps its dedicated/// code-store scoring path);- /// * `doc_count >= HNSW_MIN_DOCS` — small indexes (every ES-YAML- /// vector test is ≤ a few hundred docs) keep today's byte-identical- /// exact path;- /// * coverage: `hnsw_id_map.len() == doc_count`, i.e. every live doc- /// has a graph entry. Racy reads degrade to brute, never to wrong- /// results.+ /// * `vector_doc_count >= HNSW_MIN_DOCS` — small vector sets (every+ /// ES-YAML vector test is ≤ a few hundred docs) keep today's+ /// byte-identical exact path;+ /// * coverage: `hnsw_id_map.len() == vector_doc_count` (item 8), i.e.+ /// every doc that CARRIES the pinned vector field has a graph entry.+ /// Docs lacking the field are not vector docs and belong in neither+ /// the graph nor a brute scan, so a mixed corpus can be fully covered.+ /// Racy reads degrade to brute, never to wrong results.////// Results are EXACT-RESCORED: each of the ≤k candidate docs is fetched/// (metrics-free, so `indices.stats` get counters stay ES-parity) and@@ -4269,11 +4333,22 @@ impl Index {return None;}}- let doc_count = self.doc_count.load(Ordering::Relaxed);- if doc_count < HNSW_MIN_DOCS {+ // Item 8: both the min-size threshold and the coverage check are gated+ // on the count of docs that CARRY the pinned vector field+ // (`vector_doc_count`), NOT the index-global `doc_count`. The graph only+ // ever holds vector-bearing docs, so its size and coverage are a+ // function of that count. The old `id_map.len() == doc_count` test meant+ // a SINGLE doc lacking the vector field (a mixed log/vector corpus) left+ // `id_map.len() < doc_count` forever, silently pinning the WHOLE index+ // on O(N) brute force. A doc without the field is correctly absent from+ // both the graph and a brute scan, so ANN and brute return identical+ // results. A racy read (id_map.len() != vector_doc_count) degrades to+ // brute — never to wrong results.+ let vector_doc_count = self.vector_doc_count.load(Ordering::Relaxed);+ if vector_doc_count < HNSW_MIN_DOCS {return None;}- if self.hnsw_id_map.read().await.len() as u64 != doc_count {+ if self.hnsw_id_map.read().await.len() as u64 != vector_doc_count {return None;}@@ -4348,6 +4423,18 @@ impl Index {scored.push((doc_id, score, src));}+ // Item 8: this line is only reached when the coverage gate admitted+ // the ANN path (`hnsw_id_map.len() == vector_doc_count`). Logged so a+ // mixed-corpus query (docs without the vector field present) is+ // observably served by HNSW rather than silently pinned on brute.+ debug!(+ index = self.name.as_str(),+ field,+ vector_doc_count = self.vector_doc_count.load(Ordering::Relaxed),+ total_docs = self.doc_count.load(Ordering::Relaxed),+ candidates = scored.len(),+ "knn served via HNSW (coverage gate passed)"+ );Some(knn_result_from_scored(request, scored, k, started))}@@ -4780,9 +4867,11 @@ impl Index {.embed_batch(vec![text.to_string()]).await.map_err(|e| {- EngineError::Common(xerj_common::XerjError::invalid_query(format!(- "semantic embed failed: {e}"- )))+ // Item 9: preserve the embedder's 5xx class. An embedding+ // backend outage/misconfig is NOT a client query error;+ // wrapping it as invalid_query (400) hid proxy failures+ // behind a 400 and drove clients to retry a broken backend.+ EngineError::Common(embed_backend_error(e, "semantic query embed failed"))})?;match vectors.pop() {Some(v) if !v.is_empty() => v,@@ -4877,9 +4966,12 @@ impl Index {chunks.iter().map(|c| c.text.clone()).collect()};let vs = self.embedder.embed_batch(chunk_texts).await.map_err(|e| {- EngineError::Common(xerj_common::XerjError::invalid_query(format!(- "semantic_text embed failed for field [{field}]: {e}"- )))+ // Item 9: preserve the embedder's 5xx class on ingest too —+ // a proxy outage must not masquerade as a 400 bad document.+ EngineError::Common(embed_backend_error(+ e,+ &format!("semantic_text embed failed for field [{field}]"),+ ))})?;vs.into_iter().filter(|v| !v.is_empty()).collect()} else {@@ -5317,6 +5409,11 @@ impl Index {let mut id_rev = self.hnsw_id_rev.write().await;*id_map = loaded.id_map;*id_rev = loaded.id_rev;+ // Item 8: re-seed the coverage-gate denominator from the reloaded+ // graph (every graphed doc is a vector doc). Staleness is handled+ // separately by `hnsw_stale`.+ self.vector_doc_count+ .store(id_map.len() as u64, Ordering::Relaxed);drop(id_map);drop(id_rev);self.hnsw_next_id.store(loaded.next_id, Ordering::Relaxed);@@ -5633,6 +5730,9 @@ impl Index {"write",)));}+ // Process-wide admission: disk flood-stage block (item 3) + parent+ // memory circuit breaker (item 1). Fires before any per-index work.+ self.governor_write_gate()?;self.dataset_version.fetch_add(1, Ordering::Release);// Optimistic concurrency check (see index_document_with_version for@@ -5728,6 +5828,12 @@ impl Index {// gets a fresh node_id (the old one stays tombstoned).self.hnsw_id_map.write().await.remove(id);self.hnsw_id_rev.write().await.remove(&node_id);+ // Item 8: this doc leaves the coverage-gate denominator.+ self.vector_doc_count+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| {+ Some(v.saturating_sub(1))+ })+ .ok();}if existed {@@ -5984,10 +6090,22 @@ impl Index {}}- // Acquire a query permit before proceeding. This bounds the number of- // queries executing concurrently against this index to 64 (the semaphore- // capacity), preventing a single hot index from starving others in a- // multi-tenant deployment. The permit is automatically released when+ // Item 2 — global search pool. Acquire a PROCESS-WIDE search permit+ // sized from `limits.max_concurrent_searches` before proceeding. The+ // per-index semaphore below only ever bounded one index (and was a+ // hardcoded 64), so N hot indices could still run N×64 searches at+ // once with no global ceiling. The global permit caps total in-flight+ // searches across every index; excess searches queue on it, exactly+ // like ES's search thread pool bounds active workers. Held (via+ // `_global_permit`) for the whole call; released on drop.+ let _global_permit = match crate::governor::global() {+ Some(g) => Some(g.acquire_search().await?),+ None => None,+ };++ // Acquire a per-index query permit too. This bounds concurrency+ // against a single index, preventing one hot index from monopolising+ // the global pool. The permit is automatically released when// `_permit` is dropped at the end of this call.let _permit = self.max_concurrent_queries.acquire().await.map_err(|_| {EngineError::Common(xerj_common::XerjError::internal(@@ -5997,6 +6115,29 @@ impl Index {let search_start = std::time::Instant::now();+ // Item 1: per-query memory guard (`max_query_memory_mb`). Estimate the+ // hydration cost of the requested result window (from + size hits, each+ // materialised with its `_source` and scoring scratch) and reject+ // up-front with a 429 `circuit_breaking_exception` if it would exceed+ // the budget — before the Vec<Hit> is allocated. This is the+ // request-scoped breaker; the parent (memtable/RSS) breaker is the+ // process-wide one. A no-op when no governor is installed or the+ // budget is 0 (disabled). At the 512 MiB default a full 10 000-hit+ // window (~10 MiB) is nowhere near the cap, so normal search is+ // unaffected; the guard only bites pathological windows or a+ // deliberately low operator budget.+ if let Some(g) = crate::governor::global() {+ if g.query_memory_enabled() {+ // Conservative per-hit estimate: stored source + collector+ // scratch. Deep windows dominate query memory in practice.+ const EST_BYTES_PER_HIT: u64 = 1024;+ let window = (request.from as u64).saturating_add(request.size as u64);+ let est = window.saturating_mul(EST_BYTES_PER_HIT);+ g.check_query_alloc(est, "result-window hydration")+ .map_err(EngineError::Common)?;+ }+ }+// Determine the timeout: use the request-level timeout if set, otherwise// fall back to the default of 30 seconds.let timeout_ms = request.timeout_ms.unwrap_or(30_000);@@ -9823,6 +9964,27 @@ impl Index {// ── Index blocks ──────────────────────────────────────────────────────────+ /// Process-wide write admission (RC4 W3 items 1 & 3). Applies, in order:+ ///+ /// 1. the disk flood-stage block (item 3) — a 429 `cluster_block_exception`+ /// `read_only_allow_delete` when the data-dir filesystem is over the+ /// watermark, before the WAL can be driven to `ENOSPC`;+ /// 2. the parent memory circuit breaker (item 1) — a 429+ /// `circuit_breaking_exception` when the summed memtable footprint or+ /// the process RSS is over budget, so a 429 beats the OOM-killer.+ ///+ /// A no-op when no governor is installed (engine-only unit tests that+ /// build an `Index` directly), so their behaviour is unchanged.+ #[inline]+ fn governor_write_gate(&self) -> Result<()> {+ if let Some(g) = crate::governor::global() {+ g.check_disk_block(self.name.as_str())+ .map_err(EngineError::Common)?;+ g.check_ingest_admission().map_err(EngineError::Common)?;+ }+ Ok(())+ }+/// Returns true if the write block is set on this index.pub async fn is_write_blocked(&self) -> bool {let settings = self.settings.read().await;@@ -15734,6 +15896,17 @@ fn build_highlight_fragments(/// Any misconfiguration degrades to the lexical fallback (never a crash);/// a `semantic` query with no active backend against a plain vector field/// still returns a helpful 400.+/// Map an embedder failure to the correct ES class (item 9). An embedding+/// backend outage or misconfiguration is a server-side (5xx) failure, never a+/// 400 `invalid_query`: downgrading it hid proxy failures behind a client+/// error and drove clients to retry a broken backend. Produces an+/// `EmbeddingError` (500, `circuit_breaking_exception`) with `ctx` prepended.+/// Takes `impl Display` because the `Embedder` surfaces `anyhow::Error` (the+/// classification of transient-vs-permanent already happened inside the proxy).+fn embed_backend_error(e: impl std::fmt::Display, ctx: &str) -> xerj_common::XerjError {+ xerj_common::XerjError::embedding(format!("{ctx}: {e}"))+}+fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> xerj_ai::Embedder {let mode = cfg.mode.trim().to_ascii_lowercase();let want_proxy = mode == "proxy"diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 7026a600f..e9c1e65b4 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -20,6 +20,7 @@ pub mod aggs;pub mod audit;pub mod bulk;pub mod engine;+pub mod governor;pub mod index;pub mod memtable;pub mod painless;diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 744f8d4ab..98de7f67a 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -1251,6 +1251,13 @@ async fn async_main() -> Result<()> {std::time::Duration::from_secs(cfg.storage.flush_interval_secs),);+ // 12b. Resource governor sampler (RC4 W3 items 1 & 3): refreshes the+ // process-wide memtable / RSS / disk-usage atomics that drive the+ // parent circuit breaker and the disk flood-stage write block. This+ // is the structural guard against the 112 GiB OOM class — writes get+ // a 429 circuit_breaking_exception before the kernel OOM-kills us.+ state.engine.spawn_resource_sampler();+// 13. Start servers concurrentlylet rest_tls = tls_config.clone();let rest = tokio::spawn(async move {