Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(governor): parent circuit breaker, global search pool, disk flood-stage, ANN coverage, embed error class (RC4 W3 Stream A: items 1,2,3,8,9)

xerj-org11ee72fc595c
diff --git a/engine/crates/xerj-ai/src/embed.rs b/engine/crates/xerj-ai/src/embed.rs
index 7fc23bb66..55e340b66 100644
--- a/engine/crates/xerj-ai/src/embed.rs
+++ b/engine/crates/xerj-ai/src/embed.rs
@@ -90,6 +90,37 @@ struct EmbedDatum {
index: usize,
}
+/// A single embedding attempt's failure, tagged with whether a retry could
+/// plausibly succeed (item 9). Only `retryable` failures are re-attempted by
+/// [`EmbeddingProxy::send_with_retry`]; a permanent failure (4xx client error,
+/// contract violation) returns immediately instead of stalling ~2 min/doc.
+struct EmbedFailure {
+ err: XerjError,
+ retryable: bool,
+}
+
+impl EmbedFailure {
+ fn transient(err: XerjError) -> Self {
+ Self {
+ err,
+ retryable: true,
+ }
+ }
+ fn permanent(err: XerjError) -> Self {
+ Self {
+ err,
+ retryable: false,
+ }
+ }
+}
+
+/// Whether an upstream HTTP status warrants a retry (item 9). Server errors
+/// (5xx), rate-limit (429), and request-timeout (408) are transient; all
+/// other 4xx client errors are permanent (fail fast).
+fn status_is_retryable(code: u16) -> bool {
+ code >= 500 || code == 429 || code == 408
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// EmbeddingProxy
// ─────────────────────────────────────────────────────────────────────────────
@@ -163,9 +194,23 @@ impl EmbeddingProxy {
match self.send_once(texts, model).await {
Ok(result) => return Ok(result),
- Err(e) => {
- debug!("embed attempt {attempt} failed: {e}");
- last_err = e;
+ Err(EmbedFailure {
+ err,
+ retryable: false,
+ }) => {
+ // Non-transient: a 4xx client error (bad model / bad key /
+ // malformed payload) or a contract violation from the
+ // upstream. Retrying just stalls ~max_retries × backoff
+ // (≈2 min/doc) to arrive at the same failure. Fail fast.
+ warn!("embed failed (non-transient, not retrying): {err}");
+ return Err(err);
+ }
+ Err(EmbedFailure {
+ err,
+ retryable: true,
+ }) => {
+ debug!("embed attempt {attempt} failed (transient): {err}");
+ last_err = err;
}
}
}
@@ -173,7 +218,11 @@ impl EmbeddingProxy {
Err(last_err)
}
- async fn send_once(&self, texts: &[String], model: &str) -> Result<Vec<Vec<f32>>> {
+ async fn send_once(
+ &self,
+ texts: &[String],
+ model: &str,
+ ) -> std::result::Result<Vec<Vec<f32>>, EmbedFailure> {
let body = EmbedRequest {
input: texts,
model,
@@ -188,35 +237,44 @@ impl EmbeddingProxy {
req = req.header("Authorization", format!("Bearer {key}"));
}
- let resp = req
- .json(&body)
- .send()
- .await
- .map_err(|e| XerjError::embedding(format!("HTTP request: {e}")))?;
+ // Transport-level failures (connection refused, DNS, timeout) are
+ // transient: the proxy may be restarting or briefly unreachable.
+ let resp = req.json(&body).send().await.map_err(|e| {
+ EmbedFailure::transient(XerjError::embedding(format!("HTTP request: {e}")))
+ })?;
if !resp.status().is_success() {
let status = resp.status();
+ // Classify by status (item 9): 5xx are server-side and transient;
+ // 429 (rate limit) and 408 (request timeout) are transient; every
+ // other 4xx is a permanent client error — a wrong model, a bad API
+ // key, or a malformed request — that no amount of retrying fixes.
+ let retryable = status_is_retryable(status.as_u16());
let body = resp.text().await.unwrap_or_default();
- return Err(XerjError::embedding(format!(
- "embedding API returned {status}: {body}"
- )));
+ let err = XerjError::embedding(format!("embedding API returned {status}: {body}"));
+ return Err(if retryable {
+ EmbedFailure::transient(err)
+ } else {
+ EmbedFailure::permanent(err)
+ });
}
- let response: EmbedResponse = resp
- .json()
- .await
- .map_err(|e| XerjError::embedding(format!("response parse: {e}")))?;
+ // A 200 with an unparseable / short body is a contract violation, not
+ // a transient blip — don't spin on it.
+ let response: EmbedResponse = resp.json().await.map_err(|e| {
+ EmbedFailure::permanent(XerjError::embedding(format!("response parse: {e}")))
+ })?;
// Sort by index to restore original order
let mut data = response.data;
data.sort_by_key(|d| d.index);
if data.len() != texts.len() {
- return Err(XerjError::embedding(format!(
+ return Err(EmbedFailure::permanent(XerjError::embedding(format!(
"expected {} embeddings, got {}",
texts.len(),
data.len()
- )));
+ ))));
}
Ok(data.into_iter().map(|d| d.embedding).collect())
@@ -265,4 +323,64 @@ mod tests {
// Note: actual HTTP tests require a live embedding endpoint.
// Integration tests should use a mock server (e.g., wiremock).
+
+ #[test]
+ fn status_classification_item9() {
+ // Permanent client errors → fail fast.
+ assert!(!status_is_retryable(400)); // bad request
+ assert!(!status_is_retryable(401)); // bad api key
+ assert!(!status_is_retryable(403)); // forbidden
+ assert!(!status_is_retryable(404)); // wrong model / endpoint
+ assert!(!status_is_retryable(422)); // unprocessable payload
+ // Transient → retry.
+ assert!(status_is_retryable(408)); // request timeout
+ assert!(status_is_retryable(429)); // rate limited
+ assert!(status_is_retryable(500)); // internal
+ assert!(status_is_retryable(502)); // bad gateway
+ assert!(status_is_retryable(503)); // unavailable (proxy restarting)
+ }
+
+ #[tokio::test]
+ async fn permanent_4xx_fails_fast_no_retry() {
+ // A proxy that always 404s a bad model: with max_retries=5 and a
+ // 200ms base backoff, the OLD code would sleep 200+400+800+1600+3200ms
+ // ≈ 6.2s before giving up. Classified as permanent, we return on the
+ // first attempt — well under the first backoff.
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let addr = listener.local_addr().unwrap();
+ let handle = std::thread::spawn(move || {
+ use std::io::{Read, Write};
+ // Serve a few 404s so a retrying client would keep hitting us.
+ for _ in 0..8 {
+ if let Ok((mut s, _)) = listener.accept() {
+ let mut buf = [0u8; 1024];
+ let _ = s.read(&mut buf);
+ let body = "{\"error\":\"model not found\"}";
+ let resp = format!(
+ "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
+ body.len(),
+ body
+ );
+ let _ = s.write_all(resp.as_bytes());
+ }
+ }
+ });
+
+ let mut cfg =
+ EmbeddingProxyConfig::new(format!("http://{addr}/v1/embeddings"), "bad-model");
+ cfg.max_retries = 5;
+ let proxy = EmbeddingProxy::new(cfg).unwrap();
+
+ let start = std::time::Instant::now();
+ let res = proxy.embed(String::from("hello")).await;
+ let elapsed = start.elapsed();
+
+ assert!(res.is_err(), "a 404 must surface as an error");
+ // Fail-fast: no backoff sleeps at all (first backoff is 200ms).
+ assert!(
+ elapsed < Duration::from_millis(200),
+ "permanent 4xx should not retry; took {elapsed:?}"
+ );
+ drop(handle); // listener already closed by loop exit
+ }
}
diff --git a/engine/crates/xerj-api/src/error.rs b/engine/crates/xerj-api/src/error.rs
index 59b4558dd..44ef1ba27 100644
--- a/engine/crates/xerj-api/src/error.rs
+++ b/engine/crates/xerj-api/src/error.rs
@@ -93,6 +93,17 @@ impl IntoResponse for ApiError {
fn into_response(self) -> Response {
let mut status_code = self.inner.http_status();
+ // ES parity: an explicit write/metadata block is 403
+ // (`FORBIDDEN/…/cluster_block_exception`), but the disk *flood-stage*
+ // block (`read_only_allow_delete`) is rejected with HTTP 429
+ // (`TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark`).
+ // Both keep the `cluster_block_exception` type; only the status differs.
+ if let XerjError::IndexBlocked { block_type, .. } = &self.inner {
+ if block_type.contains("read_only_allow_delete") {
+ status_code = 429;
+ }
+ }
+
let mut error_type = xerj_error_type(&self.inner);
// ES wraps mapping/query errors with a specific phrasing in `reason`;
// our Display impl adds a "invalid mapping:" / "invalid query:" prefix
@@ -198,6 +209,7 @@ impl IntoResponse for ApiError {
/// | `TlsError` | `connect_exception` | 500 |
/// | `EmbeddingError` | `circuit_breaking_exception` | 500 |
/// | `ResourceExhausted` | `es_rejected_execution_exception` | 429 |
+/// | `CircuitBreaking` | `circuit_breaking_exception` | 429 |
/// | `VersionConflict` | `version_conflict_engine_exception` | 409 |
/// | `ResultWindowTooLarge` | `illegal_argument_exception` | 400 |
/// | `IndexBlocked` | `cluster_block_exception` | 403 |
@@ -227,6 +239,8 @@ fn xerj_error_type(e: &XerjError) -> String {
XerjError::EmbeddingError { .. } => "circuit_breaking_exception",
// ── Resource limits ───────────────────────────────────────────────
XerjError::ResourceExhausted { .. } => "es_rejected_execution_exception",
+ // ── Circuit breaker (parent memory breaker) ───────────────────────
+ XerjError::CircuitBreaking { .. } => "circuit_breaking_exception",
// ── Optimistic concurrency ────────────────────────────────────────
XerjError::VersionConflict { .. } => "version_conflict_engine_exception",
// ── Result window ─────────────────────────────────────────────────
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 107ce0490..afcf78cf3 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -706,7 +706,7 @@ impl Default for EmbeddingConfig {
/// Resource limits.
///
-/// **7 settings.**
+/// **10 settings.**
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct LimitsConfig {
@@ -741,6 +741,34 @@ pub struct LimitsConfig {
/// accumulator boundary, not after sort, so memory never grows
/// past the cap.
pub max_buckets: usize,
+ /// Process-wide ceiling on the total bytes buffered across ALL indices'
+ /// memtables (default: `0` = auto-derive to 25% of system RAM, floored
+ /// at 2048 MiB). This is the parent circuit breaker for the ingest path:
+ /// per-index back-pressure only bounds one index at `~3×flush_size_mb`,
+ /// so `N` indices could buffer `N × 1.5 GiB` with no global ceiling —
+ /// the structural cause of the 112 GiB OOM. When the summed memtable
+ /// footprint crosses this budget, writes are rejected with HTTP 429
+ /// `circuit_breaking_exception` instead of growing until the kernel
+ /// OOM-kills the process.
+ pub max_total_memtable_mb: u64,
+ /// RSS admission watermark, as a percentage of the effective process
+ /// memory limit (default: `95`). The effective limit is the cgroup
+ /// memory limit when one is set (e.g. under `systemd-run -p MemoryMax=`
+ /// or a container), else total system RAM. When resident set size
+ /// crosses this fraction of the limit, writes are rejected with HTTP 429
+ /// `circuit_breaking_exception` so a 429 beats the OOM-killer. Mirrors
+ /// Elasticsearch's real-memory parent circuit breaker (default 95%).
+ /// Set to `0` to disable the RSS admission check.
+ pub memory_watermark_percent: u8,
+ /// Disk flood-stage watermark, as a percentage of the data-dir
+ /// filesystem that is *used* (default: `95`). A background `statvfs`
+ /// poll auto-engages a write block when used space crosses this
+ /// threshold, mirroring Elasticsearch's
+ /// `cluster.routing.allocation.disk.watermark.flood_stage`. This
+ /// prevents the engine from writing until `ENOSPC`, which poisons the
+ /// WAL. The block clears automatically once usage drops back below the
+ /// threshold. Set to `0` to disable the disk watermark.
+ pub disk_flood_stage_percent: u8,
}
impl Default for LimitsConfig {
@@ -753,6 +781,9 @@ impl Default for LimitsConfig {
max_result_window: 10_000,
max_mget_docs: 10_000,
max_buckets: 65_536,
+ max_total_memtable_mb: 0, // 0 = auto-derive (25% RAM, floor 2 GiB)
+ memory_watermark_percent: 95,
+ disk_flood_stage_percent: 95,
}
}
}
diff --git a/engine/crates/xerj-common/src/error.rs b/engine/crates/xerj-common/src/error.rs
index 2e7fe4363..b12554055 100644
--- a/engine/crates/xerj-common/src/error.rs
+++ b/engine/crates/xerj-common/src/error.rs
@@ -80,6 +80,17 @@ pub enum XerjError {
#[error("resource exhausted: {reason}")]
ResourceExhausted { reason: String },
+ // ── Circuit breaker ────────────────────────────────────────────────────
+ /// A memory circuit breaker tripped: admitting the request would push the
+ /// process past its memtable / RSS budget. Surfaced as HTTP 429
+ /// `circuit_breaking_exception`, mirroring Elasticsearch's parent breaker.
+ /// Distinct from [`ResourceExhausted`] (thread-pool queue rejection →
+ /// `es_rejected_execution_exception`). Display is the bare reason (no
+ /// type prefix): the ES mapper already emits the `circuit_breaking_exception`
+ /// `type`, so the reason should read like ES's (e.g. "[parent] …").
+ #[error("{reason}")]
+ CircuitBreaking { reason: String },
+
// ── Optimistic concurrency ─────────────────────────────────────────────
/// Optimistic concurrency control check failed (if_seq_no / if_primary_term).
#[error(
@@ -197,6 +208,12 @@ impl XerjError {
}
}
+ pub fn circuit_breaking(reason: impl Into<String>) -> Self {
+ Self::CircuitBreaking {
+ reason: reason.into(),
+ }
+ }
+
pub fn internal(reason: impl Into<String>) -> Self {
Self::Internal {
reason: reason.into(),
@@ -239,7 +256,7 @@ impl XerjError {
| Self::ResultWindowTooLarge { .. } => 400,
Self::IndexBlocked { .. } => 403,
Self::AuthError { .. } => 401,
- Self::ResourceExhausted { .. } => 429,
+ Self::ResourceExhausted { .. } | Self::CircuitBreaking { .. } => 429,
Self::StorageError { .. }
| Self::WalError { .. }
| Self::SerializationError { .. }
@@ -254,6 +271,7 @@ impl XerjError {
matches!(
self,
Self::ResourceExhausted { .. }
+ | Self::CircuitBreaking { .. }
| Self::StorageError { .. }
| Self::WalError { .. }
| Self::EmbeddingError { .. }
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 76623052d..644fcc3a2 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -1012,7 +1012,12 @@ pub async fn process_bulk_with_opts(
// Whole-batch failure (e.g. ResourceExhausted → 429):
// mark every item in the batch with the same status.
let status = match &e {
- EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,
+ EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. })
+ | EngineError::Common(xerj_common::XerjError::CircuitBreaking { .. }) => 429,
+ EngineError::Common(xerj_common::XerjError::IndexBlocked {
+ block_type,
+ ..
+ }) if block_type.contains("read_only_allow_delete") => 429,
_ => 500,
};
for item_idx in item_indices {
@@ -1105,7 +1110,14 @@ pub async fn process_bulk_with_opts(
let status = match &e {
EngineError::Common(xerj_common::XerjError::ResourceExhausted {
..
- }) => 429,
+ })
+ | EngineError::Common(xerj_common::XerjError::CircuitBreaking { .. }) => {
+ 429
+ }
+ EngineError::Common(xerj_common::XerjError::IndexBlocked {
+ block_type,
+ ..
+ }) if block_type.contains("read_only_allow_delete") => 429,
_ => 500,
};
items[item_idx] = Some(BulkItemResult {
@@ -1550,7 +1562,12 @@ pub async fn process_bulk_with_opts(
let status = match &e {
EngineError::Common(xerj_common::XerjError::VersionConflict { .. }) => 409,
EngineError::Common(xerj_common::XerjError::DocumentNotFound { .. }) => 404,
- EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. }) => 429,
+ EngineError::Common(xerj_common::XerjError::ResourceExhausted { .. })
+ | EngineError::Common(xerj_common::XerjError::CircuitBreaking { .. }) => 429,
+ EngineError::Common(xerj_common::XerjError::IndexBlocked {
+ block_type,
+ ..
+ }) if block_type.contains("read_only_allow_delete") => 429,
_ => 500,
};
items[item_idx] = Some(BulkItemResult {
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 8da25acbb..20c47e74c 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -304,6 +304,15 @@ impl Engine {
// can read it with no plumbing through every agg signature.
crate::aggs::set_max_buckets(config.limits.max_buckets);
+ // Initialise the process-wide resource governor (parent circuit
+ // breaker) from config: the memtable byte budget, RSS watermark,
+ // per-query memory guard, global search pool, and disk flood-stage
+ // block. Idempotent — a second in-process Engine (tests) re-uses the
+ // first governor. The background sampler that drives its trip flags
+ // is started by `spawn_resource_sampler`, called once the engine is
+ // Arc-wrapped.
+ crate::governor::init(&config);
+
let engine = Self {
config: Arc::new(config),
indices: Arc::new(DashMap::new()),
@@ -1111,6 +1120,89 @@ impl Engine {
&self.config
}
+ /// Sum of every open index's live memtable footprint, in bytes. This is
+ /// the quantity the process-wide memtable budget (item 1) is checked
+ /// against — per-index back-pressure only ever sees one index's slice.
+ pub fn total_memtable_bytes(&self) -> u64 {
+ self.indices
+ .iter()
+ .map(|e| e.value().memtable_bytes() as u64)
+ .sum()
+ }
+
+ /// Spawn the background resource sampler (item 1/3): every
+ /// [`crate::governor::SAMPLE_INTERVAL_MS`] it refreshes the governor's
+ /// summed-memtable / RSS / disk-usage atomics, which drive the hot-path
+ /// admission checks. Uses a `Weak` self-pointer so it exits when the
+ /// engine is dropped. Idempotent-safe to call once after Arc-wrapping.
+ ///
+ /// LOAD-BEARING: runs on a dedicated OS thread, NOT a `tokio::spawn` task.
+ /// The whole point of the parent breaker is to fire under a runaway ingest
+ /// load — exactly the condition that saturates the tokio worker pool. A
+ /// tokio-task sampler starves under that load: its `interval.tick()` wakes
+ /// but never gets scheduled, so the trip flags never flip and the process
+ /// OOMs anyway (observed: a MemoryMax=2G ingest reached the 2G cap and was
+ /// OOM-killed while `memory_tripped` stayed false). A plain thread with
+ /// `std::thread::sleep` is immune to runtime starvation. All work here is
+ /// sync (DashMap scan + two syscalls), so no runtime is needed.
+ pub fn spawn_resource_sampler(self: &Arc<Self>) {
+ // Nothing to drive the flags if the governor was never installed.
+ if crate::governor::global().is_none() {
+ return;
+ }
+ let period = std::time::Duration::from_millis(crate::governor::SAMPLE_INTERVAL_MS);
+
+ // ── Thread A: memory + disk ──────────────────────────────────────
+ // Touches NO engine lock — just `/proc/self/statm` + `statvfs`. Kept
+ // on its own thread so a memtable-sum stall (below) can NEVER delay
+ // the memory breaker: the observed OOM was exactly RSS climbing past
+ // the watermark while the sampler was blocked summing memtables under
+ // a turbo batch's shard write-locks. Uses a `Weak` liveness check to
+ // exit when the engine is dropped.
+ let weak_a = Arc::downgrade(self);
+ let data_dir = self.data_dir.to_string_lossy().to_string();
+ let _ = std::thread::Builder::new()
+ .name("xerj-mem-sampler".to_string())
+ .spawn(move || {
+ tracing::info!(
+ period_ms = crate::governor::SAMPLE_INTERVAL_MS,
+ "memory/disk sampler thread started (parent circuit breaker)"
+ );
+ loop {
+ std::thread::sleep(period);
+ if weak_a.strong_count() == 0 {
+ return; // engine dropped
+ }
+ let governor = match crate::governor::global() {
+ Some(g) => g,
+ None => return,
+ };
+ let rss = crate::governor::current_rss_bytes();
+ let disk_pct = crate::governor::disk_used_pct(&data_dir);
+ governor.refresh_memory_disk(rss, disk_pct);
+ }
+ });
+
+ // ── Thread B: summed memtable budget ─────────────────────────────
+ // Reads a lock on every memtable shard, so it may briefly block under
+ // a turbo batch — that is fine here, isolated from the memory breaker.
+ let weak_b = Arc::downgrade(self);
+ let _ = std::thread::Builder::new()
+ .name("xerj-memtable-sampler".to_string())
+ .spawn(move || loop {
+ std::thread::sleep(period);
+ let engine = match weak_b.upgrade() {
+ Some(e) => e,
+ None => return, // engine dropped
+ };
+ let governor = match crate::governor::global() {
+ Some(g) => g,
+ None => return,
+ };
+ governor.refresh_memtable(engine.total_memtable_bytes());
+ });
+ }
+
// ── Transform pipeline methods ────────────────────────────────────────────
/// Compile and register a typed transform pipeline from a JSON config.
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
new file mode 100644
index 000000000..129fd672e
--- /dev/null
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -0,0 +1,717 @@
+//! Process-wide resource governor — the parent circuit breaker for the
+//! ingest and search paths.
+//!
+//! Per-index back-pressure (the memtable soft/hard block in
+//! [`crate::index`]) only ever bounds ONE index at `~3×flush_size_mb`. With
+//! `N` indices there was no global ceiling: `N × ~1.5 GiB` of memtable could
+//! accumulate until the kernel OOM-killed the process — the structural cause
+//! of the 112 GiB incident. This module is the missing ceiling. It adds:
+//!
+//! * **item 1** — a process-wide memtable byte budget, plus an RSS
+//! admission watermark measured against the cgroup/system memory limit.
+//! Crossing either rejects writes with HTTP 429
+//! `circuit_breaking_exception` (so a 429 beats the OOM-killer), and
+//! wires the hitherto-inert `max_query_memory_mb` into a per-query
+//! allocation guard.
+//! * **item 2** — a global search-concurrency pool sized from
+//! `max_concurrent_searches` (previously a hardcoded per-index
+//! `Semaphore::new(64)`, i.e. no global cap).
+//! * **item 3** — a disk flood-stage write block driven by a background
+//! `statvfs` poll, mirroring Elasticsearch's
+//! `disk.watermark.flood_stage`.
+//!
+//! A single process-wide [`OnceLock`] holds the governor. [`init`]
+//! initialises it from config; [`Engine::spawn_resource_sampler`] refreshes
+//! the RSS / memtable / disk atomics every ~250 ms, so the hot-path
+//! admission checks are relaxed atomic loads — never syscalls.
+
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, OnceLock};
+
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+use xerj_common::config::Config;
+use xerj_common::XerjError;
+
+/// The process-wide governor singleton.
+static GOVERNOR: OnceLock<Arc<ResourceGovernor>> = OnceLock::new();
+
+/// Interval at which the background sampler refreshes the RSS / memtable /
+/// disk atomics. Kept short so the RSS admission has a tight overshoot margin
+/// under a runaway ingest — the sampler runs on a dedicated OS thread (see
+/// `Engine::spawn_resource_sampler`), so this cadence is honoured even when the
+/// tokio pool is saturated.
+pub const SAMPLE_INTERVAL_MS: u64 = 100;
+
+/// Process-wide resource governor. See the module docs.
+pub struct ResourceGovernor {
+ // ── item 1: process-wide memtable budget ────────────────────────────
+ /// Ceiling on summed memtable bytes across ALL indices. `0` = disabled.
+ memtable_budget_bytes: u64,
+ /// Last sampled sum of every index's memtable footprint.
+ memtable_used_bytes: AtomicU64,
+ /// Latched: last sample crossed the memtable budget.
+ memtable_tripped: AtomicBool,
+
+ // ── item 1: RSS admission watermark ─────────────────────────────────
+ /// Effective process memory limit (cgroup limit, else system RAM).
+ memory_limit_bytes: u64,
+ /// RSS admission threshold = `memory_limit_bytes * pct/100`. `0` = off.
+ memory_watermark_bytes: u64,
+ /// Last sampled resident set size of this process.
+ rss_bytes: AtomicU64,
+ /// Latched: last sample crossed the RSS watermark.
+ memory_tripped: AtomicBool,
+
+ // ── item 1: per-query memory guard (`max_query_memory_mb`) ──────────
+ /// Maximum bytes a single query may be estimated to allocate. `0` = off.
+ max_query_memory_bytes: u64,
+
+ // ── item 2: global search pool (`max_concurrent_searches`) ──────────
+ /// Global search-concurrency permits.
+ search_pool: Arc<Semaphore>,
+ /// Configured permit count (for observability / stats).
+ max_concurrent_searches: usize,
+ /// Live count of in-flight search permits (proof of the cap). `Arc` so a
+ /// [`SearchPermit`] guard can decrement it safely on drop.
+ search_inflight: Arc<AtomicU64>,
+ /// High-water mark of concurrent searches observed (proof of the cap).
+ search_inflight_peak: Arc<AtomicU64>,
+
+ // ── item 3: disk flood-stage write block ────────────────────────────
+ /// Used-percentage watermark that engages the write block. `0` = off.
+ disk_flood_pct: u8,
+ /// Latched: the data-dir filesystem is at/over the flood-stage watermark.
+ disk_blocked: AtomicBool,
+ /// Last sampled used-percentage of the data-dir filesystem.
+ disk_used_pct: AtomicU64,
+}
+
+impl ResourceGovernor {
+ // ── Admission checks (hot path — relaxed atomic loads only) ─────────
+
+ /// Ingest admission. Returns a 429 `circuit_breaking_exception` when the
+ /// process is at/over the memtable budget or the RSS watermark. This is
+ /// the parent breaker that turns the OOM into a survivable 429.
+ pub fn check_ingest_admission(&self) -> Result<(), XerjError> {
+ if self.memtable_tripped.load(Ordering::Relaxed) {
+ let used = self.memtable_used_bytes.load(Ordering::Relaxed);
+ return Err(XerjError::circuit_breaking(format!(
+ "[parent] memtable byte budget exceeded: used={}MB, limit={}MB across all \
+ indices; writes rejected to prevent an out-of-memory kill (raise \
+ limits.max_total_memtable_mb or slow ingest)",
+ used / (1024 * 1024),
+ self.memtable_budget_bytes / (1024 * 1024),
+ )));
+ }
+ if self.memory_tripped.load(Ordering::Relaxed) {
+ let rss = self.rss_bytes.load(Ordering::Relaxed);
+ return Err(XerjError::circuit_breaking(format!(
+ "[parent] real memory circuit breaker tripped: rss={}MB >= watermark={}MB \
+ ({}% of limit={}MB); writes rejected to prevent an out-of-memory kill",
+ rss / (1024 * 1024),
+ self.memory_watermark_bytes / (1024 * 1024),
+ pct_of(self.memory_watermark_bytes, self.memory_limit_bytes),
+ self.memory_limit_bytes / (1024 * 1024),
+ )));
+ }
+ Ok(())
+ }
+
+ /// Disk flood-stage admission. Returns an ES-shaped
+ /// `read_only_allow_delete` cluster block (HTTP 429) when the data-dir
+ /// filesystem is over the flood-stage watermark. `index` names the
+ /// blocked index for the ES `root_cause`.
+ pub fn check_disk_block(&self, index: &str) -> Result<(), XerjError> {
+ if self.disk_blocked.load(Ordering::Relaxed) {
+ // The `read_only_allow_delete` substring drives the 429 status in
+ // the ES error mapper (flood-stage rejections are 429, unlike an
+ // explicit 403 write block). Mirrors ES's flood-stage message.
+ return Err(XerjError::index_blocked(
+ index,
+ format!(
+ "read_only_allow_delete (disk usage {}% exceeded flood-stage watermark [{}%])",
+ self.disk_used_pct.load(Ordering::Relaxed),
+ self.disk_flood_pct,
+ ),
+ ));
+ }
+ Ok(())
+ }
+
+ /// Per-query memory guard for `max_query_memory_mb` (item 1). Rejects a
+ /// query whose *estimated* peak allocation (`bytes`) exceeds the budget,
+ /// before the allocation is made, with a 429 `circuit_breaking_exception`.
+ /// `label` names the allocation site (e.g. "hydrate", "terms-agg").
+ pub fn check_query_alloc(&self, bytes: u64, label: &str) -> Result<(), XerjError> {
+ if self.max_query_memory_bytes != 0 && bytes > self.max_query_memory_bytes {
+ return Err(XerjError::circuit_breaking(format!(
+ "[request] query allocation ({}) would exceed limits.max_query_memory_mb={}MB \
+ at [{label}]; reduce size/aggregation cardinality",
+ human_bytes(bytes),
+ self.max_query_memory_bytes / (1024 * 1024),
+ )));
+ }
+ Ok(())
+ }
+
+ /// Whether the per-query memory guard is active (non-zero budget).
+ pub fn query_memory_enabled(&self) -> bool {
+ self.max_query_memory_bytes != 0
+ }
+
+ // ── item 2: global search pool ──────────────────────────────────────
+
+ /// Acquire one global search permit. Bounds process-wide search
+ /// concurrency to `max_concurrent_searches`; the returned guard releases
+ /// the permit (and decrements the in-flight gauge) on drop. Excess
+ /// searches queue on the semaphore, exactly like ES's search thread
+ /// pool bounds active workers.
+ pub async fn acquire_search(&self) -> Result<SearchPermit, XerjError> {
+ let permit = Arc::clone(&self.search_pool)
+ .acquire_owned()
+ .await
+ .map_err(|_| XerjError::internal("global search pool closed — shutting down"))?;
+ let now = self.search_inflight.fetch_add(1, Ordering::Relaxed) + 1;
+ self.search_inflight_peak.fetch_max(now, Ordering::Relaxed);
+ Ok(SearchPermit {
+ _permit: permit,
+ inflight: Arc::clone(&self.search_inflight),
+ })
+ }
+
+ /// Current in-flight search count (for stats / proof).
+ pub fn search_inflight(&self) -> u64 {
+ self.search_inflight.load(Ordering::Relaxed)
+ }
+
+ /// Peak concurrent search count observed since boot (for proof).
+ pub fn search_inflight_peak(&self) -> u64 {
+ self.search_inflight_peak.load(Ordering::Relaxed)
+ }
+
+ /// Configured global search-concurrency cap.
+ pub fn max_concurrent_searches(&self) -> usize {
+ self.max_concurrent_searches
+ }
+
+ // ── Sampler surface (called by the background task) ─────────────────
+
+ /// Refresh every sampled atomic in one call (memtable, RSS, disk).
+ /// Convenience wrapper used by tests; the live sampler calls
+ /// [`Self::refresh_memory_disk`] and [`Self::refresh_memtable`]
+ /// SEPARATELY so a contended memtable read can never delay the memory
+ /// admission update (see `Engine::spawn_resource_sampler`).
+ pub fn refresh(&self, memtable_used: u64, rss: u64, disk_used_pct: u64) {
+ self.refresh_memory_disk(rss, disk_used_pct);
+ self.refresh_memtable(memtable_used);
+ }
+
+ /// Update the RSS + disk atomics and their latched trip flags. Depends on
+ /// NOTHING but two syscalls — never blocks on an engine lock — so the
+ /// parent memory breaker stays responsive even while a turbo batch holds
+ /// every memtable shard's write lock.
+ pub fn refresh_memory_disk(&self, rss: u64, disk_used_pct: u64) {
+ // ── RSS watermark ──
+ self.rss_bytes.store(rss, Ordering::Relaxed);
+ let mem_next = self.memory_watermark_bytes != 0 && rss >= self.memory_watermark_bytes;
+ if mem_next != self.memory_tripped.swap(mem_next, Ordering::Relaxed) {
+ if mem_next {
+ tracing::warn!(
+ rss_mb = rss / (1024 * 1024),
+ watermark_mb = self.memory_watermark_bytes / (1024 * 1024),
+ "RSS crossed the memory watermark — engaging the parent memory circuit breaker (writes → 429)"
+ );
+ } else {
+ tracing::info!(
+ rss_mb = rss / (1024 * 1024),
+ "RSS back below the memory watermark — releasing the memory circuit breaker"
+ );
+ }
+ }
+
+ // ── disk flood stage (1% release hysteresis to avoid flapping) ──
+ if self.disk_flood_pct != 0 {
+ self.disk_used_pct.store(disk_used_pct, Ordering::Relaxed);
+ let cur = self.disk_blocked.load(Ordering::Relaxed);
+ let release_pct = (self.disk_flood_pct as u64).saturating_sub(1);
+ let next = if cur {
+ disk_used_pct >= release_pct
+ } else {
+ disk_used_pct >= self.disk_flood_pct as u64
+ };
+ if next != cur {
+ if next {
+ tracing::warn!(
+ used_pct = disk_used_pct,
+ flood_pct = self.disk_flood_pct,
+ "disk flood-stage watermark crossed — engaging read_only_allow_delete write block"
+ );
+ } else {
+ tracing::info!(
+ used_pct = disk_used_pct,
+ "disk usage back below flood-stage watermark — releasing write block"
+ );
+ }
+ }
+ self.disk_blocked.store(next, Ordering::Relaxed);
+ }
+ }
+
+ /// Update the summed-memtable atomic + its trip flag. Called AFTER
+ /// [`Self::refresh_memory_disk`] in the sampler, because computing the sum
+ /// reads a lock on every memtable shard and a turbo batch can hold those
+ /// write-locked for the whole batch — blocking here must never stall the
+ /// memory/disk update above.
+ pub fn refresh_memtable(&self, memtable_used: u64) {
+ self.memtable_used_bytes
+ .store(memtable_used, Ordering::Relaxed);
+ let next = self.memtable_budget_bytes != 0 && memtable_used >= self.memtable_budget_bytes;
+ if next != self.memtable_tripped.swap(next, Ordering::Relaxed) {
+ if next {
+ tracing::warn!(
+ used_mb = memtable_used / (1024 * 1024),
+ budget_mb = self.memtable_budget_bytes / (1024 * 1024),
+ "summed memtable crossed the process budget — engaging the parent circuit breaker (writes → 429)"
+ );
+ } else {
+ tracing::info!(
+ used_mb = memtable_used / (1024 * 1024),
+ "summed memtable back below the process budget — releasing the parent circuit breaker"
+ );
+ }
+ }
+ }
+
+ // ── Observability ───────────────────────────────────────────────────
+
+ /// Snapshot of the current governor state for `_nodes/stats`-style
+ /// surfaces.
+ pub fn snapshot(&self) -> GovernorSnapshot {
+ GovernorSnapshot {
+ memtable_used_bytes: self.memtable_used_bytes.load(Ordering::Relaxed),
+ memtable_budget_bytes: self.memtable_budget_bytes,
+ memtable_tripped: self.memtable_tripped.load(Ordering::Relaxed),
+ rss_bytes: self.rss_bytes.load(Ordering::Relaxed),
+ memory_limit_bytes: self.memory_limit_bytes,
+ memory_watermark_bytes: self.memory_watermark_bytes,
+ memory_tripped: self.memory_tripped.load(Ordering::Relaxed),
+ disk_used_pct: self.disk_used_pct.load(Ordering::Relaxed),
+ disk_flood_pct: self.disk_flood_pct,
+ disk_blocked: self.disk_blocked.load(Ordering::Relaxed),
+ max_concurrent_searches: self.max_concurrent_searches,
+ search_inflight: self.search_inflight.load(Ordering::Relaxed),
+ search_inflight_peak: self.search_inflight_peak.load(Ordering::Relaxed),
+ }
+ }
+}
+
+/// RAII guard for a held global search permit. Decrements the in-flight
+/// gauge on drop; the underlying semaphore permit is released with it.
+pub struct SearchPermit {
+ _permit: OwnedSemaphorePermit,
+ inflight: Arc<AtomicU64>,
+}
+
+impl Drop for SearchPermit {
+ fn drop(&mut self) {
+ self.inflight.fetch_sub(1, Ordering::Relaxed);
+ }
+}
+
+/// A cheap, `Copy`-able snapshot of governor state.
+#[derive(Debug, Clone, Copy)]
+pub struct GovernorSnapshot {
+ pub memtable_used_bytes: u64,
+ pub memtable_budget_bytes: u64,
+ pub memtable_tripped: bool,
+ pub rss_bytes: u64,
+ pub memory_limit_bytes: u64,
+ pub memory_watermark_bytes: u64,
+ pub memory_tripped: bool,
+ pub disk_used_pct: u64,
+ pub disk_flood_pct: u8,
+ pub disk_blocked: bool,
+ pub max_concurrent_searches: usize,
+ pub search_inflight: u64,
+ pub search_inflight_peak: u64,
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Initialisation / access
+// ─────────────────────────────────────────────────────────────────────────
+
+/// Initialise the process-wide governor from config. Idempotent: the first
+/// call wins (subsequent calls return the already-installed governor, so a
+/// second `Engine` in-process — e.g. a test — does not re-key the budgets).
+pub fn init(config: &Config) -> Arc<ResourceGovernor> {
+ Arc::clone(GOVERNOR.get_or_init(|| Arc::new(build(config))))
+}
+
+/// The installed governor, if [`init`] has run. Engine-only unit tests that
+/// construct an [`crate::index::Index`] directly (never calling
+/// `Engine::new`) get `None` and skip all admission checks — behaviour is
+/// unchanged for them.
+pub fn global() -> Option<Arc<ResourceGovernor>> {
+ GOVERNOR.get().map(Arc::clone)
+}
+
+fn build(config: &Config) -> ResourceGovernor {
+ let limits = &config.limits;
+
+ // ── memtable budget: 0 = auto-derive to 25% RAM, floored at 2 GiB ──
+ let memtable_budget_bytes = if limits.max_total_memtable_mb != 0 {
+ limits.max_total_memtable_mb.saturating_mul(1024 * 1024)
+ } else {
+ let sys = system_total_bytes();
+ (sys / 4).max(2 * 1024 * 1024 * 1024)
+ };
+
+ // ── RSS watermark against the effective memory limit ──
+ let memory_limit_bytes = effective_memory_limit_bytes();
+ let memory_watermark_bytes = if limits.memory_watermark_percent == 0 {
+ 0
+ } else {
+ let pct = limits.memory_watermark_percent.min(100) as u64;
+ ((memory_limit_bytes as u128 * pct as u128) / 100) as u64
+ };
+
+ let max_query_memory_bytes = limits.max_query_memory_mb.saturating_mul(1024 * 1024);
+
+ let max_concurrent_searches = (limits.max_concurrent_searches.max(1)) as usize;
+
+ tracing::info!(
+ memtable_budget_mb = memtable_budget_bytes / (1024 * 1024),
+ memory_limit_mb = memory_limit_bytes / (1024 * 1024),
+ memory_watermark_mb = memory_watermark_bytes / (1024 * 1024),
+ memory_watermark_pct = limits.memory_watermark_percent,
+ max_query_memory_mb = limits.max_query_memory_mb,
+ max_concurrent_searches,
+ disk_flood_pct = limits.disk_flood_stage_percent,
+ "resource governor initialised (parent circuit breaker)"
+ );
+
+ ResourceGovernor {
+ memtable_budget_bytes,
+ memtable_used_bytes: AtomicU64::new(0),
+ memtable_tripped: AtomicBool::new(false),
+ memory_limit_bytes,
+ memory_watermark_bytes,
+ rss_bytes: AtomicU64::new(0),
+ memory_tripped: AtomicBool::new(false),
+ max_query_memory_bytes,
+ search_pool: Arc::new(Semaphore::new(max_concurrent_searches)),
+ max_concurrent_searches,
+ search_inflight: Arc::new(AtomicU64::new(0)),
+ search_inflight_peak: Arc::new(AtomicU64::new(0)),
+ disk_flood_pct: limits.disk_flood_stage_percent.min(100),
+ disk_blocked: AtomicBool::new(false),
+ disk_used_pct: AtomicU64::new(0),
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// System probes (Linux; best-effort with safe fallbacks elsewhere)
+// ─────────────────────────────────────────────────────────────────────────
+
+/// Current resident set size of this process, in bytes. Reads
+/// `/proc/self/statm` (field 2 = resident pages). Returns 0 if unreadable
+/// (the RSS watermark then never trips — safe).
+pub fn current_rss_bytes() -> u64 {
+ #[cfg(target_os = "linux")]
+ {
+ if let Ok(s) = std::fs::read_to_string("/proc/self/statm") {
+ if let Some(res) = s.split_whitespace().nth(1) {
+ if let Ok(pages) = res.parse::<u64>() {
+ return pages.saturating_mul(page_size_bytes());
+ }
+ }
+ }
+ 0
+ }
+ #[cfg(not(target_os = "linux"))]
+ {
+ 0
+ }
+}
+
+#[cfg(target_os = "linux")]
+fn page_size_bytes() -> u64 {
+ // SAFETY: sysconf is a pure read of a system constant.
+ let p = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
+ if p > 0 {
+ p as u64
+ } else {
+ 4096
+ }
+}
+
+/// Total system RAM in bytes (from `/proc/meminfo`). Falls back to 8 GiB if
+/// unreadable so budgets stay sane on exotic platforms.
+fn system_total_bytes() -> u64 {
+ #[cfg(target_os = "linux")]
+ {
+ if let Ok(s) = std::fs::read_to_string("/proc/meminfo") {
+ for line in s.lines() {
+ if let Some(rest) = line.strip_prefix("MemTotal:") {
+ if let Some(kb) = rest.split_whitespace().next() {
+ if let Ok(kb) = kb.parse::<u64>() {
+ return kb.saturating_mul(1024);
+ }
+ }
+ }
+ }
+ }
+ }
+ 8 * 1024 * 1024 * 1024
+}
+
+/// Effective process memory limit: the cgroup memory limit when one is set
+/// (container / `systemd-run -p MemoryMax=`), otherwise total system RAM.
+/// Takes the min of the two so a generous cgroup value never exceeds RAM.
+pub fn effective_memory_limit_bytes() -> u64 {
+ let sys = system_total_bytes().max(1);
+ match cgroup_memory_limit_bytes() {
+ Some(c) if c > 0 && c < sys => c,
+ _ => sys,
+ }
+}
+
+/// Read the cgroup memory limit for THIS process. Handles cgroup v2
+/// (`memory.max`, walking up to a parent when a leaf reads `max`) and falls
+/// back to cgroup v1 (`memory.limit_in_bytes`). Returns `None` when no
+/// finite limit applies.
+#[cfg(target_os = "linux")]
+fn cgroup_memory_limit_bytes() -> Option<u64> {
+ // Sentinels used by the kernel/systemd to mean "unlimited".
+ const UNLIMITED_V1: u64 = 9_223_372_036_854_771_712; // PAGE_COUNTER_MAX * page
+ let finite = |v: u64| -> Option<u64> {
+ if v == 0 || v >= UNLIMITED_V1 {
+ None
+ } else {
+ Some(v)
+ }
+ };
+
+ // cgroup v2: /proc/self/cgroup has a single "0::<path>" line.
+ if let Ok(cg) = std::fs::read_to_string("/proc/self/cgroup") {
+ for line in cg.lines() {
+ if let Some(path) = line.strip_prefix("0::") {
+ // Walk from the leaf cgroup up to the root, honouring the
+ // nearest finite memory.max (systemd sets MemoryMax on the
+ // scope, which may be a parent of the leaf).
+ let mut rel = path.trim().to_string();
+ loop {
+ let full = format!("/sys/fs/cgroup{rel}/memory.max");
+ if let Ok(s) = std::fs::read_to_string(&full) {
+ let s = s.trim();
+ if s != "max" {
+ if let Ok(v) = s.parse::<u64>() {
+ if let Some(v) = finite(v) {
+ return Some(v);
+ }
+ }
+ }
+ }
+ if rel.is_empty() || rel == "/" {
+ break;
+ }
+ match rel.rfind('/') {
+ Some(0) => rel = String::new(), // step to root next iter
+ Some(i) => rel.truncate(i),
+ None => break,
+ }
+ }
+ }
+ }
+ }
+
+ // cgroup v1 fallback.
+ if let Ok(s) = std::fs::read_to_string("/sys/fs/cgroup/memory/memory.limit_in_bytes") {
+ if let Ok(v) = s.trim().parse::<u64>() {
+ return finite(v);
+ }
+ }
+ None
+}
+
+#[cfg(not(target_os = "linux"))]
+fn cgroup_memory_limit_bytes() -> Option<u64> {
+ None
+}
+
+/// `(total_bytes, avail_bytes)` for the filesystem backing `path`, via
+/// `statvfs(2)`. Returns `None` on syscall failure.
+#[cfg(unix)]
+#[allow(clippy::unnecessary_cast)]
+pub fn disk_stats(path: &str) -> Option<(u64, u64)> {
+ let c = std::ffi::CString::new(path).ok()?;
+ // SAFETY: statvfs fully initialises the struct; we only read scalars.
+ let mut st: libc::statvfs = unsafe { std::mem::zeroed() };
+ if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 {
+ return None;
+ }
+ let bsize = if st.f_frsize > 0 {
+ st.f_frsize as u64
+ } else {
+ st.f_bsize as u64
+ };
+ let total = (st.f_blocks as u64).saturating_mul(bsize);
+ let avail = (st.f_bavail as u64).saturating_mul(bsize);
+ if total == 0 {
+ return None;
+ }
+ Some((total, avail))
+}
+
+#[cfg(not(unix))]
+pub fn disk_stats(_path: &str) -> Option<(u64, u64)> {
+ None
+}
+
+/// Used-percentage of the filesystem backing `path` (0..=100). Uses
+/// `total - avail` over `total`, matching ES's disk-watermark accounting.
+/// Returns 0 when `statvfs` is unavailable (the disk block never trips).
+pub fn disk_used_pct(path: &str) -> u64 {
+ match disk_stats(path) {
+ Some((total, avail)) if total > 0 => {
+ let used = total.saturating_sub(avail);
+ ((used as u128 * 100) / total as u128) as u64
+ }
+ _ => 0,
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Small formatting helpers
+// ─────────────────────────────────────────────────────────────────────────
+
+fn pct_of(part: u64, whole: u64) -> u64 {
+ if whole == 0 {
+ 0
+ } else {
+ ((part as u128 * 100) / whole as u128) as u64
+ }
+}
+
+fn human_bytes(b: u64) -> String {
+ const KB: u64 = 1024;
+ const MB: u64 = 1024 * KB;
+ const GB: u64 = 1024 * MB;
+ if b >= GB {
+ format!("{:.1}gb", b as f64 / GB as f64)
+ } else if b >= MB {
+ format!("{:.1}mb", b as f64 / MB as f64)
+ } else if b >= KB {
+ format!("{:.1}kb", b as f64 / KB as f64)
+ } else {
+ format!("{b}b")
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn effective_limit_is_positive() {
+ assert!(effective_memory_limit_bytes() > 0);
+ }
+
+ #[test]
+ fn disk_used_pct_bounded() {
+ let p = disk_used_pct("/");
+ assert!(p <= 100);
+ }
+
+ #[test]
+ fn human_bytes_scales() {
+ assert_eq!(human_bytes(512), "512b");
+ assert_eq!(human_bytes(2 * 1024 * 1024), "2.0mb");
+ }
+
+ #[test]
+ fn build_from_default_config_trips_nothing() {
+ let cfg = Config::default();
+ let g = build(&cfg);
+ // Fresh: nothing sampled yet, so no trips and ingest is admitted.
+ assert!(g.check_ingest_admission().is_ok());
+ assert!(g.check_disk_block("i").is_ok());
+ assert_eq!(g.max_concurrent_searches(), 64);
+ }
+
+ #[test]
+ fn memtable_budget_trips_on_refresh() {
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 100; // 100 MiB ceiling
+ cfg.limits.memory_watermark_percent = 0; // isolate the memtable path
+ let g = build(&cfg);
+ assert!(g.check_ingest_admission().is_ok());
+ g.refresh(200 * 1024 * 1024, 0, 0); // 200 MiB used > 100 MiB budget
+ let err = g.check_ingest_admission().unwrap_err();
+ // 429 + the CircuitBreaking variant (the ES mapper stamps the
+ // `circuit_breaking_exception` type; the Display is the bare reason).
+ assert_eq!(err.http_status(), 429);
+ assert!(matches!(err, XerjError::CircuitBreaking { .. }));
+ assert!(format!("{err}").contains("memtable byte budget exceeded"));
+ // Recovery once usage drops back under the budget.
+ g.refresh(10 * 1024 * 1024, 0, 0);
+ assert!(g.check_ingest_admission().is_ok());
+ }
+
+ #[test]
+ fn disk_flood_blocks_with_hysteresis() {
+ let mut cfg = Config::default();
+ cfg.limits.disk_flood_stage_percent = 95;
+ let g = build(&cfg);
+ assert!(g.check_disk_block("i").is_ok());
+ g.refresh(0, 0, 96); // over flood stage
+ assert!(g.check_disk_block("i").is_err());
+ g.refresh(0, 0, 95); // still within release hysteresis (>= 94)
+ assert!(g.check_disk_block("i").is_err());
+ g.refresh(0, 0, 90); // clearly recovered
+ assert!(g.check_disk_block("i").is_ok());
+ }
+
+ #[test]
+ fn query_alloc_guard_fires() {
+ let mut cfg = Config::default();
+ cfg.limits.max_query_memory_mb = 1; // 1 MiB
+ let g = build(&cfg);
+ assert!(g.check_query_alloc(512 * 1024, "t").is_ok());
+ let err = g.check_query_alloc(4 * 1024 * 1024, "t").unwrap_err();
+ assert_eq!(err.http_status(), 429);
+ }
+
+ #[tokio::test]
+ async fn global_search_pool_caps_concurrency() {
+ // Item 2: a pool of 2 admits exactly 2 concurrent searches; the 3rd
+ // acquire blocks until one is released.
+ let mut cfg = Config::default();
+ cfg.limits.max_concurrent_searches = 2;
+ let g = Arc::new(build(&cfg));
+ assert_eq!(g.max_concurrent_searches(), 2);
+
+ let p1 = g.acquire_search().await.unwrap();
+ let p2 = g.acquire_search().await.unwrap();
+ assert_eq!(g.search_inflight(), 2);
+
+ // A 3rd acquire must NOT complete while 2 are held.
+ let g3 = Arc::clone(&g);
+ let third = tokio::spawn(async move { g3.acquire_search().await.map(|_p| ()) });
+ assert!(
+ tokio::time::timeout(std::time::Duration::from_millis(150), third)
+ .await
+ .is_err(),
+ "3rd search must block while the pool of 2 is full"
+ );
+
+ // Release one → a subsequent acquire succeeds and peak stays at 2.
+ drop(p1);
+ let _p3 = tokio::time::timeout(std::time::Duration::from_millis(500), g.acquire_search())
+ .await
+ .expect("acquire must proceed once a permit frees")
+ .unwrap();
+ assert_eq!(g.search_inflight_peak(), 2, "concurrency never exceeded 2");
+ drop(p2);
+ }
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0254d8f50..b2fb86a97 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -432,6 +432,17 @@ pub struct Index {
hnsw_id_rev: Arc<RwLock<HashMap<u64, String>>>,
/// Monotonic counter for HNSW node IDs.
hnsw_next_id: Arc<AtomicU64>,
+ /// Number of distinct live docs that CARRY the pinned vector field (item
+ /// 8). This is the coverage-gate denominator: the ANN query path may only
+ /// serve when `hnsw_id_map.len() == vector_doc_count`, i.e. every
+ /// vector-bearing doc made it into the graph. The old gate compared
+ /// against the index-global `doc_count`, so a SINGLE doc lacking the
+ /// vector field (a mixed log/vector corpus) left `id_map.len() < doc_count`
+ /// forever and silently pinned the whole index on O(N) brute force.
+ /// Incremented once per doc-id newly added to the graph, decremented on
+ /// delete; never above the true count (over-count on updates biases the
+ /// gate to brute, which is safe).
+ vector_doc_count: Arc<AtomicU64>,
/// FIELD IDENTITY: the single field this index's HNSW graph indexes.
///
/// Pinned when the graph is first created (preferring the schema's first
@@ -878,6 +889,7 @@ impl Index {
hnsw_id_map: Arc::new(RwLock::new(HashMap::new())),
hnsw_id_rev: Arc::new(RwLock::new(HashMap::new())),
hnsw_next_id: Arc::new(AtomicU64::new(1)),
+ vector_doc_count: Arc::new(AtomicU64::new(0)),
hnsw_field: Arc::new(std::sync::RwLock::new(None)),
hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(false)),
hnsw_rebuilding: Arc::new(std::sync::atomic::AtomicBool::new(false)),
@@ -1071,6 +1083,11 @@ impl Index {
}
None => (None, HashMap::new(), HashMap::new(), 1, None, false),
};
+ // Item 8: seed the coverage-gate denominator from the persisted graph.
+ // At flush time the graph covered every vector-bearing doc, so the
+ // reloaded `id_map.len()` IS the vector-doc count. If the snapshot is
+ // stale, `hnsw_stale` already forces brute regardless of this value.
+ let vector_doc_count_init = id_map_init.len() as u64;
if initial_hnsw {
info!(
name = name.as_str(),
@@ -1109,6 +1126,7 @@ impl Index {
hnsw_id_map: Arc::new(RwLock::new(id_map_init)),
hnsw_id_rev: Arc::new(RwLock::new(id_rev_init)),
hnsw_next_id: Arc::new(AtomicU64::new(next_id_init)),
+ vector_doc_count: Arc::new(AtomicU64::new(vector_doc_count_init)),
hnsw_field: Arc::new(std::sync::RwLock::new(hnsw_field_init)),
hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(hnsw_stale_init)),
hnsw_rebuilding: Arc::new(std::sync::atomic::AtomicBool::new(false)),
@@ -1223,6 +1241,9 @@ impl Index {
"write",
)));
}
+ // Process-wide admission: disk flood-stage block (item 3) + parent
+ // memory circuit breaker (item 1). Fires before any per-index work.
+ self.governor_write_gate()?;
// Invalidate the response cache: every doc write bumps the
// dataset version, which is part of the cache key, so old
// entries become invisible to lookups. No clear() needed —
@@ -1432,6 +1453,9 @@ impl Index {
"write",
)));
}
+ // Process-wide admission: disk flood-stage block (item 3) + parent
+ // memory circuit breaker (item 1). Fires before any per-index work.
+ self.governor_write_gate()?;
// ── Back-pressure ─────────────────────────────────────────────────
// Reject the batch with 429 ResourceExhausted if the memtable is
@@ -1746,6 +1770,9 @@ impl Index {
"write",
)));
}
+ // Process-wide admission: disk flood-stage block (item 3) + parent
+ // memory circuit breaker (item 1). Fires before any per-index work.
+ self.governor_write_gate()?;
let hard_block = self.flush_byte_threshold.saturating_mul(3);
let soft_block = self.flush_byte_threshold.saturating_mul(2);
@@ -2259,6 +2286,9 @@ impl Index {
"write",
)));
}
+ // Process-wide admission: disk flood-stage block (item 3) + parent
+ // memory circuit breaker (item 1). Fires before any per-index work.
+ self.governor_write_gate()?;
let batch_len = docs.len();
let index_start = std::time::Instant::now();
@@ -2353,6 +2383,9 @@ impl Index {
"write",
)));
}
+ // Process-wide admission: disk flood-stage block (item 3) + parent
+ // memory circuit breaker (item 1). Fires before any per-index work.
+ self.governor_write_gate()?;
// Fail with 409 if a live document already exists with this id.
let already_exists = {
@@ -3855,6 +3888,15 @@ impl Index {
// that node tombstoned before the fresh vector goes in (fresh
// node_id; the old node is never unmarked).
let old_node: Option<u64> = self.hnsw_id_map.read().await.get(doc_id).copied();
+ if old_node.is_none() {
+ // Item 8: a new vector-bearing doc-id. Counted on ATTEMPT (here),
+ // not on success below: if the insert then fails (dim mismatch /
+ // graph error), `id_map.len()` stays behind `vector_doc_count`,
+ // which holds the coverage gate on brute force — the safe
+ // direction (a doc that isn't in the graph must not be answered
+ // from it). An update (old_node.is_some()) is already counted.
+ self.vector_doc_count.fetch_add(1, Ordering::Relaxed);
+ }
let node_id = self.hnsw_next_id.fetch_add(1, Ordering::Relaxed);
{
let mut hnsw_guard = self.hnsw.write().await;
@@ -4140,6 +4182,12 @@ impl Index {
if let Some(h) = self.hnsw.read().await.as_ref() {
h.mark_deleted(node_id);
}
+ // Item 8: pruned doc leaves the coverage-gate denominator.
+ self.vector_doc_count
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| {
+ Some(v.saturating_sub(1))
+ })
+ .ok();
pruned += 1;
}
@@ -4194,12 +4242,26 @@ impl Index {
None => (false, 0, 0),
}
};
+ let graphed = self.hnsw_id_map.read().await.len() as u64;
+ // Item 8: coverage is graphed-docs / vector-bearing-docs. `covered`
+ // means the ANN path is eligible (every vector doc is in the graph);
+ // it stays `true` even when many docs lack the vector field (they are
+ // not vector docs and belong in neither the graph nor a brute scan).
+ let vector_docs = self.vector_doc_count.load(Ordering::Relaxed);
+ let doc_count = self.doc_count.load(Ordering::Relaxed);
+ let covered = present && graphed == vector_docs && !self.hnsw_stale.load(Ordering::Acquire);
serde_json::json!({
"present": present,
"field": self.hnsw_field.read().unwrap().clone(),
"nodes": nodes,
"tombstones": tombstones,
- "doc_coverage": self.hnsw_id_map.read().await.len() as u64,
+ "doc_coverage": graphed,
+ // How many live docs carry the pinned vector field. The ANN gate
+ // compares this against `doc_coverage`, NOT the index-wide
+ // `total_docs` — a mixed corpus can be fully covered.
+ "vector_doc_count": vector_docs,
+ "total_docs": doc_count,
+ "covered": covered,
"stale": self.hnsw_stale.load(Ordering::Acquire),
"rebuilding": self
.hnsw_rebuilding
@@ -4226,12 +4288,14 @@ impl Index {
/// flush-time stale (see `hnsw_field` / `hnsw_stale`);
/// * the field does not opt into SQ8 (`scalar8` keeps its dedicated
/// code-store scoring path);
- /// * `doc_count >= HNSW_MIN_DOCS` — small indexes (every ES-YAML
- /// vector test is ≤ a few hundred docs) keep today's byte-identical
- /// exact path;
- /// * coverage: `hnsw_id_map.len() == doc_count`, i.e. every live doc
- /// has a graph entry. Racy reads degrade to brute, never to wrong
- /// results.
+ /// * `vector_doc_count >= HNSW_MIN_DOCS` — small vector sets (every
+ /// ES-YAML vector test is ≤ a few hundred docs) keep today's
+ /// byte-identical exact path;
+ /// * coverage: `hnsw_id_map.len() == vector_doc_count` (item 8), i.e.
+ /// every doc that CARRIES the pinned vector field has a graph entry.
+ /// Docs lacking the field are not vector docs and belong in neither
+ /// the graph nor a brute scan, so a mixed corpus can be fully covered.
+ /// Racy reads degrade to brute, never to wrong results.
///
/// Results are EXACT-RESCORED: each of the ≤k candidate docs is fetched
/// (metrics-free, so `indices.stats` get counters stay ES-parity) and
@@ -4269,11 +4333,22 @@ impl Index {
return None;
}
}
- let doc_count = self.doc_count.load(Ordering::Relaxed);
- if doc_count < HNSW_MIN_DOCS {
+ // Item 8: both the min-size threshold and the coverage check are gated
+ // on the count of docs that CARRY the pinned vector field
+ // (`vector_doc_count`), NOT the index-global `doc_count`. The graph only
+ // ever holds vector-bearing docs, so its size and coverage are a
+ // function of that count. The old `id_map.len() == doc_count` test meant
+ // a SINGLE doc lacking the vector field (a mixed log/vector corpus) left
+ // `id_map.len() < doc_count` forever, silently pinning the WHOLE index
+ // on O(N) brute force. A doc without the field is correctly absent from
+ // both the graph and a brute scan, so ANN and brute return identical
+ // results. A racy read (id_map.len() != vector_doc_count) degrades to
+ // brute — never to wrong results.
+ let vector_doc_count = self.vector_doc_count.load(Ordering::Relaxed);
+ if vector_doc_count < HNSW_MIN_DOCS {
return None;
}
- if self.hnsw_id_map.read().await.len() as u64 != doc_count {
+ if self.hnsw_id_map.read().await.len() as u64 != vector_doc_count {
return None;
}
@@ -4348,6 +4423,18 @@ impl Index {
scored.push((doc_id, score, src));
}
+ // Item 8: this line is only reached when the coverage gate admitted
+ // the ANN path (`hnsw_id_map.len() == vector_doc_count`). Logged so a
+ // mixed-corpus query (docs without the vector field present) is
+ // observably served by HNSW rather than silently pinned on brute.
+ debug!(
+ index = self.name.as_str(),
+ field,
+ vector_doc_count = self.vector_doc_count.load(Ordering::Relaxed),
+ total_docs = self.doc_count.load(Ordering::Relaxed),
+ candidates = scored.len(),
+ "knn served via HNSW (coverage gate passed)"
+ );
Some(knn_result_from_scored(request, scored, k, started))
}
@@ -4780,9 +4867,11 @@ impl Index {
.embed_batch(vec![text.to_string()])
.await
.map_err(|e| {
- EngineError::Common(xerj_common::XerjError::invalid_query(format!(
- "semantic embed failed: {e}"
- )))
+ // Item 9: preserve the embedder's 5xx class. An embedding
+ // backend outage/misconfig is NOT a client query error;
+ // wrapping it as invalid_query (400) hid proxy failures
+ // behind a 400 and drove clients to retry a broken backend.
+ EngineError::Common(embed_backend_error(e, "semantic query embed failed"))
})?;
match vectors.pop() {
Some(v) if !v.is_empty() => v,
@@ -4877,9 +4966,12 @@ impl Index {
chunks.iter().map(|c| c.text.clone()).collect()
};
let vs = self.embedder.embed_batch(chunk_texts).await.map_err(|e| {
- EngineError::Common(xerj_common::XerjError::invalid_query(format!(
- "semantic_text embed failed for field [{field}]: {e}"
- )))
+ // Item 9: preserve the embedder's 5xx class on ingest too —
+ // a proxy outage must not masquerade as a 400 bad document.
+ EngineError::Common(embed_backend_error(
+ e,
+ &format!("semantic_text embed failed for field [{field}]"),
+ ))
})?;
vs.into_iter().filter(|v| !v.is_empty()).collect()
} else {
@@ -5317,6 +5409,11 @@ impl Index {
let mut id_rev = self.hnsw_id_rev.write().await;
*id_map = loaded.id_map;
*id_rev = loaded.id_rev;
+ // Item 8: re-seed the coverage-gate denominator from the reloaded
+ // graph (every graphed doc is a vector doc). Staleness is handled
+ // separately by `hnsw_stale`.
+ self.vector_doc_count
+ .store(id_map.len() as u64, Ordering::Relaxed);
drop(id_map);
drop(id_rev);
self.hnsw_next_id.store(loaded.next_id, Ordering::Relaxed);
@@ -5633,6 +5730,9 @@ impl Index {
"write",
)));
}
+ // Process-wide admission: disk flood-stage block (item 3) + parent
+ // memory circuit breaker (item 1). Fires before any per-index work.
+ self.governor_write_gate()?;
self.dataset_version.fetch_add(1, Ordering::Release);
// Optimistic concurrency check (see index_document_with_version for
@@ -5728,6 +5828,12 @@ impl Index {
// gets a fresh node_id (the old one stays tombstoned).
self.hnsw_id_map.write().await.remove(id);
self.hnsw_id_rev.write().await.remove(&node_id);
+ // Item 8: this doc leaves the coverage-gate denominator.
+ self.vector_doc_count
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| {
+ Some(v.saturating_sub(1))
+ })
+ .ok();
}
if existed {
@@ -5984,10 +6090,22 @@ impl Index {
}
}
- // Acquire a query permit before proceeding. This bounds the number of
- // queries executing concurrently against this index to 64 (the semaphore
- // capacity), preventing a single hot index from starving others in a
- // multi-tenant deployment. The permit is automatically released when
+ // Item 2 — global search pool. Acquire a PROCESS-WIDE search permit
+ // sized from `limits.max_concurrent_searches` before proceeding. The
+ // per-index semaphore below only ever bounded one index (and was a
+ // hardcoded 64), so N hot indices could still run N×64 searches at
+ // once with no global ceiling. The global permit caps total in-flight
+ // searches across every index; excess searches queue on it, exactly
+ // like ES's search thread pool bounds active workers. Held (via
+ // `_global_permit`) for the whole call; released on drop.
+ let _global_permit = match crate::governor::global() {
+ Some(g) => Some(g.acquire_search().await?),
+ None => None,
+ };
+
+ // Acquire a per-index query permit too. This bounds concurrency
+ // against a single index, preventing one hot index from monopolising
+ // the global pool. The permit is automatically released when
// `_permit` is dropped at the end of this call.
let _permit = self.max_concurrent_queries.acquire().await.map_err(|_| {
EngineError::Common(xerj_common::XerjError::internal(
@@ -5997,6 +6115,29 @@ impl Index {
let search_start = std::time::Instant::now();
+ // Item 1: per-query memory guard (`max_query_memory_mb`). Estimate the
+ // hydration cost of the requested result window (from + size hits, each
+ // materialised with its `_source` and scoring scratch) and reject
+ // up-front with a 429 `circuit_breaking_exception` if it would exceed
+ // the budget — before the Vec<Hit> is allocated. This is the
+ // request-scoped breaker; the parent (memtable/RSS) breaker is the
+ // process-wide one. A no-op when no governor is installed or the
+ // budget is 0 (disabled). At the 512 MiB default a full 10 000-hit
+ // window (~10 MiB) is nowhere near the cap, so normal search is
+ // unaffected; the guard only bites pathological windows or a
+ // deliberately low operator budget.
+ if let Some(g) = crate::governor::global() {
+ if g.query_memory_enabled() {
+ // Conservative per-hit estimate: stored source + collector
+ // scratch. Deep windows dominate query memory in practice.
+ const EST_BYTES_PER_HIT: u64 = 1024;
+ let window = (request.from as u64).saturating_add(request.size as u64);
+ let est = window.saturating_mul(EST_BYTES_PER_HIT);
+ g.check_query_alloc(est, "result-window hydration")
+ .map_err(EngineError::Common)?;
+ }
+ }
+
// Determine the timeout: use the request-level timeout if set, otherwise
// fall back to the default of 30 seconds.
let timeout_ms = request.timeout_ms.unwrap_or(30_000);
@@ -9823,6 +9964,27 @@ impl Index {
// ── Index blocks ──────────────────────────────────────────────────────────
+ /// Process-wide write admission (RC4 W3 items 1 & 3). Applies, in order:
+ ///
+ /// 1. the disk flood-stage block (item 3) — a 429 `cluster_block_exception`
+ /// `read_only_allow_delete` when the data-dir filesystem is over the
+ /// watermark, before the WAL can be driven to `ENOSPC`;
+ /// 2. the parent memory circuit breaker (item 1) — a 429
+ /// `circuit_breaking_exception` when the summed memtable footprint or
+ /// the process RSS is over budget, so a 429 beats the OOM-killer.
+ ///
+ /// A no-op when no governor is installed (engine-only unit tests that
+ /// build an `Index` directly), so their behaviour is unchanged.
+ #[inline]
+ fn governor_write_gate(&self) -> Result<()> {
+ if let Some(g) = crate::governor::global() {
+ g.check_disk_block(self.name.as_str())
+ .map_err(EngineError::Common)?;
+ g.check_ingest_admission().map_err(EngineError::Common)?;
+ }
+ Ok(())
+ }
+
/// Returns true if the write block is set on this index.
pub async fn is_write_blocked(&self) -> bool {
let settings = self.settings.read().await;
@@ -15734,6 +15896,17 @@ fn build_highlight_fragments(
/// Any misconfiguration degrades to the lexical fallback (never a crash);
/// a `semantic` query with no active backend against a plain vector field
/// still returns a helpful 400.
+/// Map an embedder failure to the correct ES class (item 9). An embedding
+/// backend outage or misconfiguration is a server-side (5xx) failure, never a
+/// 400 `invalid_query`: downgrading it hid proxy failures behind a client
+/// error and drove clients to retry a broken backend. Produces an
+/// `EmbeddingError` (500, `circuit_breaking_exception`) with `ctx` prepended.
+/// Takes `impl Display` because the `Embedder` surfaces `anyhow::Error` (the
+/// classification of transient-vs-permanent already happened inside the proxy).
+fn embed_backend_error(e: impl std::fmt::Display, ctx: &str) -> xerj_common::XerjError {
+ xerj_common::XerjError::embedding(format!("{ctx}: {e}"))
+}
+
fn make_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> xerj_ai::Embedder {
let mode = cfg.mode.trim().to_ascii_lowercase();
let want_proxy = mode == "proxy"
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 7026a600f..e9c1e65b4 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -20,6 +20,7 @@ pub mod aggs;
pub mod audit;
pub mod bulk;
pub mod engine;
+pub mod governor;
pub mod index;
pub mod memtable;
pub mod painless;
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 744f8d4ab..98de7f67a 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -1251,6 +1251,13 @@ async fn async_main() -> Result<()> {
std::time::Duration::from_secs(cfg.storage.flush_interval_secs),
);
+ // 12b. Resource governor sampler (RC4 W3 items 1 & 3): refreshes the
+ // process-wide memtable / RSS / disk-usage atomics that drive the
+ // parent circuit breaker and the disk flood-stage write block. This
+ // is the structural guard against the 112 GiB OOM class — writes get
+ // a 429 circuit_breaking_exception before the kernel OOM-kills us.
+ state.engine.spawn_resource_sampler();
+
// 13. Start servers concurrently
let rest_tls = tls_config.clone();
let rest = tokio::spawn(async move {