AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): flush a full memtable budget instead of stalling autoindex (#2325)
xerj-org78752dec7a05
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8cbcebc5a..0ce591c85 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -85,6 +85,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- A full process memtable budget now flushes instead of leaving `xerj autoindex`+ on HTTP 429 until it gives up. The rejection requests a drain (largest+ memtable first) rather than waiting for the next memory sample, and a+ search issued while that breaker is engaged publishes those memtables+ before it scans them. `xerj search` / `xerj def` saying `no indices named+ …` names the prefixes on the node in that same line, including a prefix+ recorded only on a catalog dataset document+ ([#2321](https://github.com/xerj-org/xerj/issues/2321)).+- A multi-term question no longer ranks a one-word symbol definition(`type Token`) above a hit that covers more of the question. `xerj search`,`xerj def`, `xerj code`, and the plain-string `xerj_search` tool add adiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex cde14221a..4dddc4284 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -13088,6 +13088,35 @@ pub(crate) async fn search_shard_local(.await}+/// Heap-allocate the publish future before it is polled.+///+/// `publish_memtables_if_budget_engaged` awaits `Index::flush`. That future+/// is large enough that building it inside `search_impl` overflows the worker+/// stack (#2321). A sync function creates it here, on the task that is about+/// to poll it, and only the box escapes.+#[inline(never)]+fn boxed_memtable_publish(+ engine: std::sync::Arc<xerj_engine::Engine>,+) -> std::pin::Pin<Box<dyn std::future::Future<Output = usize> + Send>> {+ Box::pin(async move { engine.publish_memtables_if_budget_engaged().await })+}++/// Publish memtables that are holding the process budget, on a fresh task.+///+/// A no-op — and no spawn — when the memtable breaker is off, so an ordinary+/// search does not flush. The spawn keeps `Index::flush`'s frame off+/// `search_impl`'s stack.+async fn publish_engaged_memtables(engine: std::sync::Arc<xerj_engine::Engine>) {+ if !xerj_engine::governor::global().is_some_and(|governor| governor.memtable_breaker_engaged())+ {+ return;+ }+ let _ = tokio::spawn(async move {+ let _published = boxed_memtable_publish(engine).await;+ })+ .await;+}+async fn search_impl(state: AppState,index: String,@@ -13099,6 +13128,10 @@ async fn search_impl(// Read once, before the first `.await`: what the in-process hub caller asked// of this search. A request off the wire gets the default and is unaffected.let in_process = in_process_search();+ // #2321: a memtable sitting on the process budget is what a `{prefix}-*`+ // search scans. Publish it before this request walks it. No-op unless+ // that breaker is engaged. Read `in_process` first — it is task-local.+ publish_engaged_memtables(std::sync::Arc::clone(&state.engine)).await;// Closed-index handling is deferred to after index resolution below so// `ignore_unavailable` / `expand_wildcards` are honored per ES semantics.diff --git a/engine/crates/xerj-api/tests/memtable_budget_autoindex.rs b/engine/crates/xerj-api/tests/memtable_budget_autoindex.rsnew file mode 100644index 000000000..8c33ca417--- /dev/null+++ b/engine/crates/xerj-api/tests/memtable_budget_autoindex.rs@@ -0,0 +1,231 @@+//! Issue #2321: `xerj autoindex` against a node whose memtable budget fills+//! must finish (the server flushes, the client backs off and resumes), and a+//! search after that run must stay fast.+//!+//! Scaled down from the report: 1 MiB process budget and a flush line the+//! corpus never reaches, so the only flush that can free the budget is the+//! pressure drain. This file is its own binary so its governor is the first+//! one installed.++use std::path::Path;+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::Arc;+use std::time::{Duration, Instant};++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tempfile::TempDir;+use tower::ServiceExt;+use xerj_autoindex::cli::IndexCfg;+use xerj_engine::Engine;++const BUDGET: u64 = 1024 * 1024;++fn boot() -> (axum::Router, Arc<Engine>, TempDir) {+ let dir = tempfile::tempdir().expect("data dir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.auth.enabled = false;+ config.limits.max_total_memtable_mb = 1;+ config.limits.memory_watermark_percent = 0;+ config.storage.flush_size_mb = 64;+ config.storage.flush_idle_secs = 0;+ config.storage.flush_interval_secs = 3600;+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ state.engine.spawn_resource_sampler();+ let engine = Arc::clone(&state.engine);+ let app = xerj_api::router::build_es_compat_router(state);+ (app, engine, dir)+}++fn index_cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {+ IndexCfg {+ root: root.to_owned(),+ endpoint_url: None,+ stub_globs: Vec::new(),+ url: url.to_owned(),+ api_key: None,+ api_key_file: None,+ workers: 2,+ scan_workers: 1,+ pdf_workers: 1,+ resource_notes: Vec::new(),+ xerj_url_note: None,+ pdf_timeout_secs: 30,+ bulk_mb: 1,+ bulk_timeout_secs: 60,+ snapshot_max_bytes: 64 << 30,+ prefix: "budget".to_owned(),+ replace_prefix: false,+ prefix_note: None,+ state_dir: Some(state_dir.to_owned()),+ fresh: false,+ follow_symlinks: false,+ follow_symlinks_outside_root: false,+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),+ max_file_bytes: 1 << 30,+ sample: 50,+ no_semantic: true,+ code_analyzer: Default::default(),+ brain: None,+ no_graph: true,+ max_minutes: 0,+ approve: None,+ max_index_bytes: None,+ budget_plan: None,+ drop_indexed: false,+ dry_run: false,+ walk_only: false,+ json: false,+ quiet: true,+ progress: xerj_autoindex::progress::ProgressMode::None,+ progress_interval: None,+ watch: false,+ label: None,+ debounce: Duration::from_millis(0),+ exclude_rels: Vec::new(),+ skip_corpus_data: false,+ }+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method(method)+ .uri(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request"),+ )+ .await+ .expect("response");+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("body");+ let parsed = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ (status, parsed)+}++fn corpus(root: &Path) {+ let pad = "a".repeat(3000);+ let mut file = String::new();+ for i in 0..600 {+ let line = serde_json::to_string(&json!({+ "body": format!("budgettoken {i} {pad}")+ }))+ .unwrap();+ file.push_str(&line);+ file.push('\n');+ }+ std::fs::write(root.join("records.jsonl"), file).unwrap();+}++#[test]+fn autoindex_completes_when_the_memtable_budget_fills_and_search_stays_fast() {+ // `#[tokio::test]` workers use tokio's 2 MiB stack. The production runtime+ // is 4 MiB (`xerj-server`'s `RT_THREAD_STACK_SIZE`) because the ES-compat+ // handlers do not fit in 2 MiB in a debug build. Match production.+ let runtime = tokio::runtime::Builder::new_multi_thread()+ .worker_threads(4)+ .thread_stack_size(4 * 1024 * 1024)+ .enable_all()+ .build()+ .expect("runtime");+ runtime.block_on(autoindex_budget_run());+}++async fn autoindex_budget_run() {+ let (app, engine, _dir) = boot();+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let served = app.clone();+ tokio::spawn(async move {+ axum::serve(listener, served).await.unwrap();+ });+ let url = format!("http://{addr}");++ let root = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ corpus(root.path());++ let peak = Arc::new(AtomicU64::new(0));+ let peak_engine = Arc::clone(&engine);+ let peak_slot = Arc::clone(&peak);+ let poller = tokio::spawn(async move {+ loop {+ let bytes = peak_engine.total_memtable_bytes();+ peak_slot.fetch_max(bytes, Ordering::Relaxed);+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ });++ let root_path = root.path().to_owned();+ let state_path = state_dir.path().to_owned();+ let run = tokio::task::spawn_blocking(move || {+ xerj_autoindex::run_index_report(index_cfg(&root_path, &state_path, &url))+ });+ let (code, report) = tokio::time::timeout(Duration::from_secs(120), run)+ .await+ .expect("autoindex did not finish within 120s — the memtable-budget stall")+ .expect("autoindex task")+ .expect("autoindex report");+ poller.abort();+ assert!(+ code == 0 || code == 3,+ "autoindex exit {code}, report {report:?}"+ );++ let peaked = peak.load(Ordering::Relaxed);+ assert!(+ peaked >= BUDGET,+ "the run must fill the 1 MiB memtable budget (peak {peaked} bytes). \+ A run that never reaches it does not exercise the drain."+ );++ let held = engine.total_memtable_bytes();+ assert!(+ held < BUDGET,+ "autoindex returned with the memtable still over budget ({held} bytes) — \+ the drain did not free it"+ );+ let infos = engine.list_indices().await;+ let segments: usize = infos+ .iter()+ .filter(|info| info.name.starts_with("budget"))+ .map(|info| info.segment_count)+ .sum();+ assert!(+ segments > 0,+ "the pressure flush must publish a segment under the budget prefix: {infos:?}"+ );++ let started = Instant::now();+ let (status, body) = call(+ &app,+ "POST",+ "/budget-*/_search",+ json!({+ "size": 3,+ "query": { "match": { "body": "budgettoken" } }+ }),+ )+ .await;+ let elapsed = started.elapsed();+ assert_eq!(status, StatusCode::OK, "search {body}");+ assert!(+ elapsed < Duration::from_secs(5),+ "search after autoindex took {elapsed:?}"+ );+ let total = body["hits"]["total"]["value"].as_u64().unwrap_or(0);+ assert!(+ total > 0,+ "budgettoken must be searchable after the run: {body}"+ );+}diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rsindex a177fc4d0..51b9b6ad3 100644--- a/engine/crates/xerj-autoindex/src/search.rs+++ b/engine/crates/xerj-autoindex/src/search.rs@@ -386,9 +386,12 @@ pub(crate) struct CorpusSnapshot {/// `xerj def` has no hits.////// A prefix that matches no index exits `1` and names the prefixes already-/// on the node. The `xerj autoindex <folder>` hint is only for a node whose-/// catalog and index list are both empty — a miss against a real corpus is-/// not a reason to tell the operator to index again.+/// on the node, in that same line. The line says it is not a search result:+/// it returns before any query runs, and a one-line reading of "no indices+/// named …" was taken for an instant empty search (#2321). The+/// `xerj autoindex <folder>` hint is only for a node whose catalog and index+/// list are both empty — a miss against a real corpus is not a reason to+/// tell the operator to index again.pub(crate) fn empty_lookup_report(command: &str,prefix: &str,@@ -400,7 +403,7 @@ pub(crate) fn empty_lookup_report(let prefix_absent = matches!(snap.under_prefix, Some(ref names) if names.is_empty());let mut lines = Vec::new();if prefix_absent {- lines.push(format!("no indices named {pattern}"));+ lines.push(missing_prefix_line(&pattern, snap));} else if command == "def" {lines.push(format!("no definition in {pattern} for: {query}"));lines.push(format!(@@ -427,15 +430,21 @@ pub(crate) fn empty_lookup_report(if catalog_empty {lines.push("(index a relevant repo first: xerj autoindex <folder>)".to_string());} else if let Some(prefixes) = snap.catalog_prefixes.as_ref().filter(|p| !p.is_empty()) {- lines.push(format!("prefixes on this node: {}", prefixes.join(", ")));+ // The missing-prefix headline already lists these. A real miss (the+ // prefix exists, the query did not match) still needs the line.+ if !prefix_absent {+ lines.push(format!("prefixes on this node: {}", prefixes.join(", ")));+ }if let Some(other) = prefixes.iter().find(|p| p.as_str() != prefix) {lines.push(format!("try: xerj {command} --prefix {other} \"{query}\""));}} else if let Some(indices) = snap.all_indices.as_ref().filter(|n| !n.is_empty()) {- lines.push(format!(- "indices on this node: {}",- join_capped(indices, 20)- ));+ if !prefix_absent {+ lines.push(format!(+ "indices on this node: {}",+ join_capped(indices, 20)+ ));+ }lines.push("the catalog lists no prefixes; --prefix is the index name before the dataset slug".to_string(),@@ -445,6 +454,24 @@ pub(crate) fn empty_lookup_report((lines, exit)}+/// The refusal line for a prefix that matches nothing.+///+/// Prefixes from the catalog win. Index names are the fallback when the+/// catalog has none (an autoindex that aborted before writing a run or+/// dataset document still left `{prefix}-{slug}` indices).+fn missing_prefix_line(pattern: &str, snap: &CorpusSnapshot) -> String {+ let mut line = format!("no indices named {pattern} — this is not a search result");+ if let Some(prefixes) = snap.catalog_prefixes.as_ref().filter(|p| !p.is_empty()) {+ line.push_str(&format!("; prefixes on this node: {}", prefixes.join(", ")));+ } else if let Some(indices) = snap.all_indices.as_ref().filter(|n| !n.is_empty()) {+ line.push_str(&format!(+ "; indices on this node: {}",+ join_capped(indices, 20)+ ));+ }+ line+}+fn join_capped(items: &[String], cap: usize) -> String {if items.len() <= cap {items.join(", ")@@ -476,19 +503,41 @@ pub(crate) fn corpus_snapshot(es: &Es, prefix: &str) -> CorpusSnapshot {}}-/// Distinct `prefix` values on catalog run documents, sorted. `None` when-/// the catalog cannot be read — that is not the same as an empty catalog.+/// Distinct `prefix` values on catalog run and dataset documents, sorted.+///+/// Run documents are the historical source. Dataset documents carry the same+/// field and are written as indexing proceeds, so a run that aborted before+/// its run document (#2321) still names the prefix. `None` when the catalog+/// cannot be read — that is not the same as an empty catalog. A dataset+/// query that fails leaves the run prefixes in place.fn catalog_run_prefixes(es: &Es) -> Option<Vec<String>> {+ let runs = catalog_kind_body(es, "run")?;+ let mut prefixes = prefixes_from_catalog_body(&runs);+ if let Some(datasets) = catalog_kind_body(es, "dataset") {+ for prefix in prefixes_from_catalog_body(&datasets) {+ if !prefixes.contains(&prefix) {+ prefixes.push(prefix);+ }+ }+ prefixes.sort();+ }+ Some(prefixes)+}++fn catalog_kind_body(es: &Es, doc_kind: &str) -> Option<Value> {let body = json!({"size": 200,- "query": { "term": { "doc_kind": "run" } },+ "query": { "term": { "doc_kind": doc_kind } },"_source": ["prefix"]});- let v = es- .search(xerj_common::AUTOINDEX_CATALOG_INDEX, &body)- .ok()?;+ es.search(xerj_common::AUTOINDEX_CATALOG_INDEX, &body).ok()+}++fn prefixes_from_catalog_body(v: &Value) -> Vec<String> {let mut prefixes = BTreeSet::new();- let hits = v.pointer("/hits/hits").and_then(Value::as_array)?;+ let Some(hits) = v.pointer("/hits/hits").and_then(Value::as_array) else {+ return Vec::new();+ };for hit in hits {if let Some(prefix) = hit.pointer("/_source/prefix").and_then(Value::as_str) {if !prefix.is_empty() {@@ -496,7 +545,7 @@ fn catalog_run_prefixes(es: &Es) -> Option<Vec<String>> {}}}- Some(prefixes.into_iter().collect())+ prefixes.into_iter().collect()}/// What `xerj search` / `xerj def` print when the query request itself fails.@@ -1704,6 +1753,15 @@ mod tests {);assert_eq!(exit, 1);let text = lines.join("\n");+ let headline = lines.first().expect("headline");+ assert!(+ headline.starts_with("no indices named typo-* — this is not a search result"),+ "{headline}"+ );+ assert!(+ headline.contains("prefixes on this node: eng, sled"),+ "the refusal line itself must name the prefixes: {headline}"+ );assert!(text.contains("no indices named typo-*"), "{text}");assert!(text.contains("prefixes on this node: eng, sled"), "{text}");assert!(@@ -1825,6 +1883,11 @@ mod tests {);assert_eq!(exit, 1);let text = lines.join("\n");+ let headline = lines.first().expect("headline");+ assert!(+ headline.contains("indices on this node: sled-docs, eng-logs"),+ "index names stand in for prefixes on the refusal line: {headline}"+ );assert!(text.contains("indices on this node: sled-docs, eng-logs"),"{text}"@@ -1833,6 +1896,23 @@ mod tests {assert!(!text.contains("xerj autoindex"), "{text}");}+ #[test]+ fn catalog_prefix_values_keep_dataset_documents_and_drop_an_empty_prefix() {+ let body = json!({+ "hits": {"hits": [+ {"_source": {"doc_kind": "dataset", "prefix": "repo"}},+ {"_source": {"doc_kind": "run", "prefix": "repo"}},+ {"_source": {"doc_kind": "dataset", "prefix": "other"}},+ {"_source": {"prefix": ""}}+ ]}+ });+ assert_eq!(+ prefixes_from_catalog_body(&body),+ vec!["other".to_string(), "repo".to_string()]+ );+ assert!(prefixes_from_catalog_body(&json!({"hits": {}})).is_empty());+ }+/// #1284: one index mapping `code` as `long` fails a wildcard `multi_match`/// or `match_phrase` on `code` for every index. Drop it. `keyword` `name`/// stays. A prose field that is numeric on any index is dropped too, anddiff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex a729ce7b1..e2b87ff3b 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -108,6 +108,44 @@ fn sample_drain_due(static MEMORY_DRAIN_TX: std::sync::OnceLock<std::sync::mpsc::SyncSender<()>> =std::sync::OnceLock::new();+/// Ask the drain worker to flush, at most once per [`MEMORY_DRAIN_MIN_INTERVAL`].+///+/// The sampler requests a drain on its own clock (`pressure_drain_due`). A+/// write the breaker has just rejected also requests one (#2321): the 429+/// itself does not free the memtable, and waiting for the next sample is how+/// a client that retries immediately spends its patience against a budget+/// nothing is flushing. This clock is separate, so the first rejection is+/// immediate even when the sampler just fired. Later rejections from this+/// path are spaced by the same interval, so a stream of 429s does not queue a+/// drain per bulk. No worker (a library `Engine` that never called+/// [`Engine::spawn_resource_sampler`]) is a no-op; callers that must flush+/// inline use [`Engine::publish_memtables_if_budget_engaged`].+pub(crate) fn request_memory_pressure_drain() {+ if !drain_request_due() {+ return;+ }+ if let Some(tx) = MEMORY_DRAIN_TX.get() {+ let _ = tx.try_send(());+ }+}++fn drain_request_due() -> bool {+ use std::sync::atomic::{AtomicU64, Ordering};+ static ORIGIN: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();+ // `u64::MAX` is "never requested". Elapsed millis from process start do+ // not reach it.+ static LAST_MS: AtomicU64 = AtomicU64::new(u64::MAX);+ let origin = ORIGIN.get_or_init(std::time::Instant::now);+ let now_ms = u64::try_from(origin.elapsed().as_millis()).unwrap_or(u64::MAX);+ let prev = LAST_MS.load(Ordering::Relaxed);+ let window_ms = u64::try_from(MEMORY_DRAIN_MIN_INTERVAL.as_millis()).unwrap_or(2_000);+ if prev != u64::MAX && now_ms.saturating_sub(prev) < window_ms {+ return false;+ }+ LAST_MS.store(now_ms, Ordering::Relaxed);+ true+}+/// Spawn attempts (and the first back-off) for the governor's sampler and/// drain threads. They are started once, at boot; losing one disables the/// parent memory breaker for the life of the process, so a transient@@ -4313,13 +4351,65 @@ impl Engine {/// new way to wedge. Returns the number of indexes drained; closed/// indexes are skipped (their caches were already released at close).pub async fn drain_rebuildable_memory(&self) -> usize {- let loaded: Vec<Arc<Index>> = self+ let loaded = self.open_indices_largest_memtable_first();+ self.release_rebuildable_memory(&loaded).await+ }++ /// Open indices, largest live memtable first (#2321).+ ///+ /// The summed budget stays engaged until enough bytes are retired. DashMap+ /// iteration order can flush a tiny index first and leave the breaker up+ /// for the rest of the walk; the index holding the bytes goes first so+ /// the next sample can admit writes while the smaller ones are still+ /// publishing.+ fn open_indices_largest_memtable_first(&self) -> Vec<Arc<Index>> {+ let mut loaded: Vec<Arc<Index>> = self.indices.iter().filter(|e| !self.closed_indices.contains_key(e.key().as_str())).map(|e| Arc::clone(e.value())).collect();- self.release_rebuildable_memory(&loaded).await+ loaded.sort_by_key(|idx| std::cmp::Reverse(idx.memtable_bytes()));+ loaded+ }++ /// Publish memtables that are holding the process budget, before a search+ /// walks them (#2321).+ ///+ /// A `{prefix}-*` search scans every index's memtable, frozen generations+ /// included. On the 8 GiB auto tier that sum sits at 2 GiB while each+ /// index is still under its own flush line — the same shape that answers+ /// writes with HTTP 429. Scanning it is the multi-minute query a restart+ /// (flush to segments) brought down to seconds. Flush, largest first, and+ /// wait until the charged bytes drop or 30 s pass. Cache release is+ /// intentionally not done here: the search that follows has to read the+ /// segments just published.+ ///+ /// A no-op when the memtable breaker is off, so an ordinary search does+ /// not flush. No governor (an `Index` built directly) is also a no-op.+ pub async fn publish_memtables_if_budget_engaged(&self) -> usize {+ let Some(governor) = crate::governor::global() else {+ return 0;+ };+ if !governor.memtable_breaker_engaged() {+ return 0;+ }+ let loaded = self.open_indices_largest_memtable_first();+ let mut published = 0usize;+ for idx in &loaded {+ if idx.memtable_bytes() > 0 && idx.flush().await.is_ok() {+ published += 1;+ }+ }+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);+ loop {+ governor.refresh_memtable(self.total_memtable_bytes());+ if !governor.memtable_breaker_engaged() || std::time::Instant::now() >= deadline {+ break;+ }+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;+ }+ published}/// Flush then release the rebuildable caches of exactly `indices`, thendiff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 90d1542b8..1a3e12035 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -27204,7 +27204,13 @@ impl Index {if let Some(g) = crate::governor::global() {g.check_disk_block(self.name.as_str()).map_err(EngineError::Common)?;- g.check_ingest_admission().map_err(EngineError::Common)?;+ if let Err(error) = g.check_ingest_admission() {+ // The rejection does not free the memtable or the RSS that+ // tripped it. Request the drain on this 429, not only on the+ // sampler's next tick (#2321, #948).+ crate::engine::request_memory_pressure_drain();+ return Err(EngineError::Common(error));+ }}Ok(())}diff --git a/engine/crates/xerj-engine/tests/memtable_budget_resume_and_search.rs b/engine/crates/xerj-engine/tests/memtable_budget_resume_and_search.rsnew file mode 100644index 000000000..4cc17028f--- /dev/null+++ b/engine/crates/xerj-engine/tests/memtable_budget_resume_and_search.rs@@ -0,0 +1,198 @@+//! Issue #2321: a summed memtable at the process budget must flush and then+//! accept writes, and a search after that must not scan the full buffer.+//!+//! rc.98 rejected those writes with HTTP 429 and did not flush, so a client+//! that retried crawled and then gave up. Searching the unflushed memtable+//! took about a minute; a restart (flush to segments) did not.+//!+//! This binary owns the process governor. One test, so a second engine cannot+//! install a different budget first.++use std::sync::Arc;+use std::time::{Duration, Instant};++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::governor;+use xerj_engine::Engine;+use xerj_query::parse_request;++const BUDGET: u64 = 1024 * 1024;++fn make_engine(dir: &TempDir) -> Arc<Engine> {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ // 1 MiB stands in for the auto tier's 2 GiB. RSS watermark off, so only+ // the memtable latch can reject. Each index's own flush line is far+ // above what this test writes.+ config.limits.max_total_memtable_mb = 1;+ config.limits.memory_watermark_percent = 0;+ config.storage.flush_size_mb = 64;+ config.storage.flush_idle_secs = 0;+ config.storage.flush_interval_secs = 3600;+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ Arc::new(Engine::new(config).expect("engine"))+}++fn text_schema() -> Schema {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("body", FieldType::Text));+ schema+}++async fn fill_until_budget(engine: &Engine, names: &[&str], body: &str, tag: &str) -> usize {+ let mut accepted = 0usize;+ for n in 0..800 {+ let name = names[n % names.len()];+ engine+ .get_index(name)+ .unwrap()+ .index_document(Some(format!("{tag}-{name}-{n}")), json!({ "body": body }))+ .await+ .unwrap_or_else(|error| panic!("{tag} write {n} before the budget: {error}"));+ accepted += 1;+ if engine.total_memtable_bytes() >= BUDGET {+ break;+ }+ }+ accepted+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn writes_resume_after_the_memtable_budget_fills_and_search_is_bounded() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ // The sampler stays off during the fill. Thread B would otherwise flush+ // as soon as the sum crossed the budget, and the rejection below would+ // depend on winning that race.+ let names = ["a", "b", "c", "d"];+ for name in names {+ engine+ .create_index(name, text_schema())+ .expect("create index");+ }+ let body = format!("budgettoken {}", "a".repeat(4096));+ let accepted = fill_until_budget(&engine, &names, &body, "fill").await;+ let summed = engine.total_memtable_bytes();+ assert!(+ summed >= BUDGET,+ "four indices must cross the 1 MiB budget (accepted {accepted}, summed {summed})"+ );+ for name in names {+ let idx = engine.get_index(name).unwrap();+ assert!(+ idx.memtable_bytes() < idx.flush_threshold(),+ "index {name} must still be under its own flush line"+ );+ }++ let gov = governor::global().expect("governor");+ gov.refresh_memtable(summed);+ assert!(+ gov.memtable_breaker_engaged(),+ "crossing the budget must engage the memtable breaker"+ );+ let rejected = engine+ .get_index("a")+ .unwrap()+ .index_document(Some("rejected".into()), json!({ "body": "no" }))+ .await+ .expect_err("a write at the budget must be rejected");+ let rejected = rejected.to_string();+ assert!(rejected.contains("memtable byte budget"), "{rejected}");++ // What `search_impl` does before it scans: publish while the breaker is+ // on. No sampler yet, so this is the only flush that can free the budget.+ let published = engine.publish_memtables_if_budget_engaged().await;+ assert!(published > 0, "the engaged breaker must flush a memtable");+ gov.refresh_memtable(engine.total_memtable_bytes());+ assert!(+ engine.total_memtable_bytes() < BUDGET,+ "publishing must leave the summed memtable under the budget"+ );+ assert!(!gov.memtable_breaker_engaged());++ let infos = engine.list_indices().await;+ let segments: usize = infos.iter().map(|info| info.segment_count).sum();+ assert!(+ segments > 0,+ "the pressure flush must publish a segment: {infos:?}"+ );++ let started = Instant::now();+ let req = parse_request(&json!({+ "query": { "match": { "body": "budgettoken" } },+ "size": 5,+ "from": 0+ }))+ .expect("parse");+ let mut hits = 0u64;+ for name in names {+ let page = engine+ .get_index(name)+ .unwrap()+ .search(&req)+ .await+ .expect("search");+ hits += page.total.value;+ }+ let elapsed = started.elapsed();+ assert!(+ elapsed < Duration::from_secs(5),+ "search after the flush took {elapsed:?} (the unflushed-memtable stall)"+ );+ assert!(hits > 0, "budgettoken must be searchable after the flush");+ assert!(+ hits as usize <= accepted,+ "search total {hits} exceeds accepted writes {accepted}"+ );++ // Second crossing, with the sampler running. Writes are rejected until+ // its drain publishes; the test does not call flush itself.+ engine.spawn_resource_sampler();+ let mut resumed = 0usize;+ let deadline = Instant::now() + Duration::from_secs(30);+ while resumed < accepted && Instant::now() < deadline {+ let name = names[resumed % names.len()];+ match engine+ .get_index(name)+ .unwrap()+ .index_document(+ Some(format!("again-{name}-{resumed}")),+ json!({ "body": body }),+ )+ .await+ {+ Ok(_) => resumed += 1,+ Err(error) => {+ let rendered = error.to_string();+ assert!(+ rendered.contains("memtable byte budget"),+ "while waiting for the sampler drain: {rendered}"+ );+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ }+ }+ assert!(+ resumed == accepted,+ "the sampler drain must admit another {accepted} writes (got {resumed}, memtable {} bytes)",+ engine.total_memtable_bytes()+ );+ // The last admitted writes can sit over the budget until the sampler's+ // next drain. That interval is 2 s; give it a few windows.+ let settle = Instant::now() + Duration::from_secs(10);+ while engine.total_memtable_bytes() >= BUDGET && Instant::now() < settle {+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ gov.refresh_memtable(engine.total_memtable_bytes());+ assert!(+ engine.total_memtable_bytes() < BUDGET,+ "the sampler drain must leave the summed memtable under the budget"+ );+ assert!(!gov.memtable_breaker_engaged());+}