Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): flush a full memtable budget instead of stalling autoindex (#2325)

xerj-org78752dec7a05
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8cbcebc5a..0ce591c85 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -85,6 +85,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- A full process memtable budget now flushes instead of leaving `xerj autoindex`
+ on HTTP 429 until it gives up. The rejection requests a drain (largest
+ memtable first) rather than waiting for the next memory sample, and a
+ search issued while that breaker is engaged publishes those memtables
+ before it scans them. `xerj search` / `xerj def` saying `no indices named
+ …` names the prefixes on the node in that same line, including a prefix
+ recorded only on a catalog dataset document
+ ([#2321](https://github.com/xerj-org/xerj/issues/2321)).
+
- A multi-term question no longer ranks a one-word symbol definition
(`type Token`) above a hit that covers more of the question. `xerj search`,
`xerj def`, `xerj code`, and the plain-string `xerj_search` tool add a
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index cde14221a..4dddc4284 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -13088,6 +13088,35 @@ pub(crate) async fn search_shard_local(
.await
}
+/// Heap-allocate the publish future before it is polled.
+///
+/// `publish_memtables_if_budget_engaged` awaits `Index::flush`. That future
+/// is large enough that building it inside `search_impl` overflows the worker
+/// stack (#2321). A sync function creates it here, on the task that is about
+/// to poll it, and only the box escapes.
+#[inline(never)]
+fn boxed_memtable_publish(
+ engine: std::sync::Arc<xerj_engine::Engine>,
+) -> std::pin::Pin<Box<dyn std::future::Future<Output = usize> + Send>> {
+ Box::pin(async move { engine.publish_memtables_if_budget_engaged().await })
+}
+
+/// Publish memtables that are holding the process budget, on a fresh task.
+///
+/// A no-op — and no spawn — when the memtable breaker is off, so an ordinary
+/// search does not flush. The spawn keeps `Index::flush`'s frame off
+/// `search_impl`'s stack.
+async fn publish_engaged_memtables(engine: std::sync::Arc<xerj_engine::Engine>) {
+ if !xerj_engine::governor::global().is_some_and(|governor| governor.memtable_breaker_engaged())
+ {
+ return;
+ }
+ let _ = tokio::spawn(async move {
+ let _published = boxed_memtable_publish(engine).await;
+ })
+ .await;
+}
+
async fn search_impl(
state: AppState,
index: String,
@@ -13099,6 +13128,10 @@ async fn search_impl(
// Read once, before the first `.await`: what the in-process hub caller asked
// of this search. A request off the wire gets the default and is unaffected.
let in_process = in_process_search();
+ // #2321: a memtable sitting on the process budget is what a `{prefix}-*`
+ // search scans. Publish it before this request walks it. No-op unless
+ // that breaker is engaged. Read `in_process` first — it is task-local.
+ publish_engaged_memtables(std::sync::Arc::clone(&state.engine)).await;
// Closed-index handling is deferred to after index resolution below so
// `ignore_unavailable` / `expand_wildcards` are honored per ES semantics.
diff --git a/engine/crates/xerj-api/tests/memtable_budget_autoindex.rs b/engine/crates/xerj-api/tests/memtable_budget_autoindex.rs
new file mode 100644
index 000000000..8c33ca417
--- /dev/null
+++ b/engine/crates/xerj-api/tests/memtable_budget_autoindex.rs
@@ -0,0 +1,231 @@
+//! Issue #2321: `xerj autoindex` against a node whose memtable budget fills
+//! must finish (the server flushes, the client backs off and resumes), and a
+//! search after that run must stay fast.
+//!
+//! Scaled down from the report: 1 MiB process budget and a flush line the
+//! corpus never reaches, so the only flush that can free the budget is the
+//! pressure drain. This file is its own binary so its governor is the first
+//! one installed.
+
+use std::path::Path;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use tower::ServiceExt;
+use xerj_autoindex::cli::IndexCfg;
+use xerj_engine::Engine;
+
+const BUDGET: u64 = 1024 * 1024;
+
+fn boot() -> (axum::Router, Arc<Engine>, TempDir) {
+ let dir = tempfile::tempdir().expect("data dir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.auth.enabled = false;
+ config.limits.max_total_memtable_mb = 1;
+ config.limits.memory_watermark_percent = 0;
+ config.storage.flush_size_mb = 64;
+ config.storage.flush_idle_secs = 0;
+ config.storage.flush_interval_secs = 3600;
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ state.engine.spawn_resource_sampler();
+ let engine = Arc::clone(&state.engine);
+ let app = xerj_api::router::build_es_compat_router(state);
+ (app, engine, dir)
+}
+
+fn index_cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
+ IndexCfg {
+ root: root.to_owned(),
+ endpoint_url: None,
+ stub_globs: Vec::new(),
+ url: url.to_owned(),
+ api_key: None,
+ api_key_file: None,
+ workers: 2,
+ scan_workers: 1,
+ pdf_workers: 1,
+ resource_notes: Vec::new(),
+ xerj_url_note: None,
+ pdf_timeout_secs: 30,
+ bulk_mb: 1,
+ bulk_timeout_secs: 60,
+ snapshot_max_bytes: 64 << 30,
+ prefix: "budget".to_owned(),
+ replace_prefix: false,
+ prefix_note: None,
+ state_dir: Some(state_dir.to_owned()),
+ fresh: false,
+ follow_symlinks: false,
+ follow_symlinks_outside_root: false,
+ ignore: xerj_autoindex::ignore_rules::IgnoreOptions::default(),
+ max_file_bytes: 1 << 30,
+ sample: 50,
+ no_semantic: true,
+ code_analyzer: Default::default(),
+ brain: None,
+ no_graph: true,
+ max_minutes: 0,
+ approve: None,
+ max_index_bytes: None,
+ budget_plan: None,
+ drop_indexed: false,
+ dry_run: false,
+ walk_only: false,
+ json: false,
+ quiet: true,
+ progress: xerj_autoindex::progress::ProgressMode::None,
+ progress_interval: None,
+ watch: false,
+ label: None,
+ debounce: Duration::from_millis(0),
+ exclude_rels: Vec::new(),
+ skip_corpus_data: false,
+ }
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+ .expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ let parsed = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, parsed)
+}
+
+fn corpus(root: &Path) {
+ let pad = "a".repeat(3000);
+ let mut file = String::new();
+ for i in 0..600 {
+ let line = serde_json::to_string(&json!({
+ "body": format!("budgettoken {i} {pad}")
+ }))
+ .unwrap();
+ file.push_str(&line);
+ file.push('\n');
+ }
+ std::fs::write(root.join("records.jsonl"), file).unwrap();
+}
+
+#[test]
+fn autoindex_completes_when_the_memtable_budget_fills_and_search_stays_fast() {
+ // `#[tokio::test]` workers use tokio's 2 MiB stack. The production runtime
+ // is 4 MiB (`xerj-server`'s `RT_THREAD_STACK_SIZE`) because the ES-compat
+ // handlers do not fit in 2 MiB in a debug build. Match production.
+ let runtime = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(4)
+ .thread_stack_size(4 * 1024 * 1024)
+ .enable_all()
+ .build()
+ .expect("runtime");
+ runtime.block_on(autoindex_budget_run());
+}
+
+async fn autoindex_budget_run() {
+ let (app, engine, _dir) = boot();
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let served = app.clone();
+ tokio::spawn(async move {
+ axum::serve(listener, served).await.unwrap();
+ });
+ let url = format!("http://{addr}");
+
+ let root = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ corpus(root.path());
+
+ let peak = Arc::new(AtomicU64::new(0));
+ let peak_engine = Arc::clone(&engine);
+ let peak_slot = Arc::clone(&peak);
+ let poller = tokio::spawn(async move {
+ loop {
+ let bytes = peak_engine.total_memtable_bytes();
+ peak_slot.fetch_max(bytes, Ordering::Relaxed);
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ });
+
+ let root_path = root.path().to_owned();
+ let state_path = state_dir.path().to_owned();
+ let run = tokio::task::spawn_blocking(move || {
+ xerj_autoindex::run_index_report(index_cfg(&root_path, &state_path, &url))
+ });
+ let (code, report) = tokio::time::timeout(Duration::from_secs(120), run)
+ .await
+ .expect("autoindex did not finish within 120s — the memtable-budget stall")
+ .expect("autoindex task")
+ .expect("autoindex report");
+ poller.abort();
+ assert!(
+ code == 0 || code == 3,
+ "autoindex exit {code}, report {report:?}"
+ );
+
+ let peaked = peak.load(Ordering::Relaxed);
+ assert!(
+ peaked >= BUDGET,
+ "the run must fill the 1 MiB memtable budget (peak {peaked} bytes). \
+ A run that never reaches it does not exercise the drain."
+ );
+
+ let held = engine.total_memtable_bytes();
+ assert!(
+ held < BUDGET,
+ "autoindex returned with the memtable still over budget ({held} bytes) — \
+ the drain did not free it"
+ );
+ let infos = engine.list_indices().await;
+ let segments: usize = infos
+ .iter()
+ .filter(|info| info.name.starts_with("budget"))
+ .map(|info| info.segment_count)
+ .sum();
+ assert!(
+ segments > 0,
+ "the pressure flush must publish a segment under the budget prefix: {infos:?}"
+ );
+
+ let started = Instant::now();
+ let (status, body) = call(
+ &app,
+ "POST",
+ "/budget-*/_search",
+ json!({
+ "size": 3,
+ "query": { "match": { "body": "budgettoken" } }
+ }),
+ )
+ .await;
+ let elapsed = started.elapsed();
+ assert_eq!(status, StatusCode::OK, "search {body}");
+ assert!(
+ elapsed < Duration::from_secs(5),
+ "search after autoindex took {elapsed:?}"
+ );
+ let total = body["hits"]["total"]["value"].as_u64().unwrap_or(0);
+ assert!(
+ total > 0,
+ "budgettoken must be searchable after the run: {body}"
+ );
+}
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rs
index a177fc4d0..51b9b6ad3 100644
--- a/engine/crates/xerj-autoindex/src/search.rs
+++ b/engine/crates/xerj-autoindex/src/search.rs
@@ -386,9 +386,12 @@ pub(crate) struct CorpusSnapshot {
/// `xerj def` has no hits.
///
/// A prefix that matches no index exits `1` and names the prefixes already
-/// on the node. The `xerj autoindex <folder>` hint is only for a node whose
-/// catalog and index list are both empty — a miss against a real corpus is
-/// not a reason to tell the operator to index again.
+/// on the node, in that same line. The line says it is not a search result:
+/// it returns before any query runs, and a one-line reading of "no indices
+/// named …" was taken for an instant empty search (#2321). The
+/// `xerj autoindex <folder>` hint is only for a node whose catalog and index
+/// list are both empty — a miss against a real corpus is not a reason to
+/// tell the operator to index again.
pub(crate) fn empty_lookup_report(
command: &str,
prefix: &str,
@@ -400,7 +403,7 @@ pub(crate) fn empty_lookup_report(
let prefix_absent = matches!(snap.under_prefix, Some(ref names) if names.is_empty());
let mut lines = Vec::new();
if prefix_absent {
- lines.push(format!("no indices named {pattern}"));
+ lines.push(missing_prefix_line(&pattern, snap));
} else if command == "def" {
lines.push(format!("no definition in {pattern} for: {query}"));
lines.push(format!(
@@ -427,15 +430,21 @@ pub(crate) fn empty_lookup_report(
if catalog_empty {
lines.push("(index a relevant repo first: xerj autoindex <folder>)".to_string());
} else if let Some(prefixes) = snap.catalog_prefixes.as_ref().filter(|p| !p.is_empty()) {
- lines.push(format!("prefixes on this node: {}", prefixes.join(", ")));
+ // The missing-prefix headline already lists these. A real miss (the
+ // prefix exists, the query did not match) still needs the line.
+ if !prefix_absent {
+ lines.push(format!("prefixes on this node: {}", prefixes.join(", ")));
+ }
if let Some(other) = prefixes.iter().find(|p| p.as_str() != prefix) {
lines.push(format!("try: xerj {command} --prefix {other} \"{query}\""));
}
} else if let Some(indices) = snap.all_indices.as_ref().filter(|n| !n.is_empty()) {
- lines.push(format!(
- "indices on this node: {}",
- join_capped(indices, 20)
- ));
+ if !prefix_absent {
+ lines.push(format!(
+ "indices on this node: {}",
+ join_capped(indices, 20)
+ ));
+ }
lines.push(
"the catalog lists no prefixes; --prefix is the index name before the dataset slug"
.to_string(),
@@ -445,6 +454,24 @@ pub(crate) fn empty_lookup_report(
(lines, exit)
}
+/// The refusal line for a prefix that matches nothing.
+///
+/// Prefixes from the catalog win. Index names are the fallback when the
+/// catalog has none (an autoindex that aborted before writing a run or
+/// dataset document still left `{prefix}-{slug}` indices).
+fn missing_prefix_line(pattern: &str, snap: &CorpusSnapshot) -> String {
+ let mut line = format!("no indices named {pattern} — this is not a search result");
+ if let Some(prefixes) = snap.catalog_prefixes.as_ref().filter(|p| !p.is_empty()) {
+ line.push_str(&format!("; prefixes on this node: {}", prefixes.join(", ")));
+ } else if let Some(indices) = snap.all_indices.as_ref().filter(|n| !n.is_empty()) {
+ line.push_str(&format!(
+ "; indices on this node: {}",
+ join_capped(indices, 20)
+ ));
+ }
+ line
+}
+
fn join_capped(items: &[String], cap: usize) -> String {
if items.len() <= cap {
items.join(", ")
@@ -476,19 +503,41 @@ pub(crate) fn corpus_snapshot(es: &Es, prefix: &str) -> CorpusSnapshot {
}
}
-/// Distinct `prefix` values on catalog run documents, sorted. `None` when
-/// the catalog cannot be read — that is not the same as an empty catalog.
+/// Distinct `prefix` values on catalog run and dataset documents, sorted.
+///
+/// Run documents are the historical source. Dataset documents carry the same
+/// field and are written as indexing proceeds, so a run that aborted before
+/// its run document (#2321) still names the prefix. `None` when the catalog
+/// cannot be read — that is not the same as an empty catalog. A dataset
+/// query that fails leaves the run prefixes in place.
fn catalog_run_prefixes(es: &Es) -> Option<Vec<String>> {
+ let runs = catalog_kind_body(es, "run")?;
+ let mut prefixes = prefixes_from_catalog_body(&runs);
+ if let Some(datasets) = catalog_kind_body(es, "dataset") {
+ for prefix in prefixes_from_catalog_body(&datasets) {
+ if !prefixes.contains(&prefix) {
+ prefixes.push(prefix);
+ }
+ }
+ prefixes.sort();
+ }
+ Some(prefixes)
+}
+
+fn catalog_kind_body(es: &Es, doc_kind: &str) -> Option<Value> {
let body = json!({
"size": 200,
- "query": { "term": { "doc_kind": "run" } },
+ "query": { "term": { "doc_kind": doc_kind } },
"_source": ["prefix"]
});
- let v = es
- .search(xerj_common::AUTOINDEX_CATALOG_INDEX, &body)
- .ok()?;
+ es.search(xerj_common::AUTOINDEX_CATALOG_INDEX, &body).ok()
+}
+
+fn prefixes_from_catalog_body(v: &Value) -> Vec<String> {
let mut prefixes = BTreeSet::new();
- let hits = v.pointer("/hits/hits").and_then(Value::as_array)?;
+ let Some(hits) = v.pointer("/hits/hits").and_then(Value::as_array) else {
+ return Vec::new();
+ };
for hit in hits {
if let Some(prefix) = hit.pointer("/_source/prefix").and_then(Value::as_str) {
if !prefix.is_empty() {
@@ -496,7 +545,7 @@ fn catalog_run_prefixes(es: &Es) -> Option<Vec<String>> {
}
}
}
- Some(prefixes.into_iter().collect())
+ prefixes.into_iter().collect()
}
/// What `xerj search` / `xerj def` print when the query request itself fails.
@@ -1704,6 +1753,15 @@ mod tests {
);
assert_eq!(exit, 1);
let text = lines.join("\n");
+ let headline = lines.first().expect("headline");
+ assert!(
+ headline.starts_with("no indices named typo-* — this is not a search result"),
+ "{headline}"
+ );
+ assert!(
+ headline.contains("prefixes on this node: eng, sled"),
+ "the refusal line itself must name the prefixes: {headline}"
+ );
assert!(text.contains("no indices named typo-*"), "{text}");
assert!(text.contains("prefixes on this node: eng, sled"), "{text}");
assert!(
@@ -1825,6 +1883,11 @@ mod tests {
);
assert_eq!(exit, 1);
let text = lines.join("\n");
+ let headline = lines.first().expect("headline");
+ assert!(
+ headline.contains("indices on this node: sled-docs, eng-logs"),
+ "index names stand in for prefixes on the refusal line: {headline}"
+ );
assert!(
text.contains("indices on this node: sled-docs, eng-logs"),
"{text}"
@@ -1833,6 +1896,23 @@ mod tests {
assert!(!text.contains("xerj autoindex"), "{text}");
}
+ #[test]
+ fn catalog_prefix_values_keep_dataset_documents_and_drop_an_empty_prefix() {
+ let body = json!({
+ "hits": {"hits": [
+ {"_source": {"doc_kind": "dataset", "prefix": "repo"}},
+ {"_source": {"doc_kind": "run", "prefix": "repo"}},
+ {"_source": {"doc_kind": "dataset", "prefix": "other"}},
+ {"_source": {"prefix": ""}}
+ ]}
+ });
+ assert_eq!(
+ prefixes_from_catalog_body(&body),
+ vec!["other".to_string(), "repo".to_string()]
+ );
+ assert!(prefixes_from_catalog_body(&json!({"hits": {}})).is_empty());
+ }
+
/// #1284: one index mapping `code` as `long` fails a wildcard `multi_match`
/// or `match_phrase` on `code` for every index. Drop it. `keyword` `name`
/// stays. A prose field that is numeric on any index is dropped too, and
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index a729ce7b1..e2b87ff3b 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -108,6 +108,44 @@ fn sample_drain_due(
static MEMORY_DRAIN_TX: std::sync::OnceLock<std::sync::mpsc::SyncSender<()>> =
std::sync::OnceLock::new();
+/// Ask the drain worker to flush, at most once per [`MEMORY_DRAIN_MIN_INTERVAL`].
+///
+/// The sampler requests a drain on its own clock (`pressure_drain_due`). A
+/// write the breaker has just rejected also requests one (#2321): the 429
+/// itself does not free the memtable, and waiting for the next sample is how
+/// a client that retries immediately spends its patience against a budget
+/// nothing is flushing. This clock is separate, so the first rejection is
+/// immediate even when the sampler just fired. Later rejections from this
+/// path are spaced by the same interval, so a stream of 429s does not queue a
+/// drain per bulk. No worker (a library `Engine` that never called
+/// [`Engine::spawn_resource_sampler`]) is a no-op; callers that must flush
+/// inline use [`Engine::publish_memtables_if_budget_engaged`].
+pub(crate) fn request_memory_pressure_drain() {
+ if !drain_request_due() {
+ return;
+ }
+ if let Some(tx) = MEMORY_DRAIN_TX.get() {
+ let _ = tx.try_send(());
+ }
+}
+
+fn drain_request_due() -> bool {
+ use std::sync::atomic::{AtomicU64, Ordering};
+ static ORIGIN: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();
+ // `u64::MAX` is "never requested". Elapsed millis from process start do
+ // not reach it.
+ static LAST_MS: AtomicU64 = AtomicU64::new(u64::MAX);
+ let origin = ORIGIN.get_or_init(std::time::Instant::now);
+ let now_ms = u64::try_from(origin.elapsed().as_millis()).unwrap_or(u64::MAX);
+ let prev = LAST_MS.load(Ordering::Relaxed);
+ let window_ms = u64::try_from(MEMORY_DRAIN_MIN_INTERVAL.as_millis()).unwrap_or(2_000);
+ if prev != u64::MAX && now_ms.saturating_sub(prev) < window_ms {
+ return false;
+ }
+ LAST_MS.store(now_ms, Ordering::Relaxed);
+ true
+}
+
/// Spawn attempts (and the first back-off) for the governor's sampler and
/// drain threads. They are started once, at boot; losing one disables the
/// parent memory breaker for the life of the process, so a transient
@@ -4313,13 +4351,65 @@ impl Engine {
/// new way to wedge. Returns the number of indexes drained; closed
/// indexes are skipped (their caches were already released at close).
pub async fn drain_rebuildable_memory(&self) -> usize {
- let loaded: Vec<Arc<Index>> = self
+ let loaded = self.open_indices_largest_memtable_first();
+ self.release_rebuildable_memory(&loaded).await
+ }
+
+ /// Open indices, largest live memtable first (#2321).
+ ///
+ /// The summed budget stays engaged until enough bytes are retired. DashMap
+ /// iteration order can flush a tiny index first and leave the breaker up
+ /// for the rest of the walk; the index holding the bytes goes first so
+ /// the next sample can admit writes while the smaller ones are still
+ /// publishing.
+ fn open_indices_largest_memtable_first(&self) -> Vec<Arc<Index>> {
+ let mut loaded: Vec<Arc<Index>> = self
.indices
.iter()
.filter(|e| !self.closed_indices.contains_key(e.key().as_str()))
.map(|e| Arc::clone(e.value()))
.collect();
- self.release_rebuildable_memory(&loaded).await
+ loaded.sort_by_key(|idx| std::cmp::Reverse(idx.memtable_bytes()));
+ loaded
+ }
+
+ /// Publish memtables that are holding the process budget, before a search
+ /// walks them (#2321).
+ ///
+ /// A `{prefix}-*` search scans every index's memtable, frozen generations
+ /// included. On the 8 GiB auto tier that sum sits at 2 GiB while each
+ /// index is still under its own flush line — the same shape that answers
+ /// writes with HTTP 429. Scanning it is the multi-minute query a restart
+ /// (flush to segments) brought down to seconds. Flush, largest first, and
+ /// wait until the charged bytes drop or 30 s pass. Cache release is
+ /// intentionally not done here: the search that follows has to read the
+ /// segments just published.
+ ///
+ /// A no-op when the memtable breaker is off, so an ordinary search does
+ /// not flush. No governor (an `Index` built directly) is also a no-op.
+ pub async fn publish_memtables_if_budget_engaged(&self) -> usize {
+ let Some(governor) = crate::governor::global() else {
+ return 0;
+ };
+ if !governor.memtable_breaker_engaged() {
+ return 0;
+ }
+ let loaded = self.open_indices_largest_memtable_first();
+ let mut published = 0usize;
+ for idx in &loaded {
+ if idx.memtable_bytes() > 0 && idx.flush().await.is_ok() {
+ published += 1;
+ }
+ }
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ loop {
+ governor.refresh_memtable(self.total_memtable_bytes());
+ if !governor.memtable_breaker_engaged() || std::time::Instant::now() >= deadline {
+ break;
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;
+ }
+ published
}
/// Flush then release the rebuildable caches of exactly `indices`, then
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 90d1542b8..1a3e12035 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -27204,7 +27204,13 @@ impl Index {
if let Some(g) = crate::governor::global() {
g.check_disk_block(self.name.as_str())
.map_err(EngineError::Common)?;
- g.check_ingest_admission().map_err(EngineError::Common)?;
+ if let Err(error) = g.check_ingest_admission() {
+ // The rejection does not free the memtable or the RSS that
+ // tripped it. Request the drain on this 429, not only on the
+ // sampler's next tick (#2321, #948).
+ crate::engine::request_memory_pressure_drain();
+ return Err(EngineError::Common(error));
+ }
}
Ok(())
}
diff --git a/engine/crates/xerj-engine/tests/memtable_budget_resume_and_search.rs b/engine/crates/xerj-engine/tests/memtable_budget_resume_and_search.rs
new file mode 100644
index 000000000..4cc17028f
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/memtable_budget_resume_and_search.rs
@@ -0,0 +1,198 @@
+//! Issue #2321: a summed memtable at the process budget must flush and then
+//! accept writes, and a search after that must not scan the full buffer.
+//!
+//! rc.98 rejected those writes with HTTP 429 and did not flush, so a client
+//! that retried crawled and then gave up. Searching the unflushed memtable
+//! took about a minute; a restart (flush to segments) did not.
+//!
+//! This binary owns the process governor. One test, so a second engine cannot
+//! install a different budget first.
+
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::governor;
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+const BUDGET: u64 = 1024 * 1024;
+
+fn make_engine(dir: &TempDir) -> Arc<Engine> {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ // 1 MiB stands in for the auto tier's 2 GiB. RSS watermark off, so only
+ // the memtable latch can reject. Each index's own flush line is far
+ // above what this test writes.
+ config.limits.max_total_memtable_mb = 1;
+ config.limits.memory_watermark_percent = 0;
+ config.storage.flush_size_mb = 64;
+ config.storage.flush_idle_secs = 0;
+ config.storage.flush_interval_secs = 3600;
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ Arc::new(Engine::new(config).expect("engine"))
+}
+
+fn text_schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+}
+
+async fn fill_until_budget(engine: &Engine, names: &[&str], body: &str, tag: &str) -> usize {
+ let mut accepted = 0usize;
+ for n in 0..800 {
+ let name = names[n % names.len()];
+ engine
+ .get_index(name)
+ .unwrap()
+ .index_document(Some(format!("{tag}-{name}-{n}")), json!({ "body": body }))
+ .await
+ .unwrap_or_else(|error| panic!("{tag} write {n} before the budget: {error}"));
+ accepted += 1;
+ if engine.total_memtable_bytes() >= BUDGET {
+ break;
+ }
+ }
+ accepted
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn writes_resume_after_the_memtable_budget_fills_and_search_is_bounded() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ // The sampler stays off during the fill. Thread B would otherwise flush
+ // as soon as the sum crossed the budget, and the rejection below would
+ // depend on winning that race.
+ let names = ["a", "b", "c", "d"];
+ for name in names {
+ engine
+ .create_index(name, text_schema())
+ .expect("create index");
+ }
+ let body = format!("budgettoken {}", "a".repeat(4096));
+ let accepted = fill_until_budget(&engine, &names, &body, "fill").await;
+ let summed = engine.total_memtable_bytes();
+ assert!(
+ summed >= BUDGET,
+ "four indices must cross the 1 MiB budget (accepted {accepted}, summed {summed})"
+ );
+ for name in names {
+ let idx = engine.get_index(name).unwrap();
+ assert!(
+ idx.memtable_bytes() < idx.flush_threshold(),
+ "index {name} must still be under its own flush line"
+ );
+ }
+
+ let gov = governor::global().expect("governor");
+ gov.refresh_memtable(summed);
+ assert!(
+ gov.memtable_breaker_engaged(),
+ "crossing the budget must engage the memtable breaker"
+ );
+ let rejected = engine
+ .get_index("a")
+ .unwrap()
+ .index_document(Some("rejected".into()), json!({ "body": "no" }))
+ .await
+ .expect_err("a write at the budget must be rejected");
+ let rejected = rejected.to_string();
+ assert!(rejected.contains("memtable byte budget"), "{rejected}");
+
+ // What `search_impl` does before it scans: publish while the breaker is
+ // on. No sampler yet, so this is the only flush that can free the budget.
+ let published = engine.publish_memtables_if_budget_engaged().await;
+ assert!(published > 0, "the engaged breaker must flush a memtable");
+ gov.refresh_memtable(engine.total_memtable_bytes());
+ assert!(
+ engine.total_memtable_bytes() < BUDGET,
+ "publishing must leave the summed memtable under the budget"
+ );
+ assert!(!gov.memtable_breaker_engaged());
+
+ let infos = engine.list_indices().await;
+ let segments: usize = infos.iter().map(|info| info.segment_count).sum();
+ assert!(
+ segments > 0,
+ "the pressure flush must publish a segment: {infos:?}"
+ );
+
+ let started = Instant::now();
+ let req = parse_request(&json!({
+ "query": { "match": { "body": "budgettoken" } },
+ "size": 5,
+ "from": 0
+ }))
+ .expect("parse");
+ let mut hits = 0u64;
+ for name in names {
+ let page = engine
+ .get_index(name)
+ .unwrap()
+ .search(&req)
+ .await
+ .expect("search");
+ hits += page.total.value;
+ }
+ let elapsed = started.elapsed();
+ assert!(
+ elapsed < Duration::from_secs(5),
+ "search after the flush took {elapsed:?} (the unflushed-memtable stall)"
+ );
+ assert!(hits > 0, "budgettoken must be searchable after the flush");
+ assert!(
+ hits as usize <= accepted,
+ "search total {hits} exceeds accepted writes {accepted}"
+ );
+
+ // Second crossing, with the sampler running. Writes are rejected until
+ // its drain publishes; the test does not call flush itself.
+ engine.spawn_resource_sampler();
+ let mut resumed = 0usize;
+ let deadline = Instant::now() + Duration::from_secs(30);
+ while resumed < accepted && Instant::now() < deadline {
+ let name = names[resumed % names.len()];
+ match engine
+ .get_index(name)
+ .unwrap()
+ .index_document(
+ Some(format!("again-{name}-{resumed}")),
+ json!({ "body": body }),
+ )
+ .await
+ {
+ Ok(_) => resumed += 1,
+ Err(error) => {
+ let rendered = error.to_string();
+ assert!(
+ rendered.contains("memtable byte budget"),
+ "while waiting for the sampler drain: {rendered}"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ }
+ }
+ assert!(
+ resumed == accepted,
+ "the sampler drain must admit another {accepted} writes (got {resumed}, memtable {} bytes)",
+ engine.total_memtable_bytes()
+ );
+ // The last admitted writes can sit over the budget until the sampler's
+ // next drain. That interval is 2 s; give it a few windows.
+ let settle = Instant::now() + Duration::from_secs(10);
+ while engine.total_memtable_bytes() >= BUDGET && Instant::now() < settle {
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ gov.refresh_memtable(engine.total_memtable_bytes());
+ assert!(
+ engine.total_memtable_bytes() < BUDGET,
+ "the sampler drain must leave the summed memtable under the budget"
+ );
+ assert!(!gov.memtable_breaker_engaged());
+}