Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): finalize-count reads time bounds by sort, not min/max aggs (#2750)

xerj-orgd3e2eddcb39f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce9ce2f21..8ba9d4fbb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -97,6 +97,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
longer take a node from below the watermark into the parent breaker band and
turn ingest into HTTP 429s. The relief for a search node resting between the
merge-pool ceiling and the watermark is kept (#2427).
+- `xerj autoindex` no longer stalls for up to 10 minutes per large dataset in
+ `finalize-count`. The dataset time range was read with `min`/`max`
+ aggregations, which make the node copy every document first; on a
+ 393 k-record dataset that priced at 768 MB against the 512 MB
+ `limits.max_query_memory_mb` default and the client re-asked for its full
+ 600 s patience. The bounds now come from two sorted size-1 searches, and a
+ 429 from the per-query memory guard is no longer retried, since the same
+ query is refused again (#2427).
- The `xerj-autoindex` corpus-lock tests no longer fail at random in CI
(#2520, Refs #2427). One test put `-c core.fsmonitor=false` after the git
subcommand, so git exited 129 at once and the test only passed by winning a
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 874afa7dd..dc77d4402 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -640,6 +640,13 @@ pub const THROTTLE_PATIENCE: Duration = Duration::from_secs(600);
/// tier's 2 GiB memtable budget, and short of a multi-minute spin.
pub const ZERO_PROGRESS_PATIENCE: Duration = Duration::from_secs(90);
+/// The engine's per-query guard (`limits.max_query_memory_mb`) refusing a
+/// request whose estimated allocation exceeds the budget. Unlike the parent
+/// breaker this is a property of the query and the data, not of the moment.
+pub(crate) fn is_query_allocation_refusal(reason: &str) -> bool {
+ reason.contains("would exceed limits.max_query_memory_mb")
+}
+
/// The parent memory breaker, as opposed to a disk flood-stage 429 or a
/// thread-pool rejection. Both of its reasons carry one of these phrases.
pub(crate) fn is_parent_memory_breaker(reason: &str) -> bool {
@@ -1528,7 +1535,19 @@ impl Es {
) -> Attempt<T> {
let status = resp.status();
if status.as_u16() == 429 {
- return Attempt::Throttled(server_reason(resp));
+ let reason = server_reason(resp);
+ if let Some(reason) = reason.as_deref().filter(|r| is_query_allocation_refusal(r)) {
+ // The per-query memory guard prices THIS request against a
+ // fixed budget. Nothing a wait can change makes the same query
+ // over the same documents cheaper, so retrying only stalls the
+ // caller for the whole throttle patience (#2427: 600 s per
+ // dataset in finalize-count).
+ return Attempt::Abort(anyhow!(
+ "{what}: HTTP 429 Too Many Requests: {reason} (not retried: the per-query \
+ memory guard refuses the same query again)"
+ ));
+ }
+ return Attempt::Throttled(reason);
}
if status.is_server_error() {
return Attempt::Failed(match server_reason(resp) {
@@ -4774,6 +4793,59 @@ mod tests {
);
}
+ /// #2427: a 429 from the per-query memory guard is deterministic for the
+ /// same query, so even a loading client (admission on, long patience)
+ /// gives up after the first answer instead of re-asking until the
+ /// patience runs out.
+ #[test]
+ fn a_query_allocation_429_is_not_retried() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let hits = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let server_hits = Arc::clone(&hits);
+ let stop = Arc::new(AtomicBool::new(false));
+ let server_stop = Arc::clone(&stop);
+ listener.set_nonblocking(true).unwrap();
+ let server = std::thread::spawn(move || loop {
+ match listener.accept() {
+ Ok((mut stream, _)) => {
+ stream.set_nonblocking(false).unwrap();
+ let _ = read_request(&mut stream);
+ server_hits.fetch_add(1, Ordering::SeqCst);
+ let body = br#"{"error":{"type":"circuit_breaking_exception","reason":"[request] query allocation (768.2mb) would exceed limits.max_query_memory_mb=512MB at [aggregation corpus materialisation]; reduce size/aggregation cardinality"},"status":429}"#;
+ respond_status(&mut stream, "429 Too Many Requests", body);
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ if server_stop.load(Ordering::SeqCst) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(2));
+ }
+ Err(error) => panic!("accept: {error}"),
+ }
+ });
+ let es = client(address, 4)
+ .with_throttle_patience(Duration::from_secs(2))
+ .with_zero_progress_patience(Duration::from_secs(2));
+ let started = Instant::now();
+ let error = es
+ .search(
+ "i",
+ &serde_json::json!({"size": 0, "aggs": {"mn": {"min": {"field": "t"}}}}),
+ )
+ .unwrap_err();
+ let elapsed = started.elapsed();
+ stop.store(true, Ordering::SeqCst);
+ server.join().unwrap();
+ let rendered = format!("{error:#}");
+ assert!(rendered.contains("max_query_memory_mb"), "{rendered}");
+ assert_eq!(hits.load(Ordering::SeqCst), 1, "asked once: {rendered}");
+ assert!(
+ elapsed < Duration::from_millis(500),
+ "a query-allocation refusal must not wait out the patience, took {elapsed:?}"
+ );
+ }
+
// ── #955: a request is bounded in actions and bytes, and a request the
// server still calls too large is halved instead of ending the run ──
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 777325019..2e1cd301a 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -8655,6 +8655,52 @@ fn catalog_runs(es: &Es) -> Result<Vec<Value>> {
})
.unwrap_or_default())
}
+/// A dataset's time bounds from two size-1 searches sorted on its time field.
+///
+/// #2427: this used to be one `size:0` search with `min`/`max` aggs. Neither
+/// has a columnar fast path, so the node deep-cloned every document into
+/// owned values first ("aggregation corpus materialisation", ~2 KB a doc
+/// against `limits.max_query_memory_mb`). A 393 k-record dataset priced at
+/// 768 MB against the 512 MB default, the node answered 429, and the client
+/// re-asked for its full 600 s patience before the run moved on without the
+/// bounds. A sort reads doc values and clones nothing — the same shape the
+/// generation read-back has used since #1183.
+fn dataset_time_range(
+ es: &Es,
+ index: &str,
+ field: &str,
+) -> Result<(Option<String>, Option<String>)> {
+ let bound = |order: &str| -> Result<Option<String>> {
+ let body = json!({
+ "size": 1,
+ "query": {"exists": {"field": field}},
+ "sort": [{field: {"order": order}}],
+ "_source": [field]
+ });
+ let v = es.search(index, &body)?;
+ let hit = match v.pointer("/hits/hits/0") {
+ Some(hit) => hit,
+ None => return Ok(None),
+ };
+ // The stored value first, exactly like the generation read-back
+ // (`sync_executor::extreme_time`); the sort key (epoch ms for a
+ // date) only when `_source` lacks the field.
+ let from_source = hit
+ .pointer("/_source")
+ .and_then(|src| src.get(field))
+ .and_then(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => n.as_i64().map(xerj_common::schema::epoch_ms_to_iso8601_utc),
+ _ => None,
+ });
+ Ok(from_source.or_else(|| {
+ hit.pointer("/sort/0")
+ .and_then(Value::as_i64)
+ .map(xerj_common::schema::epoch_ms_to_iso8601_utc)
+ }))
+ };
+ Ok((bound("asc")?, bound("desc")?))
+}
fn run_index_report_inner(
mut cfg: IndexCfg,
@@ -12090,22 +12136,8 @@ fn run_index_report_inner(
}
}
if let Some(t) = &d.time_field {
- let body = json!({"size":0,"aggs":{
- "mn":{"min":{"field":t}},"mx":{"max":{"field":t}}}});
- if let Ok(v) = es.search(&d.index, &body) {
- let get = |k: &str| -> Option<String> {
- let a = v.pointer(&format!("/aggregations/{k}"))?;
- a.get("value_as_string")
- .and_then(|s| s.as_str())
- .map(|s| s.to_string())
- .or_else(|| {
- a.get("value").and_then(|f| f.as_f64()).and_then(|ms| {
- chrono::DateTime::from_timestamp_millis(ms as i64)
- .map(|d| infer::dates::to_rfc3339_millis(&d))
- })
- })
- };
- ds_timerange.insert(d.slug.clone(), (get("mn"), get("mx")));
+ if let Ok(range) = dataset_time_range(&es, &d.index, t) {
+ ds_timerange.insert(d.slug.clone(), range);
}
}
}
@@ -17357,3 +17389,91 @@ mod map_folder_preview {
);
}
}
+
+#[cfg(test)]
+mod dataset_time_range_tests {
+ use super::*;
+ use std::io::Read;
+ use std::net::TcpListener;
+ use std::sync::Arc;
+
+ const T_MIN: i64 = 1_790_812_800_000; // 2026-10-01T00:00:00Z
+ const T_MAX: i64 = 1_790_985_600_000; // 2026-10-03T00:00:00Z
+
+ fn iso(ms: i64) -> String {
+ xerj_common::schema::epoch_ms_to_iso8601_utc(ms)
+ }
+
+ fn read_body(stream: &mut std::net::TcpStream) -> Value {
+ stream
+ .set_read_timeout(Some(std::time::Duration::from_secs(2)))
+ .unwrap();
+ let mut request = Vec::new();
+ let mut buffer = [0u8; 4096];
+ loop {
+ let count = stream.read(&mut buffer).unwrap_or(0);
+ if count == 0 {
+ break;
+ }
+ request.extend_from_slice(&buffer[..count]);
+ if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") {
+ let headers = String::from_utf8_lossy(&request[..end]).to_ascii_lowercase();
+ let length = headers
+ .lines()
+ .find_map(|l| l.strip_prefix("content-length:")?.trim().parse().ok())
+ .unwrap_or(0usize);
+ if request.len() >= end + 4 + length {
+ return serde_json::from_slice(&request[end + 4..end + 4 + length])
+ .unwrap_or(Value::Null);
+ }
+ }
+ }
+ Value::Null
+ }
+
+ /// A node that behaves like xerj on a large index: any aggregation that
+ /// needs the corpus is refused by the per-query memory guard, while a
+ /// sorted size-1 search answers from doc values.
+ #[test]
+ fn time_bounds_come_from_sorted_searches_not_materialising_aggs() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let bodies = Arc::new(Mutex::new(Vec::<Value>::new()));
+ let seen = Arc::clone(&bodies);
+ let server = std::thread::spawn(move || {
+ for stream in listener.incoming().take(2) {
+ let mut stream = stream.unwrap();
+ let body = read_body(&mut stream);
+ seen.lock().unwrap().push(body.clone());
+ let (status, reply) = if body.get("aggs").is_some() {
+ (
+ "429 Too Many Requests",
+ json!({"error":{"type":"circuit_breaking_exception","reason":"[request] query allocation (768.2mb) would exceed limits.max_query_memory_mb=512MB at [aggregation corpus materialisation]"},"status":429}),
+ )
+ } else {
+ let asc = body.pointer("/sort/0/t/order") == Some(&json!("asc"));
+ let key = if asc { T_MIN } else { T_MAX };
+ (
+ "200 OK",
+ json!({"hits":{"total":{"value":3},"hits":[
+ {"_id":"x","_source":{"t":iso(key)},"sort":[key]}]}}),
+ )
+ };
+ let reply = reply.to_string();
+ let _ = write!(
+ stream,
+ "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{reply}",
+ reply.len()
+ );
+ }
+ });
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ let range = dataset_time_range(&es, "i", "t").unwrap();
+ server.join().unwrap();
+ assert_eq!(range, (Some(iso(T_MIN)), Some(iso(T_MAX))));
+ for body in bodies.lock().unwrap().iter() {
+ assert!(body.get("aggs").is_none(), "materialising agg sent: {body}");
+ assert_eq!(body.get("size"), Some(&json!(1)), "{body}");
+ }
+ }
+}