AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): finalize-count reads time bounds by sort, not min/max aggs (#2750)
xerj-orgd3e2eddcb39f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ce9ce2f21..8ba9d4fbb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -97,6 +97,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0longer take a node from below the watermark into the parent breaker band andturn ingest into HTTP 429s. The relief for a search node resting between themerge-pool ceiling and the watermark is kept (#2427).+- `xerj autoindex` no longer stalls for up to 10 minutes per large dataset in+ `finalize-count`. The dataset time range was read with `min`/`max`+ aggregations, which make the node copy every document first; on a+ 393 k-record dataset that priced at 768 MB against the 512 MB+ `limits.max_query_memory_mb` default and the client re-asked for its full+ 600 s patience. The bounds now come from two sorted size-1 searches, and a+ 429 from the per-query memory guard is no longer retried, since the same+ query is refused again (#2427).- The `xerj-autoindex` corpus-lock tests no longer fail at random in CI(#2520, Refs #2427). One test put `-c core.fsmonitor=false` after the gitsubcommand, so git exited 129 at once and the test only passed by winning adiff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 874afa7dd..dc77d4402 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -640,6 +640,13 @@ pub const THROTTLE_PATIENCE: Duration = Duration::from_secs(600);/// tier's 2 GiB memtable budget, and short of a multi-minute spin.pub const ZERO_PROGRESS_PATIENCE: Duration = Duration::from_secs(90);+/// The engine's per-query guard (`limits.max_query_memory_mb`) refusing a+/// request whose estimated allocation exceeds the budget. Unlike the parent+/// breaker this is a property of the query and the data, not of the moment.+pub(crate) fn is_query_allocation_refusal(reason: &str) -> bool {+ reason.contains("would exceed limits.max_query_memory_mb")+}+/// The parent memory breaker, as opposed to a disk flood-stage 429 or a/// thread-pool rejection. Both of its reasons carry one of these phrases.pub(crate) fn is_parent_memory_breaker(reason: &str) -> bool {@@ -1528,7 +1535,19 @@ impl Es {) -> Attempt<T> {let status = resp.status();if status.as_u16() == 429 {- return Attempt::Throttled(server_reason(resp));+ let reason = server_reason(resp);+ if let Some(reason) = reason.as_deref().filter(|r| is_query_allocation_refusal(r)) {+ // The per-query memory guard prices THIS request against a+ // fixed budget. Nothing a wait can change makes the same query+ // over the same documents cheaper, so retrying only stalls the+ // caller for the whole throttle patience (#2427: 600 s per+ // dataset in finalize-count).+ return Attempt::Abort(anyhow!(+ "{what}: HTTP 429 Too Many Requests: {reason} (not retried: the per-query \+ memory guard refuses the same query again)"+ ));+ }+ return Attempt::Throttled(reason);}if status.is_server_error() {return Attempt::Failed(match server_reason(resp) {@@ -4774,6 +4793,59 @@ mod tests {);}+ /// #2427: a 429 from the per-query memory guard is deterministic for the+ /// same query, so even a loading client (admission on, long patience)+ /// gives up after the first answer instead of re-asking until the+ /// patience runs out.+ #[test]+ fn a_query_allocation_429_is_not_retried() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let hits = Arc::new(std::sync::atomic::AtomicUsize::new(0));+ let server_hits = Arc::clone(&hits);+ let stop = Arc::new(AtomicBool::new(false));+ let server_stop = Arc::clone(&stop);+ listener.set_nonblocking(true).unwrap();+ let server = std::thread::spawn(move || loop {+ match listener.accept() {+ Ok((mut stream, _)) => {+ stream.set_nonblocking(false).unwrap();+ let _ = read_request(&mut stream);+ server_hits.fetch_add(1, Ordering::SeqCst);+ let body = br#"{"error":{"type":"circuit_breaking_exception","reason":"[request] query allocation (768.2mb) would exceed limits.max_query_memory_mb=512MB at [aggregation corpus materialisation]; reduce size/aggregation cardinality"},"status":429}"#;+ respond_status(&mut stream, "429 Too Many Requests", body);+ }+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ if server_stop.load(Ordering::SeqCst) {+ return;+ }+ std::thread::sleep(Duration::from_millis(2));+ }+ Err(error) => panic!("accept: {error}"),+ }+ });+ let es = client(address, 4)+ .with_throttle_patience(Duration::from_secs(2))+ .with_zero_progress_patience(Duration::from_secs(2));+ let started = Instant::now();+ let error = es+ .search(+ "i",+ &serde_json::json!({"size": 0, "aggs": {"mn": {"min": {"field": "t"}}}}),+ )+ .unwrap_err();+ let elapsed = started.elapsed();+ stop.store(true, Ordering::SeqCst);+ server.join().unwrap();+ let rendered = format!("{error:#}");+ assert!(rendered.contains("max_query_memory_mb"), "{rendered}");+ assert_eq!(hits.load(Ordering::SeqCst), 1, "asked once: {rendered}");+ assert!(+ elapsed < Duration::from_millis(500),+ "a query-allocation refusal must not wait out the patience, took {elapsed:?}"+ );+ }+// ── #955: a request is bounded in actions and bytes, and a request the// server still calls too large is halved instead of ending the run ──diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 777325019..2e1cd301a 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -8655,6 +8655,52 @@ fn catalog_runs(es: &Es) -> Result<Vec<Value>> {}).unwrap_or_default())}+/// A dataset's time bounds from two size-1 searches sorted on its time field.+///+/// #2427: this used to be one `size:0` search with `min`/`max` aggs. Neither+/// has a columnar fast path, so the node deep-cloned every document into+/// owned values first ("aggregation corpus materialisation", ~2 KB a doc+/// against `limits.max_query_memory_mb`). A 393 k-record dataset priced at+/// 768 MB against the 512 MB default, the node answered 429, and the client+/// re-asked for its full 600 s patience before the run moved on without the+/// bounds. A sort reads doc values and clones nothing — the same shape the+/// generation read-back has used since #1183.+fn dataset_time_range(+ es: &Es,+ index: &str,+ field: &str,+) -> Result<(Option<String>, Option<String>)> {+ let bound = |order: &str| -> Result<Option<String>> {+ let body = json!({+ "size": 1,+ "query": {"exists": {"field": field}},+ "sort": [{field: {"order": order}}],+ "_source": [field]+ });+ let v = es.search(index, &body)?;+ let hit = match v.pointer("/hits/hits/0") {+ Some(hit) => hit,+ None => return Ok(None),+ };+ // The stored value first, exactly like the generation read-back+ // (`sync_executor::extreme_time`); the sort key (epoch ms for a+ // date) only when `_source` lacks the field.+ let from_source = hit+ .pointer("/_source")+ .and_then(|src| src.get(field))+ .and_then(|v| match v {+ Value::String(s) => Some(s.clone()),+ Value::Number(n) => n.as_i64().map(xerj_common::schema::epoch_ms_to_iso8601_utc),+ _ => None,+ });+ Ok(from_source.or_else(|| {+ hit.pointer("/sort/0")+ .and_then(Value::as_i64)+ .map(xerj_common::schema::epoch_ms_to_iso8601_utc)+ }))+ };+ Ok((bound("asc")?, bound("desc")?))+}fn run_index_report_inner(mut cfg: IndexCfg,@@ -12090,22 +12136,8 @@ fn run_index_report_inner(}}if let Some(t) = &d.time_field {- let body = json!({"size":0,"aggs":{- "mn":{"min":{"field":t}},"mx":{"max":{"field":t}}}});- if let Ok(v) = es.search(&d.index, &body) {- let get = |k: &str| -> Option<String> {- let a = v.pointer(&format!("/aggregations/{k}"))?;- a.get("value_as_string")- .and_then(|s| s.as_str())- .map(|s| s.to_string())- .or_else(|| {- a.get("value").and_then(|f| f.as_f64()).and_then(|ms| {- chrono::DateTime::from_timestamp_millis(ms as i64)- .map(|d| infer::dates::to_rfc3339_millis(&d))- })- })- };- ds_timerange.insert(d.slug.clone(), (get("mn"), get("mx")));+ if let Ok(range) = dataset_time_range(&es, &d.index, t) {+ ds_timerange.insert(d.slug.clone(), range);}}}@@ -17357,3 +17389,91 @@ mod map_folder_preview {);}}++#[cfg(test)]+mod dataset_time_range_tests {+ use super::*;+ use std::io::Read;+ use std::net::TcpListener;+ use std::sync::Arc;++ const T_MIN: i64 = 1_790_812_800_000; // 2026-10-01T00:00:00Z+ const T_MAX: i64 = 1_790_985_600_000; // 2026-10-03T00:00:00Z++ fn iso(ms: i64) -> String {+ xerj_common::schema::epoch_ms_to_iso8601_utc(ms)+ }++ fn read_body(stream: &mut std::net::TcpStream) -> Value {+ stream+ .set_read_timeout(Some(std::time::Duration::from_secs(2)))+ .unwrap();+ let mut request = Vec::new();+ let mut buffer = [0u8; 4096];+ loop {+ let count = stream.read(&mut buffer).unwrap_or(0);+ if count == 0 {+ break;+ }+ request.extend_from_slice(&buffer[..count]);+ if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") {+ let headers = String::from_utf8_lossy(&request[..end]).to_ascii_lowercase();+ let length = headers+ .lines()+ .find_map(|l| l.strip_prefix("content-length:")?.trim().parse().ok())+ .unwrap_or(0usize);+ if request.len() >= end + 4 + length {+ return serde_json::from_slice(&request[end + 4..end + 4 + length])+ .unwrap_or(Value::Null);+ }+ }+ }+ Value::Null+ }++ /// A node that behaves like xerj on a large index: any aggregation that+ /// needs the corpus is refused by the per-query memory guard, while a+ /// sorted size-1 search answers from doc values.+ #[test]+ fn time_bounds_come_from_sorted_searches_not_materialising_aggs() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let bodies = Arc::new(Mutex::new(Vec::<Value>::new()));+ let seen = Arc::clone(&bodies);+ let server = std::thread::spawn(move || {+ for stream in listener.incoming().take(2) {+ let mut stream = stream.unwrap();+ let body = read_body(&mut stream);+ seen.lock().unwrap().push(body.clone());+ let (status, reply) = if body.get("aggs").is_some() {+ (+ "429 Too Many Requests",+ json!({"error":{"type":"circuit_breaking_exception","reason":"[request] query allocation (768.2mb) would exceed limits.max_query_memory_mb=512MB at [aggregation corpus materialisation]"},"status":429}),+ )+ } else {+ let asc = body.pointer("/sort/0/t/order") == Some(&json!("asc"));+ let key = if asc { T_MIN } else { T_MAX };+ (+ "200 OK",+ json!({"hits":{"total":{"value":3},"hits":[+ {"_id":"x","_source":{"t":iso(key)},"sort":[key]}]}}),+ )+ };+ let reply = reply.to_string();+ let _ = write!(+ stream,+ "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{reply}",+ reply.len()+ );+ }+ });+ let es = Es::new(&format!("http://{address}"), None).unwrap();+ let range = dataset_time_range(&es, "i", "t").unwrap();+ server.join().unwrap();+ assert_eq!(range, (Some(iso(T_MIN)), Some(iso(T_MAX))));+ for body in bodies.lock().unwrap().iter() {+ assert!(body.get("aggs").is_none(), "materialising agg sent: {body}");+ assert_eq!(body.get("size"), Some(&json!(1)), "{body}");+ }+ }+}