AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): agg-free catalog read-back + windowed finalize refresh (#1183)
xerj-org07533e67f494
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 4aa75676b..d1384314f 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -19,6 +19,10 @@ static HTTP_E2E_LOCK: Mutex<()> = Mutex::new(());struct HttpState {docs: HashMap<(String, String), Value>,requests: Vec<(String, String)>,+ /// Every `POST …/_search` body, in order (#1183): the dataset read-back's+ /// agg-free shape and its sorted time-bound queries are asserted against+ /// these.+ search_bodies: Vec<Value>,data_bulk_requests: usize,fail_next_data_bulk: bool,/// Apply the first half of the next data bulk's items and *then* report@@ -162,6 +166,21 @@ impl HttpEndpoint {.collect()}+ /// Every `POST …/_search` body the endpoint has seen, in order (#1183).+ fn search_bodies(&self) -> Vec<Value> {+ self.state.lock().unwrap().search_bodies.clone()+ }++ /// One catalog document by id, from the fake's stored docs.+ fn catalog_doc(&self, id: &str) -> Option<Value> {+ self.state+ .lock()+ .unwrap()+ .docs+ .get(&(catalog::CATALOG_INDEX.to_string(), id.to_string()))+ .cloned()+ }+fn data_bulk_requests(&self) -> usize {self.state.lock().unwrap().data_bulk_requests}@@ -264,6 +283,9 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {let deleted = delete_http(path, &body, state);(200, json!({"deleted": deleted, "failures": []}))} else if method == "POST" && path.ends_with("/_search") {+ if let Ok(query) = serde_json::from_slice::<Value>(&body) {+ state.lock().unwrap().search_bodies.push(query);+ }(200, search_http(path, &body, state))} else if method == "GET" && path.ends_with("/_count") {let index = path.trim_start_matches('/').trim_end_matches("/_count");@@ -790,7 +812,39 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value// catalog sweep pages, and a page has to be a page. A request that omits// them still gets everything, which is what every older fixture assumes.// Ordering is by `_id` so a page boundary is not `HashMap` iteration order.- matching.sort_by_key(|(key, _)| *key);+ //+ // #1183: honour `sort` when the request states it — the dataset read-back's+ // time bounds are size-1 searches sorted on the time field, and the first+ // hit must BE the extreme, the same invariant a doc-values sort gives the+ // real server. A request without `sort` keeps the `_id` order.+ let sort_spec: Option<(String, String)> = query+ .get("sort")+ .and_then(Value::as_array)+ .and_then(|clauses| clauses.first())+ .and_then(|clause| {+ if let Some(field) = clause.as_str() {+ return Some((field.to_owned(), "asc".to_owned()));+ }+ clause+ .as_object()+ .and_then(|object| object.iter().next())+ .map(|(field, spec)| {+ (+ field.clone(),+ spec.get("order")+ .and_then(Value::as_str)+ .unwrap_or("asc")+ .to_owned(),+ )+ })+ });+ match &sort_spec {+ Some((field, order)) if order == "desc" => {+ matching.sort_by(|(_, a), (_, b)| cmp_docs_by_field(b, a, field))+ }+ Some((field, _)) => matching.sort_by(|(_, a), (_, b)| cmp_docs_by_field(a, b, field)),+ None => matching.sort_by_key(|(key, _)| *key),+ }let total = matching.len();let from = query.get("from")@@ -815,6 +869,20 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value})}+/// Compare two docs on one field for the fake's `sort` support: numbers+/// numerically, strings lexically, a missing value sorting first (the null+/// ordering the real server gives an unmapped/absent field).+fn cmp_docs_by_field(a: &Value, b: &Value, field: &str) -> std::cmp::Ordering {+ let rank = |doc: &Value| match doc.get(field) {+ Some(Value::Number(n)) => (1u8, n.as_f64().unwrap_or(0.0), String::new()),+ Some(Value::String(s)) => (2u8, 0.0, s.clone()),+ _ => (0u8, 0.0, String::new()),+ };+ rank(a)+ .partial_cmp(&rank(b))+ .unwrap_or(std::cmp::Ordering::Equal)+}+fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {IndexCfg {root: root.to_owned(),@@ -1248,6 +1316,89 @@ fn no_semantic_generation_does_not_require_embedding_identity_endpoint() {assert_eq!(endpoint.data_docs().len(), 1);}+/// #1183: the finalize read-back must not carry `aggs`. A `min`/`max` agg on+/// the dataset's time field has no columnar fast path under the sharded+/// memtable, so a real server deep-clones every matching document into owned+/// Values ("aggregation corpus materialisation", ~2KB a doc against+/// `max_query_memory_mb`) — on the xerj-search rebuild one ~570k-record+/// dataset estimated 1.1GB against the 512MB default, the server answered+/// 429 circuit_breaking_exception, and the client retried for its full 600s+/// budget before aborting: the multi-hour "finalize-catalog deadlock". The+/// count now runs without aggs; the time bounds come from size-1 searches+/// sorted on the time field, which sort from doc values and clone nothing.+#[test]+fn dataset_read_back_is_agg_free_and_bounds_time_by_sort() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("a.csv"),+ "id,cloned_at\n1,2026-10-01T00:00:00.000Z\n2,2026-10-02T00:00:00.000Z\n3,2026-10-03T00:00:00.000Z\n",+ )+ .unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);++ let (code, _) = run_index_report(config).unwrap();+ assert_eq!(code, 0);++ // 1. No search this run issued carries an aggregation — the read-back's+ // count included.+ let bodies = endpoint.search_bodies();+ assert!(!bodies.is_empty());+ for body in &bodies {+ assert!(+ body.get("aggs").is_none() && body.get("aggregations").is_none(),+ "#1183: a search carried aggs: {body}"+ );+ }++ // 2. The time bounds are the sorted size-1 searches: one ascending, one+ // descending, on the dataset's elected time field, under the dataset's+ // own ax_dataset filter.+ let sorted: Vec<&Value> = bodies+ .iter()+ .filter(|body| body.pointer("/sort/0/cloned_at").is_some())+ .collect();+ assert_eq!(+ sorted.len(),+ 2,+ "one ascending and one descending bound query: {bodies:?}"+ );+ assert!(+ sorted+ .iter()+ .any(|body| body.pointer("/sort/0/cloned_at/order") == Some(&json!("asc"))),+ "ascending bound missing: {bodies:?}"+ );+ assert!(+ sorted+ .iter()+ .any(|body| body.pointer("/sort/0/cloned_at/order") == Some(&json!("desc"))),+ "descending bound missing: {bodies:?}"+ );+ for body in &sorted {+ assert_eq!(body.get("size"), Some(&json!(1)), "size-1 query: {body}");+ assert!(+ body.pointer("/query/bool/filter").is_some(),+ "bound query keeps the dataset filter: {body}"+ );+ }++ // 3. The catalog document carries the corpus's TRUE bounds — the fake+ // honours `sort`, so the first hit is the extreme, exactly like the+ // doc-values sort on a real server.+ let dataset = endpoint+ .catalog_doc("ds:incremental-http:csv")+ .expect("dataset catalog document must exist");+ assert_eq!(dataset["time_field"], json!("cloned_at"));+ assert_eq!(dataset["time_min"], json!("2026-10-01T00:00:00.000Z"));+ assert_eq!(dataset["time_max"], json!("2026-10-03T00:00:00.000Z"));+}+#[test]fn initial_generation_and_noop_are_end_to_end_idempotent() {let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex 369266a35..6e803ef72 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -700,20 +700,29 @@ impl<'a> EsSyncBackend<'a> {// were sealed under, and stays exact even if two datasets ever share// one index. `ax_dataset` is written at every sink site and is a// mapped `PROVENANCE_FIELDS` keyword.- let mut body = serde_json::json!({+ //+ // #1183: this read-back must never carry `aggs`. A `size:0 + aggs`+ // query whose agg has no columnar fast path makes the server+ // deep-clone every matching document into owned Values before the+ // agg runs (the "aggregation corpus materialisation" charge in the+ // engine's search path, ~2 KB a doc against+ // `limits.max_query_memory_mb`). On the xerj-search rebuild one+ // ~570 k-record dataset estimated 1.1 GB against the 512 MB+ // default, the server answered 429 circuit_breaking_exception, the+ // client retried for its full 600 s budget, and the run aborted —+ // the "finalize-catalog deadlock". The count needs no corpus; the+ // time bounds come from two size-1 searches sorted on the time+ // field (`extreme_time` below), which sort from doc values and+ // clone nothing.+ let filter = serde_json::json!([+ {"terms": {"ax_file": content_ids}},+ {"term": {"ax_dataset": dataset.slug}}+ ]);+ let body = serde_json::json!({"size": 0,"track_total_hits": true,- "query": {"bool": {"filter": [- {"terms": {"ax_file": content_ids}},- {"term": {"ax_dataset": dataset.slug}}- ]}}+ "query": {"bool": {"filter": filter}}});- if let Some(field) = &dataset.time_field {- body["aggs"] = serde_json::json!({- "time_min": {"min": {"field": field}},- "time_max": {"max": {"field": field}}- });- }let response = self.es.search(&dataset.index, &body)?;let record_count = response.pointer("/hits/total/value")@@ -791,11 +800,13 @@ impl<'a> EsSyncBackend<'a> {sum.checked_add(group.expected_junk_records).context("dataset junk-record count overflow")})?;- let time = |name: &str| {- response- .pointer(&format!("/aggregations/{name}/value_as_string"))- .and_then(Value::as_str)- .map(ToOwned::to_owned)+ let time_bound = |order: &'static str| -> Result<Option<String>> {+ match &dataset.time_field {+ Some(field) => {+ self.extreme_time(&dataset.index, &filter, field, order)+ }+ None => Ok(None),+ }};out.insert(dataset.slug.clone(),@@ -804,8 +815,8 @@ impl<'a> EsSyncBackend<'a> {junk_records,bytes,formats,- time_min: time("time_min"),- time_max: time("time_max"),+ time_min: time_bound("asc")?,+ time_max: time_bound("desc")?,sample_queries: crate::catalog::build_sample_queries(dataset, &[]),notes: dataset.group@@ -823,6 +834,45 @@ impl<'a> EsSyncBackend<'a> {}Ok(out)}++ /// One end of a dataset's time range, via a size-1 search sorted on the+ /// time field under the dataset's own filter (#1183). This replaced a+ /// `min`/`max` aggregation on the same field: the agg made the server+ /// materialise the aggregation corpus for the whole dataset and refuse at+ /// the query-memory breaker on large ones, while a sort reads doc values+ /// and clones nothing. Exact for the same reason the agg was: the filter+ /// is the dataset's own identity, and the sort's first hit IS the extreme.+ ///+ /// The returned string matches the agg's `value_as_string`: the stored+ /// `_source` value verbatim when the field holds strings (autoindex date+ /// fields are ISO strings), and epoch-milliseconds rendered by the same+ /// ISO helper the engine's `min`/`max` aggs use when it holds numbers —+ /// so the catalog document does not change shape between corpus builds.+ fn extreme_time(+ &self,+ index: &str,+ filter: &Value,+ field: &str,+ order: &str,+ ) -> Result<Option<String>> {+ let body = serde_json::json!({+ "size": 1,+ "query": {"bool": {"filter": filter}},+ "sort": [{field: {"order": order}}],+ "_source": [field]+ });+ let response = self.es.search(index, &body)?;+ Ok(response+ .pointer("/hits/hits/0/_source")+ .and_then(|src| src.get(field))+ .and_then(|v| match v {+ Value::String(s) => Some(s.clone()),+ Value::Number(n) => n+ .as_i64()+ .map(xerj_common::schema::epoch_ms_to_iso8601_utc),+ _ => None,+ }))+ }}impl SyncOperationBackend for EsSyncBackend<'_> {@@ -945,14 +995,28 @@ impl SyncOperationBackend for EsSyncBackend<'_> {// path's phase so a reader of either route sees the same vocabulary.self.pr.phase("finalize-catalog", desired.plan.datasets.len() as u64, 0);- for index in desired+ // #1183: the refreshes used to run one index per request, serially —+ // 1,479 round-trips on the xerj-search corpus, each taking seconds+ // against a node whose memory breaker was draining between them, so+ // the phase read as `stalled` for the better part of an hour before+ // the read-back behind it even started. The refresh path takes a+ // comma-list (the server resolves it through the same selector as a+ // wildcard), so one request per window of indexes, with one progress+ // tick per index so the phase denominator keeps its meaning.+ let indexes: Vec<&str> = desired.plan.datasets.iter().map(|dataset| dataset.index.as_str()).collect::<std::collections::BTreeSet<_>>()- {- self.es.refresh(index)?;+ .into_iter()+ .collect();+ const REFRESH_WINDOW: usize = 50;+ for batch in indexes.chunks(REFRESH_WINDOW) {+ self.es.refresh(&batch.join(","))?;+ for _ in batch {+ self.pr.item_done(0);+ }}self.es.refresh(crate::catalog::CATALOG_INDEX)?;let prior_run_id =diff --git a/engine/crates/xerj-common/src/schema.rs b/engine/crates/xerj-common/src/schema.rsindex 8de0b0cc3..91b5c0ce3 100644--- a/engine/crates/xerj-common/src/schema.rs+++ b/engine/crates/xerj-common/src/schema.rs@@ -13,6 +13,19 @@ use serde::{Deserialize, Serialize};use crate::error::XerjError;use crate::types::{FieldConfig, FieldType, Schema};+/// Render epoch milliseconds as an ISO-8601 UTC string with millisecond+/// precision (`2026-10-06T11:09:13.000Z`). One definition for every caller+/// that has to render a date the way `min`/`max` aggregations render their+/// `value_as_string`: the engine's agg layer (xerj-engine/src/aggs.rs, its+/// former private copy) and the autoindex catalog read-back, which replaced+/// those aggs with sorted size-1 reads (#1183) and must not diverge from the+/// rendering the catalog already published.+pub fn epoch_ms_to_iso8601_utc(ms: i64) -> String {+ chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms)+ .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))+ .unwrap_or_default()+}+// ═════════════════════════════════════════════════════════════════════════════// Dynamic mapping mode// ═════════════════════════════════════════════════════════════════════════════diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 1a5aabe06..d4c164eb1 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -4936,11 +4936,11 @@ pub(crate) fn java_to_strftime(pattern: &str) -> String {/// Format a Unix epoch in milliseconds as the ISO-8601 UTC string with/// millisecond precision, matching Elasticsearch's default date format/// (`"yyyy-MM-dd'T'HH:mm:ss.SSSZ"` → `"2016-05-03T00:00:00.000Z"`).-pub(crate) fn epoch_ms_to_iso8601_utc(ms: i64) -> String {- chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms)- .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))- .unwrap_or_default()-}+///+/// Moved to `xerj_common::schema` (one definition: the autoindex catalog+/// read-back renders raw-millis date fields with the same format, #1183) and+/// re-exported here so this module's call sites keep the bare name.+pub(crate) use xerj_common::schema::epoch_ms_to_iso8601_utc;/// Count fractional-second digits in an ISO-8601 string/// (e.g. "2021-08-12T00:00:00.000000000+02:00" → 9).