Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): agg-free catalog read-back + windowed finalize refresh (#1183)

xerj-org07533e67f494
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 4aa75676b..d1384314f 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -19,6 +19,10 @@ static HTTP_E2E_LOCK: Mutex<()> = Mutex::new(());
struct HttpState {
docs: HashMap<(String, String), Value>,
requests: Vec<(String, String)>,
+ /// Every `POST …/_search` body, in order (#1183): the dataset read-back's
+ /// agg-free shape and its sorted time-bound queries are asserted against
+ /// these.
+ search_bodies: Vec<Value>,
data_bulk_requests: usize,
fail_next_data_bulk: bool,
/// Apply the first half of the next data bulk's items and *then* report
@@ -162,6 +166,21 @@ impl HttpEndpoint {
.collect()
}
+ /// Every `POST …/_search` body the endpoint has seen, in order (#1183).
+ fn search_bodies(&self) -> Vec<Value> {
+ self.state.lock().unwrap().search_bodies.clone()
+ }
+
+ /// One catalog document by id, from the fake's stored docs.
+ fn catalog_doc(&self, id: &str) -> Option<Value> {
+ self.state
+ .lock()
+ .unwrap()
+ .docs
+ .get(&(catalog::CATALOG_INDEX.to_string(), id.to_string()))
+ .cloned()
+ }
+
fn data_bulk_requests(&self) -> usize {
self.state.lock().unwrap().data_bulk_requests
}
@@ -264,6 +283,9 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {
let deleted = delete_http(path, &body, state);
(200, json!({"deleted": deleted, "failures": []}))
} else if method == "POST" && path.ends_with("/_search") {
+ if let Ok(query) = serde_json::from_slice::<Value>(&body) {
+ state.lock().unwrap().search_bodies.push(query);
+ }
(200, search_http(path, &body, state))
} else if method == "GET" && path.ends_with("/_count") {
let index = path.trim_start_matches('/').trim_end_matches("/_count");
@@ -790,7 +812,39 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value
// catalog sweep pages, and a page has to be a page. A request that omits
// them still gets everything, which is what every older fixture assumes.
// Ordering is by `_id` so a page boundary is not `HashMap` iteration order.
- matching.sort_by_key(|(key, _)| *key);
+ //
+ // #1183: honour `sort` when the request states it — the dataset read-back's
+ // time bounds are size-1 searches sorted on the time field, and the first
+ // hit must BE the extreme, the same invariant a doc-values sort gives the
+ // real server. A request without `sort` keeps the `_id` order.
+ let sort_spec: Option<(String, String)> = query
+ .get("sort")
+ .and_then(Value::as_array)
+ .and_then(|clauses| clauses.first())
+ .and_then(|clause| {
+ if let Some(field) = clause.as_str() {
+ return Some((field.to_owned(), "asc".to_owned()));
+ }
+ clause
+ .as_object()
+ .and_then(|object| object.iter().next())
+ .map(|(field, spec)| {
+ (
+ field.clone(),
+ spec.get("order")
+ .and_then(Value::as_str)
+ .unwrap_or("asc")
+ .to_owned(),
+ )
+ })
+ });
+ match &sort_spec {
+ Some((field, order)) if order == "desc" => {
+ matching.sort_by(|(_, a), (_, b)| cmp_docs_by_field(b, a, field))
+ }
+ Some((field, _)) => matching.sort_by(|(_, a), (_, b)| cmp_docs_by_field(a, b, field)),
+ None => matching.sort_by_key(|(key, _)| *key),
+ }
let total = matching.len();
let from = query
.get("from")
@@ -815,6 +869,20 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value
})
}
+/// Compare two docs on one field for the fake's `sort` support: numbers
+/// numerically, strings lexically, a missing value sorting first (the null
+/// ordering the real server gives an unmapped/absent field).
+fn cmp_docs_by_field(a: &Value, b: &Value, field: &str) -> std::cmp::Ordering {
+ let rank = |doc: &Value| match doc.get(field) {
+ Some(Value::Number(n)) => (1u8, n.as_f64().unwrap_or(0.0), String::new()),
+ Some(Value::String(s)) => (2u8, 0.0, s.clone()),
+ _ => (0u8, 0.0, String::new()),
+ };
+ rank(a)
+ .partial_cmp(&rank(b))
+ .unwrap_or(std::cmp::Ordering::Equal)
+}
+
fn cfg(root: &Path, state_dir: &Path, url: &str, semantic: bool) -> IndexCfg {
IndexCfg {
root: root.to_owned(),
@@ -1248,6 +1316,89 @@ fn no_semantic_generation_does_not_require_embedding_identity_endpoint() {
assert_eq!(endpoint.data_docs().len(), 1);
}
+/// #1183: the finalize read-back must not carry `aggs`. A `min`/`max` agg on
+/// the dataset's time field has no columnar fast path under the sharded
+/// memtable, so a real server deep-clones every matching document into owned
+/// Values ("aggregation corpus materialisation", ~2KB a doc against
+/// `max_query_memory_mb`) — on the xerj-search rebuild one ~570k-record
+/// dataset estimated 1.1GB against the 512MB default, the server answered
+/// 429 circuit_breaking_exception, and the client retried for its full 600s
+/// budget before aborting: the multi-hour "finalize-catalog deadlock". The
+/// count now runs without aggs; the time bounds come from size-1 searches
+/// sorted on the time field, which sort from doc values and clone nothing.
+#[test]
+fn dataset_read_back_is_agg_free_and_bounds_time_by_sort() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("a.csv"),
+ "id,cloned_at\n1,2026-10-01T00:00:00.000Z\n2,2026-10-02T00:00:00.000Z\n3,2026-10-03T00:00:00.000Z\n",
+ )
+ .unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let (code, _) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+
+ // 1. No search this run issued carries an aggregation — the read-back's
+ // count included.
+ let bodies = endpoint.search_bodies();
+ assert!(!bodies.is_empty());
+ for body in &bodies {
+ assert!(
+ body.get("aggs").is_none() && body.get("aggregations").is_none(),
+ "#1183: a search carried aggs: {body}"
+ );
+ }
+
+ // 2. The time bounds are the sorted size-1 searches: one ascending, one
+ // descending, on the dataset's elected time field, under the dataset's
+ // own ax_dataset filter.
+ let sorted: Vec<&Value> = bodies
+ .iter()
+ .filter(|body| body.pointer("/sort/0/cloned_at").is_some())
+ .collect();
+ assert_eq!(
+ sorted.len(),
+ 2,
+ "one ascending and one descending bound query: {bodies:?}"
+ );
+ assert!(
+ sorted
+ .iter()
+ .any(|body| body.pointer("/sort/0/cloned_at/order") == Some(&json!("asc"))),
+ "ascending bound missing: {bodies:?}"
+ );
+ assert!(
+ sorted
+ .iter()
+ .any(|body| body.pointer("/sort/0/cloned_at/order") == Some(&json!("desc"))),
+ "descending bound missing: {bodies:?}"
+ );
+ for body in &sorted {
+ assert_eq!(body.get("size"), Some(&json!(1)), "size-1 query: {body}");
+ assert!(
+ body.pointer("/query/bool/filter").is_some(),
+ "bound query keeps the dataset filter: {body}"
+ );
+ }
+
+ // 3. The catalog document carries the corpus's TRUE bounds — the fake
+ // honours `sort`, so the first hit is the extreme, exactly like the
+ // doc-values sort on a real server.
+ let dataset = endpoint
+ .catalog_doc("ds:incremental-http:csv")
+ .expect("dataset catalog document must exist");
+ assert_eq!(dataset["time_field"], json!("cloned_at"));
+ assert_eq!(dataset["time_min"], json!("2026-10-01T00:00:00.000Z"));
+ assert_eq!(dataset["time_max"], json!("2026-10-03T00:00:00.000Z"));
+}
+
#[test]
fn initial_generation_and_noop_are_end_to_end_idempotent() {
let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index 369266a35..6e803ef72 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -700,20 +700,29 @@ impl<'a> EsSyncBackend<'a> {
// were sealed under, and stays exact even if two datasets ever share
// one index. `ax_dataset` is written at every sink site and is a
// mapped `PROVENANCE_FIELDS` keyword.
- let mut body = serde_json::json!({
+ //
+ // #1183: this read-back must never carry `aggs`. A `size:0 + aggs`
+ // query whose agg has no columnar fast path makes the server
+ // deep-clone every matching document into owned Values before the
+ // agg runs (the "aggregation corpus materialisation" charge in the
+ // engine's search path, ~2 KB a doc against
+ // `limits.max_query_memory_mb`). On the xerj-search rebuild one
+ // ~570 k-record dataset estimated 1.1 GB against the 512 MB
+ // default, the server answered 429 circuit_breaking_exception, the
+ // client retried for its full 600 s budget, and the run aborted —
+ // the "finalize-catalog deadlock". The count needs no corpus; the
+ // time bounds come from two size-1 searches sorted on the time
+ // field (`extreme_time` below), which sort from doc values and
+ // clone nothing.
+ let filter = serde_json::json!([
+ {"terms": {"ax_file": content_ids}},
+ {"term": {"ax_dataset": dataset.slug}}
+ ]);
+ let body = serde_json::json!({
"size": 0,
"track_total_hits": true,
- "query": {"bool": {"filter": [
- {"terms": {"ax_file": content_ids}},
- {"term": {"ax_dataset": dataset.slug}}
- ]}}
+ "query": {"bool": {"filter": filter}}
});
- if let Some(field) = &dataset.time_field {
- body["aggs"] = serde_json::json!({
- "time_min": {"min": {"field": field}},
- "time_max": {"max": {"field": field}}
- });
- }
let response = self.es.search(&dataset.index, &body)?;
let record_count = response
.pointer("/hits/total/value")
@@ -791,11 +800,13 @@ impl<'a> EsSyncBackend<'a> {
sum.checked_add(group.expected_junk_records)
.context("dataset junk-record count overflow")
})?;
- let time = |name: &str| {
- response
- .pointer(&format!("/aggregations/{name}/value_as_string"))
- .and_then(Value::as_str)
- .map(ToOwned::to_owned)
+ let time_bound = |order: &'static str| -> Result<Option<String>> {
+ match &dataset.time_field {
+ Some(field) => {
+ self.extreme_time(&dataset.index, &filter, field, order)
+ }
+ None => Ok(None),
+ }
};
out.insert(
dataset.slug.clone(),
@@ -804,8 +815,8 @@ impl<'a> EsSyncBackend<'a> {
junk_records,
bytes,
formats,
- time_min: time("time_min"),
- time_max: time("time_max"),
+ time_min: time_bound("asc")?,
+ time_max: time_bound("desc")?,
sample_queries: crate::catalog::build_sample_queries(dataset, &[]),
notes: dataset
.group
@@ -823,6 +834,45 @@ impl<'a> EsSyncBackend<'a> {
}
Ok(out)
}
+
+ /// One end of a dataset's time range, via a size-1 search sorted on the
+ /// time field under the dataset's own filter (#1183). This replaced a
+ /// `min`/`max` aggregation on the same field: the agg made the server
+ /// materialise the aggregation corpus for the whole dataset and refuse at
+ /// the query-memory breaker on large ones, while a sort reads doc values
+ /// and clones nothing. Exact for the same reason the agg was: the filter
+ /// is the dataset's own identity, and the sort's first hit IS the extreme.
+ ///
+ /// The returned string matches the agg's `value_as_string`: the stored
+ /// `_source` value verbatim when the field holds strings (autoindex date
+ /// fields are ISO strings), and epoch-milliseconds rendered by the same
+ /// ISO helper the engine's `min`/`max` aggs use when it holds numbers —
+ /// so the catalog document does not change shape between corpus builds.
+ fn extreme_time(
+ &self,
+ index: &str,
+ filter: &Value,
+ field: &str,
+ order: &str,
+ ) -> Result<Option<String>> {
+ let body = serde_json::json!({
+ "size": 1,
+ "query": {"bool": {"filter": filter}},
+ "sort": [{field: {"order": order}}],
+ "_source": [field]
+ });
+ let response = self.es.search(index, &body)?;
+ Ok(response
+ .pointer("/hits/hits/0/_source")
+ .and_then(|src| src.get(field))
+ .and_then(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ Value::Number(n) => n
+ .as_i64()
+ .map(xerj_common::schema::epoch_ms_to_iso8601_utc),
+ _ => None,
+ }))
+ }
}
impl SyncOperationBackend for EsSyncBackend<'_> {
@@ -945,14 +995,28 @@ impl SyncOperationBackend for EsSyncBackend<'_> {
// path's phase so a reader of either route sees the same vocabulary.
self.pr
.phase("finalize-catalog", desired.plan.datasets.len() as u64, 0);
- for index in desired
+ // #1183: the refreshes used to run one index per request, serially —
+ // 1,479 round-trips on the xerj-search corpus, each taking seconds
+ // against a node whose memory breaker was draining between them, so
+ // the phase read as `stalled` for the better part of an hour before
+ // the read-back behind it even started. The refresh path takes a
+ // comma-list (the server resolves it through the same selector as a
+ // wildcard), so one request per window of indexes, with one progress
+ // tick per index so the phase denominator keeps its meaning.
+ let indexes: Vec<&str> = desired
.plan
.datasets
.iter()
.map(|dataset| dataset.index.as_str())
.collect::<std::collections::BTreeSet<_>>()
- {
- self.es.refresh(index)?;
+ .into_iter()
+ .collect();
+ const REFRESH_WINDOW: usize = 50;
+ for batch in indexes.chunks(REFRESH_WINDOW) {
+ self.es.refresh(&batch.join(","))?;
+ for _ in batch {
+ self.pr.item_done(0);
+ }
}
self.es.refresh(crate::catalog::CATALOG_INDEX)?;
let prior_run_id =
diff --git a/engine/crates/xerj-common/src/schema.rs b/engine/crates/xerj-common/src/schema.rs
index 8de0b0cc3..91b5c0ce3 100644
--- a/engine/crates/xerj-common/src/schema.rs
+++ b/engine/crates/xerj-common/src/schema.rs
@@ -13,6 +13,19 @@ use serde::{Deserialize, Serialize};
use crate::error::XerjError;
use crate::types::{FieldConfig, FieldType, Schema};
+/// Render epoch milliseconds as an ISO-8601 UTC string with millisecond
+/// precision (`2026-10-06T11:09:13.000Z`). One definition for every caller
+/// that has to render a date the way `min`/`max` aggregations render their
+/// `value_as_string`: the engine's agg layer (xerj-engine/src/aggs.rs, its
+/// former private copy) and the autoindex catalog read-back, which replaced
+/// those aggs with sorted size-1 reads (#1183) and must not diverge from the
+/// rendering the catalog already published.
+pub fn epoch_ms_to_iso8601_utc(ms: i64) -> String {
+ chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms)
+ .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
+ .unwrap_or_default()
+}
+
// ═════════════════════════════════════════════════════════════════════════════
// Dynamic mapping mode
// ═════════════════════════════════════════════════════════════════════════════
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 1a5aabe06..d4c164eb1 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -4936,11 +4936,11 @@ pub(crate) fn java_to_strftime(pattern: &str) -> String {
/// Format a Unix epoch in milliseconds as the ISO-8601 UTC string with
/// millisecond precision, matching Elasticsearch's default date format
/// (`"yyyy-MM-dd'T'HH:mm:ss.SSSZ"` → `"2016-05-03T00:00:00.000Z"`).
-pub(crate) fn epoch_ms_to_iso8601_utc(ms: i64) -> String {
- chrono::DateTime::<chrono::Utc>::from_timestamp_millis(ms)
- .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
- .unwrap_or_default()
-}
+///
+/// Moved to `xerj_common::schema` (one definition: the autoindex catalog
+/// read-back renders raw-millis date fields with the same format, #1183) and
+/// re-exported here so this module's call sites keep the bare name.
+pub(crate) use xerj_common::schema::epoch_ms_to_iso8601_utc;
/// Count fractional-second digits in an ISO-8601 string
/// (e.g. "2021-08-12T00:00:00.000000000+02:00" → 9).