Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(scroll): enforce the summed scroll snapshot cap on the alias route too (#405)

Vincenzo Lombardo8b9aacd94fb6
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index d4552b188..99f34a2e4 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -19784,10 +19784,6 @@ pub async fn search_with_scroll(
// Execute search across all indices and collect ALL hits.
let mut all_hits: Vec<(String, xerj_query::executor::Hit)> = Vec::new();
let mut total_count: u64 = 0;
- // Issue #198: set when any index matches more documents than the
- // per-index snapshot fetch could carry — the scroll would be a
- // silently truncated export.
- let mut scroll_truncated = false;
for idx_name in &index_names {
let idx = match state.engine.get_index(idx_name) {
@@ -19805,11 +19801,6 @@ pub async fn search_with_scroll(
if let Some(reason) = &result.script_failure {
return script_limit_response(reason);
}
- if result.hits.len() >= SCROLL_SNAPSHOT_MAX_HITS
- && result.total.value > result.hits.len() as u64
- {
- scroll_truncated = true;
- }
total_count += result.total.value;
for hit in result.hits {
all_hits.push((idx_name.clone(), hit));
@@ -19823,10 +19814,19 @@ pub async fn search_with_scroll(
// If scroll param present, store context and return first page.
if params.scroll.is_some() {
- // Issue #198: refuse to register a context whose snapshot is
- // missing documents — a truncated scroll looks complete to the
- // caller (reindex/backup/migration) and silently loses data.
- if scroll_truncated {
+ // Issue #198 (and #405: this route used to flag truncation only
+ // when a SINGLE index alone exceeded the per-index snapshot fetch
+ // — `full_req.size` is `SCROLL_SNAPSHOT_MAX_HITS` PER index, so an
+ // N-index request could materialise up to
+ // `N * SCROLL_SNAPSHOT_MAX_HITS` hits into one context and never
+ // trip that check, even though `POST /{index}/_search?scroll=`
+ // (search_impl) rejects the identical summed total. `total_count`
+ // above is already the exact sum across every index — compare it
+ // against the cap directly, matching search_impl's check
+ // (es_compat.rs, `is_scroll_request` block) exactly, so a request
+ // whose combined result set cannot fit the snapshot is refused
+ // loudly regardless of which route opened it.
+ if total_count > SCROLL_SNAPSHOT_MAX_HITS as u64 {
return scroll_window_exceeded(total_count, SCROLL_SNAPSHOT_MAX_HITS);
}
let scroll_id = Uuid::new_v4().to_string();
diff --git a/engine/crates/xerj-api/tests/scroll_alias_multi_index_cap.rs b/engine/crates/xerj-api/tests/scroll_alias_multi_index_cap.rs
new file mode 100644
index 000000000..264207f08
--- /dev/null
+++ b/engine/crates/xerj-api/tests/scroll_alias_multi_index_cap.rs
@@ -0,0 +1,187 @@
+//! Issue #405 (split out of #370): the scroll snapshot cap
+//! ([`SCROLL_SNAPSHOT_MAX_HITS`]) was enforced against the SUMMED total on
+//! `POST /{index}/_search?scroll=` but only PER INDEX on the alias route
+//! `POST /{index}/_search_scroll?scroll=`.
+//!
+//! `search_with_scroll` (the alias route's handler) fetches up to
+//! `SCROLL_SNAPSHOT_MAX_HITS` hits from EACH resolved index separately, so an
+//! N-index request could pin `N * SCROLL_SNAPSHOT_MAX_HITS` hits into one
+//! scroll context — the exact silently-truncated-export failure mode issue
+//! #198 exists to refuse — without ever tripping the old per-index check,
+//! since no single index alone exceeded the cap.
+//!
+//! Mirrors the fixture style of `scroll_snapshot_cap_is_documented.rs`
+//! (issue #370/#198): an under-cap control so the test cannot pass vacuously,
+//! then the real over-cap repro from #405 — two indices, each under the cap
+//! individually, whose SUM exceeds it.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+use xerj_api::es_compat::SCROLL_SNAPSHOT_MAX_HITS;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {
+ let response = app.clone().oneshot(req).await.expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ let value: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, value)
+}
+
+async fn seed(app: &axum::Router, index: &str, docs: usize) {
+ let (st, body) = send(
+ app,
+ Request::builder()
+ .method("PUT")
+ .uri(format!("/{index}"))
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({"mappings": {"properties": {"n": {"type": "long"}}}}).to_string(),
+ ))
+ .unwrap(),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create {index}: {body}");
+
+ let mut ndjson = String::with_capacity(docs * 32);
+ for n in 0..docs {
+ ndjson.push_str(&format!(
+ "{{\"index\":{{\"_id\":\"{n}\"}}}}\n{{\"n\":{n}}}\n"
+ ));
+ }
+ let (st, body) = send(
+ app,
+ Request::builder()
+ .method("POST")
+ .uri(format!("/{index}/_bulk"))
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(ndjson))
+ .unwrap(),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "bulk {index}: {body}");
+ assert_eq!(
+ body["errors"],
+ json!(false),
+ "bulk seed reported item errors for {index}: {body}"
+ );
+
+ let (st, body) = send(
+ app,
+ Request::builder()
+ .method("POST")
+ .uri(format!("/{index}/_refresh"))
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "refresh {index}: {body}");
+}
+
+async fn open_alias_scroll(
+ app: &axum::Router,
+ index_spec: &str,
+ size: usize,
+) -> (StatusCode, Value) {
+ send(
+ app,
+ Request::builder()
+ .method("POST")
+ .uri(format!("/{index_spec}/_search_scroll?scroll=1m"))
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({"size": size, "query": {"match_all": {}}}).to_string(),
+ ))
+ .unwrap(),
+ )
+ .await
+}
+
+/// Two indices, each individually UNDER the cap, whose combined total
+/// exceeds it — the exact shape #405 reports (2 x 6,000 against a 10,000
+/// cap). The old per-index check never fired here: neither `idx.search()`
+/// call alone hit `SCROLL_SNAPSHOT_MAX_HITS`, so `scroll_truncated` stayed
+/// false while the context silently pinned all 12,000 hits.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn alias_route_refuses_a_summed_total_over_the_cap() {
+ let (app, _dir) = app().await;
+ let per_index = SCROLL_SNAPSHOT_MAX_HITS / 2 + 1; // 5,001 each -> 10,002 summed
+ seed(&app, "sa", per_index).await;
+ seed(&app, "sb", per_index).await;
+
+ let (status, body) = open_alias_scroll(&app, "sa,sb", 100).await;
+ assert_eq!(
+ status,
+ StatusCode::BAD_REQUEST,
+ "a {}-document combined scroll (2 x {per_index}, each under the {SCROLL_SNAPSHOT_MAX_HITS} \
+ cap individually) must be refused on the alias route exactly as it already is on \
+ `_search?scroll=`, not silently pinned into an under-reporting context: {body}",
+ per_index * 2
+ );
+ let reason = body["error"]["reason"]
+ .as_str()
+ .unwrap_or_else(|| panic!("400 has no error.reason: {body}"));
+ assert!(
+ reason.contains(&(per_index * 2).to_string()),
+ "the refusal must name the true summed total ({}), not a per-index one: {reason}",
+ per_index * 2
+ );
+}
+
+/// Control: the same two-index request, safely under the cap on each side
+/// AND in aggregate, must still succeed — without this, a tree that 400s
+/// every multi-index alias scroll would pass the assertion above vacuously.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn alias_route_multi_index_under_cap_still_scrolls() {
+ let (app, _dir) = app().await;
+ seed(&app, "sa", 128).await;
+ seed(&app, "sb", 128).await;
+
+ let (status, body) = open_alias_scroll(&app, "sa,sb", 50).await;
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "a 256-document combined scroll is well under the {SCROLL_SNAPSHOT_MAX_HITS} cap \
+ and must succeed: {body}"
+ );
+ assert!(
+ body["_scroll_id"].is_string(),
+ "under-cap alias scroll returned no _scroll_id: {body}"
+ );
+ assert_eq!(
+ body["hits"]["total"]["value"].as_u64(),
+ Some(256),
+ "under-cap alias scroll reported the wrong total: {body}"
+ );
+}
+
+/// A single index alone over the cap must still be refused on the alias
+/// route (the case the OLD per-index check already caught correctly) — this
+/// fix must not regress it while closing the summed-total gap.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn alias_route_still_refuses_a_single_index_over_the_cap() {
+ let (app, _dir) = app().await;
+ let over = SCROLL_SNAPSHOT_MAX_HITS + 1;
+ seed(&app, "solo", over).await;
+
+ let (status, body) = open_alias_scroll(&app, "solo", 100).await;
+ assert_eq!(
+ status,
+ StatusCode::BAD_REQUEST,
+ "a single {over}-document index alone over the cap must still be refused: {body}"
+ );
+}