Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): enforce the search.max_keep_alive cluster setting (#2937)

Amartya Vikram Singhad02526f9c1a
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 396e9aa13..17923db33 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -14389,6 +14389,10 @@ async fn search_impl(
// below only fills in when the body value is None, so it cannot
// undo this.
body.track_total_hits = Some(Value::Bool(true));
+ if let Some(refused) = keep_alive_over_cluster_limit(&state, params.scroll.as_deref()).await
+ {
+ return refused;
+ }
}
// `?sort=field:order` → convert to sort array.
if let Some(ref sort_str) = params.sort {
@@ -27776,6 +27780,10 @@ pub async fn search_with_scroll(
if state.engine.scrolls.len() >= scroll_cap {
return too_many_scroll_contexts(scroll_cap);
}
+ if let Some(refused) = keep_alive_over_cluster_limit(&state, params.scroll.as_deref()).await
+ {
+ return refused;
+ }
}
// Resolve aliases and expand comma-separated indices.
@@ -28282,6 +28290,16 @@ async fn scroll_page_response(
if Uuid::parse_str(&scroll_id).is_err() {
return cannot_parse_scroll_id();
}
+ let live = state
+ .engine
+ .scrolls
+ .get(&scroll_id)
+ .is_some_and(|ctx| ctx.expires_at > Instant::now());
+ if live {
+ if let Some(refused) = keep_alive_over_cluster_limit(state, scroll_keep_alive).await {
+ return refused;
+ }
+ }
match state.engine.scrolls.get_mut(&scroll_id) {
Some(mut ctx) => {
@@ -44444,6 +44462,10 @@ pub async fn open_pit(
Query(params): Query<PitQueryParams>,
body: OptionalJson<Value>,
) -> impl IntoResponse {
+ if let Some(refused) = keep_alive_over_cluster_limit(&state, params.keep_alive.as_deref()).await
+ {
+ return refused;
+ }
let pit_id = Uuid::new_v4().to_string();
let body_val = body.0.unwrap_or(Value::Null);
// Resolve keep_alive: parse `?keep_alive=5m` (ES syntax) or fall
@@ -44863,6 +44885,273 @@ fn strip_index_constraints(q: &mut Value, out: &mut Vec<String>) {
/// Used by `_pit?keep_alive=5m` (and any other endpoint that accepts
/// the same shape). Sub-second units round up to 1 second so
/// `keep_alive=500ms` doesn't produce a zero-duration PIT.
+/// A time value in ES's syntax (`500ms`, `30s`, `1m`, `2h`, `1d`, also
+/// `micros`/`nanos`) as milliseconds. `None` for anything else, including a
+/// bare number, which ES refuses for a time setting.
+fn parse_time_value_millis(s: &str) -> Option<u64> {
+ let s = s.trim();
+ let (digits, per_unit_ns): (&str, u128) = [
+ ("nanos", 1u128),
+ ("micros", 1_000),
+ ("ms", 1_000_000),
+ ("s", 1_000_000_000),
+ ("m", 60_000_000_000),
+ ("h", 3_600_000_000_000),
+ ("d", 86_400_000_000_000),
+ ]
+ .iter()
+ .find_map(|(suffix, ns)| s.strip_suffix(suffix).map(|d| (d, *ns)))?;
+ let n: u128 = digits.trim().parse().ok()?;
+ u64::try_from(n.saturating_mul(per_unit_ns) / 1_000_000).ok()
+}
+
+/// Milliseconds the way ES's `TimeValue#toString` prints them: the largest
+/// whole unit, one truncated decimal, a trailing `.0` dropped (`120000` ->
+/// `2m`, `90000` -> `1.5m`).
+fn format_time_value_millis(ms: u64) -> String {
+ if ms == 0 {
+ return "0s".to_string();
+ }
+ let (unit, suffix) = [
+ (86_400_000u64, "d"),
+ (3_600_000, "h"),
+ (60_000, "m"),
+ (1_000, "s"),
+ (1, "ms"),
+ ]
+ .into_iter()
+ .find(|(unit, _)| ms >= *unit)
+ .unwrap_or((1, "ms"));
+ let whole = ms / unit;
+ let tenth = (ms % unit) * 10 / unit;
+ if tenth == 0 {
+ format!("{whole}{suffix}")
+ } else {
+ format!("{whole}.{tenth}{suffix}")
+ }
+}
+
+/// ES's `search.max_keep_alive` check (`SearchService#checkKeepAliveLimit`):
+/// a scroll or point-in-time keep-alive above the cluster setting is refused
+/// with a 400, on open and on every continuation that asks for a new
+/// keep-alive. XERJ stored the setting (`PUT _cluster/settings` round-trips
+/// it) but never read it, so a cluster that lowered the limit still accepted
+/// any keep-alive up to the node's own `search_context` / `pit` cap.
+///
+/// Only an explicitly set limit is enforced here, transient over persistent
+/// as in ES. With none set, the node config's cap keeps applying silently, as
+/// its documentation says; that cap is an operator guard, not ES's setting.
+async fn keep_alive_over_cluster_limit(
+ state: &AppState,
+ requested: Option<&str>,
+) -> Option<Response> {
+ let requested_ms = parse_time_value_millis(requested?)?;
+ let max_ms = {
+ let settings = state.engine.cluster_settings.read().await;
+ ["transient", "persistent"].iter().find_map(|scope| {
+ settings
+ .get(*scope)
+ .and_then(|s| s.get("search.max_keep_alive"))
+ .and_then(Value::as_str)
+ .and_then(parse_time_value_millis)
+ })?
+ };
+ (requested_ms > max_ms).then(|| {
+ search_shard_validation_error(&format!(
+ "Keep alive for request ({}) is too large. It must be less than ({}). This limit can \
+ be set by changing the [search.max_keep_alive] cluster level setting.",
+ format_time_value_millis(requested_ms),
+ format_time_value_millis(max_ms),
+ ))
+ })
+}
+
+#[cfg(test)]
+mod max_keep_alive_tests {
+ use super::*;
+ use axum::{body::Body, http::Request};
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ async fn send(app: &axum::Router, method: &str, uri: &str, body: Value) -> (StatusCode, Value) {
+ let resp = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(uri)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = resp.status();
+ let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (status, serde_json::from_slice(&bytes).unwrap())
+ }
+
+ async fn app_with_limit(scope: &str, max: &str) -> (AppState, axum::Router) {
+ let state = test_state();
+ let app = crate::router::build_es_compat_router(state.clone());
+ for id in ["1", "2"] {
+ let (status, body) = send(
+ &app,
+ "PUT",
+ &format!("/k/_doc/{id}?refresh=true"),
+ json!({"foo": 1}),
+ )
+ .await;
+ assert!(status.is_success(), "{body}");
+ }
+ let (status, body) = send(
+ &app,
+ "PUT",
+ "/_cluster/settings",
+ json!({ scope: {"search.max_keep_alive": max} }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ (state, app)
+ }
+
+ fn reason(resp: &Value) -> &str {
+ resp["error"]["root_cause"][0]["reason"]
+ .as_str()
+ .unwrap_or_default()
+ }
+
+ const TOO_LARGE_2M: &str = "Keep alive for request (2m) is too large. It must be less than \
+ (1m). This limit can be set by changing the \
+ [search.max_keep_alive] cluster level setting.";
+
+ /// FAIL-BEFORE: with `search.max_keep_alive: 1m`, each of these answered
+ /// 200 and opened (or re-armed) a context for the longer keep-alive.
+ #[tokio::test]
+ async fn a_keep_alive_over_the_cluster_limit_is_a_400() {
+ let (state, app) = app_with_limit("persistent", "1m").await;
+
+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=2m", json!({"size": 1})).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");
+ assert_eq!(reason(&resp), TOO_LARGE_2M);
+ let (status, resp) = send(
+ &app,
+ "POST",
+ "/k/_search_scroll?scroll=2m",
+ json!({"size": 1}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");
+ assert_eq!(reason(&resp), TOO_LARGE_2M);
+ assert!(
+ state.engine.scrolls.is_empty(),
+ "a refused open leaves no context"
+ );
+
+ // At the limit the scroll opens; a continuation asking for more is
+ // refused, and the context survives it.
+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=1m", json!({"size": 1})).await;
+ assert_eq!(status, StatusCode::OK, "{resp}");
+ let id = resp["_scroll_id"].as_str().expect("scroll id").to_string();
+ let (status, resp) = send(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "3m", "scroll_id": id}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");
+ assert!(
+ reason(&resp).starts_with("Keep alive for request (3m) is too large."),
+ "{resp}"
+ );
+ let (status, resp) = send(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "30s", "scroll_id": id}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{resp}");
+
+ let (status, resp) = send(&app, "POST", "/k/_pit?keep_alive=90s", json!({})).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");
+ assert!(
+ reason(&resp).starts_with("Keep alive for request (1.5m) is too large."),
+ "{resp}"
+ );
+ }
+
+ /// Transient wins over persistent, as in ES; removing the limit restores
+ /// the node's default behaviour.
+ #[tokio::test]
+ async fn the_effective_limit_follows_the_cluster_settings() {
+ let (_state, app) = app_with_limit("persistent", "1m").await;
+ let (status, body) = send(
+ &app,
+ "PUT",
+ "/_cluster/settings",
+ json!({"transient": {"search.max_keep_alive": "10m"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=5m", json!({"size": 1})).await;
+ assert_eq!(status, StatusCode::OK, "transient 10m must win: {resp}");
+
+ let clear = json!({"persistent": {"search.max_keep_alive": null},
+ "transient": {"search.max_keep_alive": null}});
+ let (status, body) = send(&app, "PUT", "/_cluster/settings", clear).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=2h", json!({"size": 1})).await;
+ assert_eq!(status, StatusCode::OK, "{resp}");
+ let (status, resp) = send(&app, "POST", "/k/_pit?keep_alive=2h", json!({})).await;
+ assert_eq!(status, StatusCode::OK, "{resp}");
+ }
+
+ #[test]
+ fn time_values_parse_and_print_like_es() {
+ for (raw, ms) in [
+ ("500ms", Some(500)),
+ ("30s", Some(30_000)),
+ ("1m", Some(60_000)),
+ ("2h", Some(7_200_000)),
+ ("1d", Some(86_400_000)),
+ ("1500000micros", Some(1_500)),
+ ("60", None),
+ ("abc", None),
+ ] {
+ assert_eq!(parse_time_value_millis(raw), ms, "{raw}");
+ }
+ for (ms, printed) in [
+ (0, "0s"),
+ (500, "500ms"),
+ (1_500, "1.5s"),
+ (60_000, "1m"),
+ (90_000, "1.5m"),
+ (119_999, "1.9m"),
+ (86_400_000, "1d"),
+ ] {
+ assert_eq!(format_time_value_millis(ms), printed, "{ms}");
+ }
+ }
+}
+
fn parse_keep_alive_to_secs(s: &str) -> Option<u64> {
let s = s.trim();
if s.is_empty() {
diff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txt
index 3216203ae..475ce4477 100644
--- a/engine/tests/es-compat-yaml/catch-lenient.txt
+++ b/engine/tests/es-compat-yaml/catch-lenient.txt
@@ -50,7 +50,6 @@ indices/indices.put_template/10_basic.yml · Put template create
scroll/11_clear.yml · Body params with array param override query string
scroll/11_clear.yml · Body params with string param scroll id override query string
scroll/11_clear.yml · Clear scroll
-scroll/20_keep_alive.yml · Max keep alive
search/110_field_collapsing.yml · field collapsing and scroll
search/110_field_collapsing.yml · field collapsing and search_after with invalid sort
search/110_field_collapsing.yml · Test collapse with inner_hits and missing name fails
diff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rs
index 88a3074ea..96feb4965 100644
--- a/engine/tests/es-compat-yaml/src/main.rs
+++ b/engine/tests/es-compat-yaml/src/main.rs
@@ -232,6 +232,7 @@ fn run_file(
.to_string();
let mut setup_steps: Vec<YamlValue> = Vec::new();
+ let mut teardown_steps: Vec<YamlValue> = Vec::new();
let mut tests: Vec<(String, Vec<YamlValue>)> = Vec::new();
for doc in &docs {
@@ -242,6 +243,16 @@ fn run_file(
}
continue;
}
+ // ES runs `teardown` after every case of the file, pass or fail.
+ // It used to be collected as one more test named "teardown",
+ // which ran once, before the cases it was meant to clean up
+ // after, so a cluster setting a case changed leaked into every
+ // later file (`scroll/20_keep_alive.yml` resets
+ // `search.max_keep_alive` there).
+ if let Some(YamlValue::Sequence(steps)) = m.get(YamlValue::String("teardown".into())) {
+ teardown_steps = steps.clone();
+ continue;
+ }
for (key, val) in m {
if let (YamlValue::String(name), YamlValue::Sequence(steps)) = (key, val) {
tests.push((name.clone(), steps.clone()));
@@ -282,6 +293,14 @@ fn run_file(
}
let result = run_test_case(client, base_url, &[], steps, verbose);
+ run_teardown(
+ client,
+ base_url,
+ &teardown_steps,
+ &short_name,
+ name,
+ verbose,
+ );
match result {
TestResult::Pass => {
if verbose {
@@ -315,6 +334,34 @@ fn run_file(
}
}
+/// A teardown step that fails does not change the case's result (the case
+/// already ran); it is reported under `--verbose`, and the suite wipe at the
+/// start of the next case still runs.
+fn run_teardown(
+ client: &reqwest::blocking::Client,
+ base_url: &str,
+ steps: &[YamlValue],
+ short_name: &str,
+ name: &str,
+ verbose: bool,
+) {
+ let mut response = Value::Null;
+ let mut vars: HashMap<String, String> = HashMap::new();
+ for step in steps {
+ if let Err(e) = execute_step(client, base_url, step, &mut response, &mut vars) {
+ if verbose {
+ println!(
+ " {} {} · {} — teardown: {}",
+ "WARN".yellow(),
+ short_name,
+ name,
+ e
+ );
+ }
+ }
+ }
+}
+
enum TestResult {
Pass,
Fail(String),