AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): enforce the search.max_keep_alive cluster setting (#2937)
Amartya Vikram Singhad02526f9c1a
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 396e9aa13..17923db33 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -14389,6 +14389,10 @@ async fn search_impl(// below only fills in when the body value is None, so it cannot// undo this.body.track_total_hits = Some(Value::Bool(true));+ if let Some(refused) = keep_alive_over_cluster_limit(&state, params.scroll.as_deref()).await+ {+ return refused;+ }}// `?sort=field:order` → convert to sort array.if let Some(ref sort_str) = params.sort {@@ -27776,6 +27780,10 @@ pub async fn search_with_scroll(if state.engine.scrolls.len() >= scroll_cap {return too_many_scroll_contexts(scroll_cap);}+ if let Some(refused) = keep_alive_over_cluster_limit(&state, params.scroll.as_deref()).await+ {+ return refused;+ }}// Resolve aliases and expand comma-separated indices.@@ -28282,6 +28290,16 @@ async fn scroll_page_response(if Uuid::parse_str(&scroll_id).is_err() {return cannot_parse_scroll_id();}+ let live = state+ .engine+ .scrolls+ .get(&scroll_id)+ .is_some_and(|ctx| ctx.expires_at > Instant::now());+ if live {+ if let Some(refused) = keep_alive_over_cluster_limit(state, scroll_keep_alive).await {+ return refused;+ }+ }match state.engine.scrolls.get_mut(&scroll_id) {Some(mut ctx) => {@@ -44444,6 +44462,10 @@ pub async fn open_pit(Query(params): Query<PitQueryParams>,body: OptionalJson<Value>,) -> impl IntoResponse {+ if let Some(refused) = keep_alive_over_cluster_limit(&state, params.keep_alive.as_deref()).await+ {+ return refused;+ }let pit_id = Uuid::new_v4().to_string();let body_val = body.0.unwrap_or(Value::Null);// Resolve keep_alive: parse `?keep_alive=5m` (ES syntax) or fall@@ -44863,6 +44885,273 @@ fn strip_index_constraints(q: &mut Value, out: &mut Vec<String>) {/// Used by `_pit?keep_alive=5m` (and any other endpoint that accepts/// the same shape). Sub-second units round up to 1 second so/// `keep_alive=500ms` doesn't produce a zero-duration PIT.+/// A time value in ES's syntax (`500ms`, `30s`, `1m`, `2h`, `1d`, also+/// `micros`/`nanos`) as milliseconds. `None` for anything else, including a+/// bare number, which ES refuses for a time setting.+fn parse_time_value_millis(s: &str) -> Option<u64> {+ let s = s.trim();+ let (digits, per_unit_ns): (&str, u128) = [+ ("nanos", 1u128),+ ("micros", 1_000),+ ("ms", 1_000_000),+ ("s", 1_000_000_000),+ ("m", 60_000_000_000),+ ("h", 3_600_000_000_000),+ ("d", 86_400_000_000_000),+ ]+ .iter()+ .find_map(|(suffix, ns)| s.strip_suffix(suffix).map(|d| (d, *ns)))?;+ let n: u128 = digits.trim().parse().ok()?;+ u64::try_from(n.saturating_mul(per_unit_ns) / 1_000_000).ok()+}++/// Milliseconds the way ES's `TimeValue#toString` prints them: the largest+/// whole unit, one truncated decimal, a trailing `.0` dropped (`120000` ->+/// `2m`, `90000` -> `1.5m`).+fn format_time_value_millis(ms: u64) -> String {+ if ms == 0 {+ return "0s".to_string();+ }+ let (unit, suffix) = [+ (86_400_000u64, "d"),+ (3_600_000, "h"),+ (60_000, "m"),+ (1_000, "s"),+ (1, "ms"),+ ]+ .into_iter()+ .find(|(unit, _)| ms >= *unit)+ .unwrap_or((1, "ms"));+ let whole = ms / unit;+ let tenth = (ms % unit) * 10 / unit;+ if tenth == 0 {+ format!("{whole}{suffix}")+ } else {+ format!("{whole}.{tenth}{suffix}")+ }+}++/// ES's `search.max_keep_alive` check (`SearchService#checkKeepAliveLimit`):+/// a scroll or point-in-time keep-alive above the cluster setting is refused+/// with a 400, on open and on every continuation that asks for a new+/// keep-alive. XERJ stored the setting (`PUT _cluster/settings` round-trips+/// it) but never read it, so a cluster that lowered the limit still accepted+/// any keep-alive up to the node's own `search_context` / `pit` cap.+///+/// Only an explicitly set limit is enforced here, transient over persistent+/// as in ES. With none set, the node config's cap keeps applying silently, as+/// its documentation says; that cap is an operator guard, not ES's setting.+async fn keep_alive_over_cluster_limit(+ state: &AppState,+ requested: Option<&str>,+) -> Option<Response> {+ let requested_ms = parse_time_value_millis(requested?)?;+ let max_ms = {+ let settings = state.engine.cluster_settings.read().await;+ ["transient", "persistent"].iter().find_map(|scope| {+ settings+ .get(*scope)+ .and_then(|s| s.get("search.max_keep_alive"))+ .and_then(Value::as_str)+ .and_then(parse_time_value_millis)+ })?+ };+ (requested_ms > max_ms).then(|| {+ search_shard_validation_error(&format!(+ "Keep alive for request ({}) is too large. It must be less than ({}). This limit can \+ be set by changing the [search.max_keep_alive] cluster level setting.",+ format_time_value_millis(requested_ms),+ format_time_value_millis(max_ms),+ ))+ })+}++#[cfg(test)]+mod max_keep_alive_tests {+ use super::*;+ use axum::{body::Body, http::Request};+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ async fn send(app: &axum::Router, method: &str, uri: &str, body: Value) -> (StatusCode, Value) {+ let resp = app+ .clone()+ .oneshot(+ Request::builder()+ .method(method)+ .uri(uri)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .unwrap(),+ )+ .await+ .unwrap();+ let status = resp.status();+ let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, serde_json::from_slice(&bytes).unwrap())+ }++ async fn app_with_limit(scope: &str, max: &str) -> (AppState, axum::Router) {+ let state = test_state();+ let app = crate::router::build_es_compat_router(state.clone());+ for id in ["1", "2"] {+ let (status, body) = send(+ &app,+ "PUT",+ &format!("/k/_doc/{id}?refresh=true"),+ json!({"foo": 1}),+ )+ .await;+ assert!(status.is_success(), "{body}");+ }+ let (status, body) = send(+ &app,+ "PUT",+ "/_cluster/settings",+ json!({ scope: {"search.max_keep_alive": max} }),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ (state, app)+ }++ fn reason(resp: &Value) -> &str {+ resp["error"]["root_cause"][0]["reason"]+ .as_str()+ .unwrap_or_default()+ }++ const TOO_LARGE_2M: &str = "Keep alive for request (2m) is too large. It must be less than \+ (1m). This limit can be set by changing the \+ [search.max_keep_alive] cluster level setting.";++ /// FAIL-BEFORE: with `search.max_keep_alive: 1m`, each of these answered+ /// 200 and opened (or re-armed) a context for the longer keep-alive.+ #[tokio::test]+ async fn a_keep_alive_over_the_cluster_limit_is_a_400() {+ let (state, app) = app_with_limit("persistent", "1m").await;++ let (status, resp) = send(&app, "POST", "/k/_search?scroll=2m", json!({"size": 1})).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");+ assert_eq!(reason(&resp), TOO_LARGE_2M);+ let (status, resp) = send(+ &app,+ "POST",+ "/k/_search_scroll?scroll=2m",+ json!({"size": 1}),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");+ assert_eq!(reason(&resp), TOO_LARGE_2M);+ assert!(+ state.engine.scrolls.is_empty(),+ "a refused open leaves no context"+ );++ // At the limit the scroll opens; a continuation asking for more is+ // refused, and the context survives it.+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=1m", json!({"size": 1})).await;+ assert_eq!(status, StatusCode::OK, "{resp}");+ let id = resp["_scroll_id"].as_str().expect("scroll id").to_string();+ let (status, resp) = send(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "3m", "scroll_id": id}),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");+ assert!(+ reason(&resp).starts_with("Keep alive for request (3m) is too large."),+ "{resp}"+ );+ let (status, resp) = send(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "30s", "scroll_id": id}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{resp}");++ let (status, resp) = send(&app, "POST", "/k/_pit?keep_alive=90s", json!({})).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{resp}");+ assert!(+ reason(&resp).starts_with("Keep alive for request (1.5m) is too large."),+ "{resp}"+ );+ }++ /// Transient wins over persistent, as in ES; removing the limit restores+ /// the node's default behaviour.+ #[tokio::test]+ async fn the_effective_limit_follows_the_cluster_settings() {+ let (_state, app) = app_with_limit("persistent", "1m").await;+ let (status, body) = send(+ &app,+ "PUT",+ "/_cluster/settings",+ json!({"transient": {"search.max_keep_alive": "10m"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=5m", json!({"size": 1})).await;+ assert_eq!(status, StatusCode::OK, "transient 10m must win: {resp}");++ let clear = json!({"persistent": {"search.max_keep_alive": null},+ "transient": {"search.max_keep_alive": null}});+ let (status, body) = send(&app, "PUT", "/_cluster/settings", clear).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, resp) = send(&app, "POST", "/k/_search?scroll=2h", json!({"size": 1})).await;+ assert_eq!(status, StatusCode::OK, "{resp}");+ let (status, resp) = send(&app, "POST", "/k/_pit?keep_alive=2h", json!({})).await;+ assert_eq!(status, StatusCode::OK, "{resp}");+ }++ #[test]+ fn time_values_parse_and_print_like_es() {+ for (raw, ms) in [+ ("500ms", Some(500)),+ ("30s", Some(30_000)),+ ("1m", Some(60_000)),+ ("2h", Some(7_200_000)),+ ("1d", Some(86_400_000)),+ ("1500000micros", Some(1_500)),+ ("60", None),+ ("abc", None),+ ] {+ assert_eq!(parse_time_value_millis(raw), ms, "{raw}");+ }+ for (ms, printed) in [+ (0, "0s"),+ (500, "500ms"),+ (1_500, "1.5s"),+ (60_000, "1m"),+ (90_000, "1.5m"),+ (119_999, "1.9m"),+ (86_400_000, "1d"),+ ] {+ assert_eq!(format_time_value_millis(ms), printed, "{ms}");+ }+ }+}+fn parse_keep_alive_to_secs(s: &str) -> Option<u64> {let s = s.trim();if s.is_empty() {diff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txtindex 3216203ae..475ce4477 100644--- a/engine/tests/es-compat-yaml/catch-lenient.txt+++ b/engine/tests/es-compat-yaml/catch-lenient.txt@@ -50,7 +50,6 @@ indices/indices.put_template/10_basic.yml · Put template createscroll/11_clear.yml · Body params with array param override query stringscroll/11_clear.yml · Body params with string param scroll id override query stringscroll/11_clear.yml · Clear scroll-scroll/20_keep_alive.yml · Max keep alivesearch/110_field_collapsing.yml · field collapsing and scrollsearch/110_field_collapsing.yml · field collapsing and search_after with invalid sortsearch/110_field_collapsing.yml · Test collapse with inner_hits and missing name failsdiff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rsindex 88a3074ea..96feb4965 100644--- a/engine/tests/es-compat-yaml/src/main.rs+++ b/engine/tests/es-compat-yaml/src/main.rs@@ -232,6 +232,7 @@ fn run_file(.to_string();let mut setup_steps: Vec<YamlValue> = Vec::new();+ let mut teardown_steps: Vec<YamlValue> = Vec::new();let mut tests: Vec<(String, Vec<YamlValue>)> = Vec::new();for doc in &docs {@@ -242,6 +243,16 @@ fn run_file(}continue;}+ // ES runs `teardown` after every case of the file, pass or fail.+ // It used to be collected as one more test named "teardown",+ // which ran once, before the cases it was meant to clean up+ // after, so a cluster setting a case changed leaked into every+ // later file (`scroll/20_keep_alive.yml` resets+ // `search.max_keep_alive` there).+ if let Some(YamlValue::Sequence(steps)) = m.get(YamlValue::String("teardown".into())) {+ teardown_steps = steps.clone();+ continue;+ }for (key, val) in m {if let (YamlValue::String(name), YamlValue::Sequence(steps)) = (key, val) {tests.push((name.clone(), steps.clone()));@@ -282,6 +293,14 @@ fn run_file(}let result = run_test_case(client, base_url, &[], steps, verbose);+ run_teardown(+ client,+ base_url,+ &teardown_steps,+ &short_name,+ name,+ verbose,+ );match result {TestResult::Pass => {if verbose {@@ -315,6 +334,34 @@ fn run_file(}}+/// A teardown step that fails does not change the case's result (the case+/// already ran); it is reported under `--verbose`, and the suite wipe at the+/// start of the next case still runs.+fn run_teardown(+ client: &reqwest::blocking::Client,+ base_url: &str,+ steps: &[YamlValue],+ short_name: &str,+ name: &str,+ verbose: bool,+) {+ let mut response = Value::Null;+ let mut vars: HashMap<String, String> = HashMap::new();+ for step in steps {+ if let Err(e) = execute_step(client, base_url, step, &mut response, &mut vars) {+ if verbose {+ println!(+ " {} {} · {} — teardown: {}",+ "WARN".yellow(),+ short_name,+ name,+ e+ );+ }+ }+ }+}+enum TestResult {Pass,Fail(String),